mirror of
https://github.com/minio/minio.git
synced 2025-11-06 20:33:07 -05:00
SQL select query for CSV/JSON (#6648)
select * , select column names have been implemented for CSV. select * is implemented for JSON.
This commit is contained in:
Ashish Kumar Sinha
committed by
kannappanr
kannappanr
parent
acf46cc3b5
commit
c0b4bf0a3e
@@ -17,10 +17,11 @@
|
||||
package s3select
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/s3select/format"
|
||||
)
|
||||
|
||||
// Unit Test for the checkForDuplicates function.
|
||||
@@ -35,6 +36,7 @@ func TestCheckForDuplicates(t *testing.T) {
|
||||
{[]string{"name", "id", "last_name", "last_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), ErrAmbiguousFieldName},
|
||||
{[]string{"name", "id", "last_name", "another_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), nil},
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
err := checkForDuplicates(table.myReq, table.myHeaders, table.myDup, table.myLow)
|
||||
if err != table.myErr {
|
||||
@@ -43,106 +45,14 @@ func TestCheckForDuplicates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test for the function which processes columnnames to make sure that they are
|
||||
// compatible with spaces.
|
||||
func TestMyProcessing(t *testing.T) {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tables := []struct {
|
||||
myReq []string
|
||||
myHeaders map[string]int
|
||||
myDup map[string]bool
|
||||
myLow map[string]int
|
||||
myOpts *Options
|
||||
input *Input
|
||||
length int
|
||||
testOutput string
|
||||
myErr error
|
||||
}{
|
||||
{[]string{"name", "id", "last_name", "CAST"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 4, "CAST", nil},
|
||||
{[]string{"name", "id", "last_name", "another_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 4, "another_name", nil},
|
||||
{[]string{"name", "id", "last_name", "another_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 4, "another_name", nil},
|
||||
{[]string{"name", "id", "random_name", "fame_name", "another_col"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 5, "fame_name", nil},
|
||||
}
|
||||
for _, table := range tables {
|
||||
err = checkForDuplicates(table.myReq, table.myHeaders, table.myDup, table.myLow)
|
||||
if err != table.myErr {
|
||||
t.Error()
|
||||
}
|
||||
if len(table.myReq) != table.length {
|
||||
t.Errorf("UnexpectedError")
|
||||
}
|
||||
if table.myReq[3] != table.testOutput {
|
||||
t.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyRowIndexResults is a unit test which makes sure that the rows that are
|
||||
// being printed are appropriate to the query being requested.
|
||||
func TestMyRowIndexResults(t *testing.T) {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tables := []struct {
|
||||
myReq []string
|
||||
myHeaders map[string]int
|
||||
myDup map[string]bool
|
||||
myLow map[string]int
|
||||
myOpts *Options
|
||||
input *Input
|
||||
myRecord []string
|
||||
myTarget string
|
||||
myAsterix string
|
||||
columns []string
|
||||
err error
|
||||
}{
|
||||
{[]string{"1", "2"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"target", "random", "hello", "stuff"}, "target,random", "target,random,hello,stuff", []string{"1", "2", "3", "4"}, nil},
|
||||
{[]string{"2", "3", "4"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"random", "hullo", "thing", "stuff"}, "hullo,thing,stuff", "random,hullo,thing,stuff", []string{"1", "2", "3", "4"}, nil},
|
||||
{[]string{"3", "2"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"random", "hullo", "thing", "stuff"}, "thing,hullo", "random,hullo,thing,stuff", []string{"1", "2", "3", "4"}, nil},
|
||||
{[]string{"11", "1"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"random", "hullo", "thing", "stuff"}, "", "random,hullo,thing,stuff", []string{"1", "2", "3", "4"}, ErrInvalidColumnIndex},
|
||||
}
|
||||
for _, table := range tables {
|
||||
checkForDuplicates(table.columns, table.myHeaders, table.myDup, table.myLow)
|
||||
myRow, err := s3s.processColNameIndex(table.myRecord, table.myReq, table.columns)
|
||||
if err != table.err {
|
||||
t.Error()
|
||||
}
|
||||
if myRow != table.myTarget {
|
||||
t.Error()
|
||||
}
|
||||
myRow = table.input.printAsterix(table.myRecord)
|
||||
if myRow != table.myAsterix {
|
||||
t.Error()
|
||||
// This function returns the index of a string in a list
|
||||
func stringIndex(a string, list []string) int {
|
||||
for i, v := range list {
|
||||
if v == a {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// TestMyHelperFunctions is a unit test which tests some small helper string
|
||||
@@ -159,7 +69,7 @@ func TestMyHelperFunctions(t *testing.T) {
|
||||
{"test3", []string{"test1", "test2", "test3", "test4", "test5"}, 2, true},
|
||||
}
|
||||
for _, table := range tables {
|
||||
if stringInSlice(table.myReq, table.myList) != table.expected {
|
||||
if format.StringInSlice(table.myReq, table.myList) != table.expected {
|
||||
t.Error()
|
||||
}
|
||||
if stringIndex(table.myReq, table.myList) != table.myIndex {
|
||||
@@ -233,82 +143,6 @@ func TestMyConversion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Unit Tests for Parser.
|
||||
func TestMyParser(t *testing.T) {
|
||||
tables := []struct {
|
||||
myQuery string
|
||||
err error
|
||||
reqCols []string
|
||||
alias string
|
||||
myLimit int
|
||||
aggFuncs []string
|
||||
header []string
|
||||
}{
|
||||
{"SELECT * FROM S3OBJECT", nil, []string{"*"}, "S3OBJECT", 0, make([]string, 1), []string{"name1", "name2", "name3", "name4"}},
|
||||
{"SELECT * FROM S3OBJECT AS A", nil, []string{"*"}, "A", 0, make([]string, 1), []string{"name1", "name2", "name3", "name4"}},
|
||||
{"SELECT col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "name2", "name3", "name4"}},
|
||||
{"SELECT col_name,col_other FROM S3OBJECT AS A LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, make([]string, 2), []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT col_name,col_other FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, make([]string, 2), []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT col_name,col_other FROM S3OBJECT AS A WHERE col_name = 'Name LIMIT 5", ErrLexerInvalidChar, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT count(*) FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"*"}, "A", 5, []string{"count"}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT sum(col_name),sum(col_other) FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, []string{"sum", "sum"}, []string{"col_name", "col_other"}},
|
||||
{"SELECT A.col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT A.`col name` FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT A._col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT A._col_name FROM S3OBJECT AS A WHERE randomname > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT A._col_name FROM S3OBJECT AS A WHERE A._11 > 5", ErrInvalidColumnIndex, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE A._3 > 5", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT COALESCE(col_name,col_other),COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE A._3 > 5", nil, []string{"", ""}, "A", 0, []string{"", ""}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT COALESCE(col_name,col_other) ,col_name , COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE col_name > 5", nil, []string{"", "col_name", ""}, "A", 0, []string{"", "", ""}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT NULLIF(col_name,col_other) ,col_name , COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE col_name > 5", nil, []string{"", "col_name", ""}, "A", 0, []string{"", "", ""}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT NULLIF(col_name,col_other) FROM S3OBJECT AS A WHERE col_name > 5", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT NULLIF(randomname,col_other) FROM S3OBJECT AS A WHERE col_name > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT col_name FROM S3OBJECT AS A WHERE COALESCE(random,5) > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT col_name FROM S3OBJECT AS A WHERE NULLIF(random,5) > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT col_name FROM S3OBJECT AS A WHERE LOWER(col_name) BETWEEN 5 AND 7", nil, []string{"col_name"}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT UPPER(col_name) FROM S3OBJECT AS A WHERE LOWER(col_name) BETWEEN 5 AND 7", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT UPPER(*) FROM S3OBJECT AS A WHERE LOWER(col_name) BETWEEN 5 AND 7", ErrParseUnsupportedCallWithStar, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT NULLIF(col_name,col_name) FROM S3OBJECT AS A WHERE NULLIF(LOWER(col_name),col_name) BETWEEN 5 AND 7", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
{"SELECT COALESCE(col_name,col_name) FROM S3OBJECT AS A WHERE NULLIF(LOWER(col_name),col_name) BETWEEN 5 AND 7", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
|
||||
}
|
||||
for _, table := range tables {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
HeaderOpt: true,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
s3s.header = table.header
|
||||
reqCols, alias, myLimit, _, aggFunctionNames, _, err := s3s.ParseSelect(table.myQuery)
|
||||
if table.err != err {
|
||||
t.Error()
|
||||
}
|
||||
if !reflect.DeepEqual(reqCols, table.reqCols) {
|
||||
t.Error()
|
||||
}
|
||||
if alias != table.alias {
|
||||
t.Error()
|
||||
}
|
||||
if myLimit != int64(table.myLimit) {
|
||||
t.Error()
|
||||
}
|
||||
if !reflect.DeepEqual(table.aggFuncs, aggFunctionNames) {
|
||||
t.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Unit tests for the main function that performs aggreggation.
|
||||
func TestMyAggregationFunc(t *testing.T) {
|
||||
columnsMap := make(map[string]int)
|
||||
@@ -321,21 +155,22 @@ func TestMyAggregationFunc(t *testing.T) {
|
||||
columnsMap map[string]int
|
||||
storeReqCols []string
|
||||
storeFunctions []string
|
||||
record []string
|
||||
record string
|
||||
err error
|
||||
expectedVal float64
|
||||
}{
|
||||
{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"count"}, []string{"1", "2"}, nil, 11},
|
||||
{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"min"}, []string{"1", "2"}, nil, 1},
|
||||
{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"max"}, []string{"1", "2"}, nil, 10},
|
||||
{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"sum"}, []string{"1", "2"}, nil, 11},
|
||||
{1, 1, []float64{10}, columnsMap, []string{"Col1"}, []string{"avg"}, []string{"1", "2"}, nil, 5.500},
|
||||
{10, 5, []float64{0.000}, columnsMap, []string{"Col1"}, []string{"random"}, []string{"1", "2"}, ErrParseNonUnaryAgregateFunctionCall, 0},
|
||||
{0, 5, []float64{0}, columnsMap, []string{"0"}, []string{"count"}, []string{"1", "2"}, nil, 1},
|
||||
{10, 5, []float64{10}, columnsMap, []string{"1"}, []string{"min"}, []string{"1", "12"}, nil, 1},
|
||||
{10, 5, []float64{10, 11, 12, 13, 14}, columnsMap, []string{"Col1"}, []string{"count"}, "{\"Col1\":\"1\",\"Col2\":\"2\"}", nil, 11},
|
||||
{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"min"}, "{\"Col1\":\"1\",\"Col2\":\"2\"}", nil, 1},
|
||||
{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"max"}, "{\"Col1\":\"1\",\"Col2\":\"2\"}", nil, 10},
|
||||
{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"sum"}, "{\"Col1\":\"1\",\"Col2\":\"2\"}", nil, 11},
|
||||
{1, 1, []float64{10}, columnsMap, []string{"Col1"}, []string{"avg"}, "{\"Col1\":\"1\",\"Col2\":\"2\"}", nil, 5.500},
|
||||
{10, 5, []float64{0.0000}, columnsMap, []string{"Col1"}, []string{"random"}, "{\"Col1\":\"1\",\"Col2\":\"2\"}", ErrParseNonUnaryAgregateFunctionCall, 0},
|
||||
{0, 5, []float64{0}, columnsMap, []string{"0"}, []string{"count"}, "{\"Col1\":\"1\",\"Col2\":\"2\"}", nil, 1},
|
||||
{10, 5, []float64{10}, columnsMap, []string{"1"}, []string{"min"}, "{\"_1\":\"1\",\"_2\":\"2\"}", nil, 1},
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
err := aggregationFunctions(table.counter, table.filtrCount, table.myAggVals, table.columnsMap, table.storeReqCols, table.storeFunctions, table.record)
|
||||
err := aggregationFunctions(table.counter, table.filtrCount, table.myAggVals, table.storeReqCols, table.storeFunctions, table.record)
|
||||
if table.err != err {
|
||||
t.Error()
|
||||
}
|
||||
@@ -346,156 +181,6 @@ func TestMyAggregationFunc(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Unit Tests for the function which converts a float array to string.
|
||||
func TestToStringAgg(t *testing.T) {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
HeaderOpt: true,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tables := []struct {
|
||||
myAggVal []float64
|
||||
expected string
|
||||
}{
|
||||
{[]float64{10, 11, 12, 13, 14}, "10,11,12,13,14"},
|
||||
{[]float64{10, 11.3, 12, 13, 14}, "10,11.300000,12,13,14"},
|
||||
{[]float64{10.235, 11.3, 12, 13, 14}, "10.235000,11.300000,12,13,14"},
|
||||
{[]float64{10.235, 11.3, 12.123, 13.456, 14.789}, "10.235000,11.300000,12.123000,13.456000,14.789000"},
|
||||
{[]float64{10}, "10"},
|
||||
}
|
||||
for _, table := range tables {
|
||||
val := s3s.aggFuncToStr(table.myAggVal)
|
||||
if val != table.expected {
|
||||
t.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyRowColLiteralResults is a unit test which makes sure that the rows that
|
||||
// are being printed are appropriate to the query being requested.
|
||||
func TestMyRowColLiteralResults(t *testing.T) {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
HeaderOpt: true,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tables := []struct {
|
||||
myReq []string
|
||||
myHeaders map[string]int
|
||||
myDup map[string]bool
|
||||
myLow map[string]int
|
||||
myOpts *Options
|
||||
tempList []string
|
||||
input *Input
|
||||
myRecord []string
|
||||
myTarget string
|
||||
columns []string
|
||||
err error
|
||||
}{
|
||||
{[]string{"draft", "year"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"draft", "year"}, s3s, []string{"target", "random", "hello", "stuff"}, "target,random", []string{"draft", "year", "random", "another"}, nil},
|
||||
{[]string{"year", "draft"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"year", "draft"}, s3s, []string{"draft", "2012", "thing", "stuff"}, "2012,draft", []string{"draft", "year", "random", "another"}, nil},
|
||||
{[]string{"yearrandomstuff", "draft"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"yearrandomstuff", "draft"}, s3s, []string{"draft", "2012", "thing", "stuff"}, "", []string{"draft", "year", "random", "another"}, ErrMissingHeaders},
|
||||
{[]string{"draft", "randomstuff"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"yearrandomstuff", "draft"}, s3s, []string{"draft", "2012", "thing", "stuff"}, "", []string{"draft", "year", "random", "another"}, ErrMissingHeaders},
|
||||
}
|
||||
for _, table := range tables {
|
||||
checkForDuplicates(table.columns, table.myHeaders, table.myDup, table.myLow)
|
||||
myRow, err := table.input.processColNameLiteral(table.myRecord, table.myReq, table.tempList, table.myHeaders, nil)
|
||||
if err != table.err {
|
||||
t.Error()
|
||||
}
|
||||
if myRow != table.myTarget {
|
||||
t.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyWhereEval is a function which provides unit tests for the function
|
||||
// which evaluates the where clause.
|
||||
func TestMyWhereEval(t *testing.T) {
|
||||
columnsMap := make(map[string]int)
|
||||
columnsMap["Col1"] = 0
|
||||
columnsMap["Col2"] = 1
|
||||
tables := []struct {
|
||||
myQuery string
|
||||
record []string
|
||||
err error
|
||||
expected bool
|
||||
header []string
|
||||
}{
|
||||
{"SELECT * FROM S3OBJECT", []string{"record_1,record_2,record_3,record_4"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT WHERE Col1 < -1", []string{"0", "1"}, nil, false, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT WHERE Col1 < -1 OR Col2 > 15", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT WHERE Col1 > -1 AND Col2 > 15", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT WHERE Col1 > 1.00", []string{"151.0000", "12"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT WHERE Col1 > 100", []string{"random", "12"}, nil, false, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT WHERE Col1 BETWEEN 100 AND 0", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT WHERE Col1 BETWEEN 100.0 AND 0.0", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT AS A WHERE A.1 BETWEEN 160 AND 150", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT AS A WHERE A._1 BETWEEN 160 AND 0", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT AS A WHERE A._1 BETWEEN 0 AND 160", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT A._1 LIKE 'r%'", []string{"record_1,record_2,record_3,record_4"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT s._2 FROM S3Object s WHERE s._2 = 'Steven'", []string{"record_1", "Steven", "Steven", "record_4"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT AS A WHERE Col1 BETWEEN 0 AND 160", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT AS A WHERE Col1 BETWEEN 160 AND 0", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT AS A WHERE UPPER(Col1) BETWEEN 160 AND 0", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT AS A WHERE UPPER(Col1) = 'RANDOM'", []string{"random", "12"}, nil, true, []string{"Col1", "Col2"}},
|
||||
{"SELECT * FROM S3OBJECT AS A WHERE LOWER(UPPER(Col1) = 'random'", []string{"random", "12"}, nil, true, []string{"Col1", "Col2"}},
|
||||
}
|
||||
for _, table := range tables {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
HeaderOpt: true,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
s3s.header = table.header
|
||||
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
_, alias, _, whereClause, _, _, _ := s3s.ParseSelect(table.myQuery)
|
||||
myVal, err := matchesMyWhereClause(table.record, columnsMap, alias, whereClause)
|
||||
if table.err != err {
|
||||
t.Error()
|
||||
}
|
||||
if myVal != table.expected {
|
||||
t.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyStringComparator is a unit test which ensures that the appropriate
|
||||
// values are being compared for strings.
|
||||
func TestMyStringComparator(t *testing.T) {
|
||||
@@ -594,231 +279,13 @@ func TestMySizeFunction(t *testing.T) {
|
||||
{[]string{"test1", "test2", "test3", "test4", "test5"}, 30},
|
||||
}
|
||||
for _, table := range tables {
|
||||
if processSize(table.myRecord) != table.expected {
|
||||
if format.ProcessSize(table.myRecord) != table.expected {
|
||||
t.Error()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// TestInterpreter is a function which provides unit testing for the main
|
||||
// interpreter function.
|
||||
func TestInterpreter(t *testing.T) {
|
||||
tables := []struct {
|
||||
myQuery string
|
||||
myChan chan *Row
|
||||
err error
|
||||
header []string
|
||||
}{
|
||||
{"Select random from S3OBJECT", make(chan *Row), ErrMissingHeaders, []string{"name1", "name2", "name3", "name4"}},
|
||||
{"Select * from S3OBJECT as A WHERE name2 > 5.00", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
|
||||
{"Select * from S3OBJECT", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
|
||||
{"Select A_1 from S3OBJECT as A", make(chan *Row), nil, []string{"1", "2", "3", "4"}},
|
||||
{"Select count(*) from S3OBJECT", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
|
||||
{"Select * from S3OBJECT WHERE name1 > 5.00", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
|
||||
}
|
||||
for _, table := range tables {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
HeaderOpt: true,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
s3s.header = table.header
|
||||
reqCols, alias, myLimit, whereClause, aggFunctionNames, _, err := s3s.ParseSelect(table.myQuery)
|
||||
if err != table.err {
|
||||
t.Fatal()
|
||||
}
|
||||
if err == nil {
|
||||
go s3s.processSelectReq(reqCols, alias, whereClause, myLimit, aggFunctionNames, table.myChan, nil)
|
||||
select {
|
||||
case row, ok := <-table.myChan:
|
||||
if ok && len(row.record) > 0 {
|
||||
} else if ok && row.err != nil {
|
||||
if row.err != table.err {
|
||||
t.Error()
|
||||
}
|
||||
close(table.myChan)
|
||||
} else if !ok {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyXMLFunction is a function that provides unit testing for the XML
|
||||
// creating function.
|
||||
func TestMyXMLFunction(t *testing.T) {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
HeaderOpt: true,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tables := []struct {
|
||||
expectedStat int
|
||||
expectedProgress int
|
||||
}{
|
||||
{150, 156},
|
||||
}
|
||||
for _, table := range tables {
|
||||
myVal, _ := s3s.createStatXML()
|
||||
myOtherVal, _ := s3s.createProgressXML()
|
||||
if len(myVal) != table.expectedStat {
|
||||
t.Error()
|
||||
}
|
||||
if len(myOtherVal) != table.expectedProgress {
|
||||
fmt.Println(len(myOtherVal))
|
||||
t.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyProtocolFunction is a function which provides unit testing for several
|
||||
// of the functions which write the binary protocol.
|
||||
func TestMyProtocolFunction(t *testing.T) {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
HeaderOpt: true,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tables := []struct {
|
||||
payloadMsg string
|
||||
expectedRecord int
|
||||
expectedEnd int
|
||||
}{
|
||||
{"random payload", 115, 56},
|
||||
}
|
||||
for _, table := range tables {
|
||||
var currentMessage = &bytes.Buffer{}
|
||||
if len(s3s.writeRecordMessage(table.payloadMsg, currentMessage).Bytes()) != table.expectedRecord {
|
||||
t.Error()
|
||||
}
|
||||
currentMessage.Reset()
|
||||
if len(s3s.writeEndMessage(currentMessage).Bytes()) != table.expectedEnd {
|
||||
t.Error()
|
||||
}
|
||||
currentMessage.Reset()
|
||||
if len(s3s.writeContinuationMessage(currentMessage).Bytes()) != 57 {
|
||||
t.Error()
|
||||
}
|
||||
currentMessage.Reset()
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyInfoProtocolFunctions is a function which provides unit testing for the
|
||||
// stat and progress messages of the protocols.
|
||||
func TestMyInfoProtocolFunctions(t *testing.T) {
|
||||
options := &Options{
|
||||
HasHeader: true,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
myVal, _ := s3s.createStatXML()
|
||||
myOtherVal, _ := s3s.createProgressXML()
|
||||
|
||||
tables := []struct {
|
||||
payloadStatMsg string
|
||||
payloadProgressMsg string
|
||||
expectedStat int
|
||||
expectedProgress int
|
||||
}{
|
||||
{myVal, myOtherVal, 233, 243},
|
||||
}
|
||||
for _, table := range tables {
|
||||
var currBuf = &bytes.Buffer{}
|
||||
if len(s3s.writeStatMessage(table.payloadStatMsg, currBuf).Bytes()) != table.expectedStat {
|
||||
t.Error()
|
||||
}
|
||||
currBuf.Reset()
|
||||
if len(s3s.writeProgressMessage(table.payloadProgressMsg, currBuf).Bytes()) != table.expectedProgress {
|
||||
t.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyErrorProtocolFunctions is a function which provides unit testing for
|
||||
// the error message type of protocol.
|
||||
func TestMyErrorProtocolFunctions(t *testing.T) {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
HeaderOpt: true,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
tables := []struct {
|
||||
err error
|
||||
expectedError int
|
||||
}{
|
||||
{ErrInvalidCast, 248},
|
||||
{ErrTruncatedInput, 200},
|
||||
{ErrUnsupportedSyntax, 114},
|
||||
{ErrCSVParsingError, 157},
|
||||
}
|
||||
for _, table := range tables {
|
||||
var currentMessage = &bytes.Buffer{}
|
||||
if len(s3s.writeErrorMessage(table.err, currentMessage).Bytes()) != table.expectedError {
|
||||
t.Error()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
func TestMatch(t *testing.T) {
|
||||
testCases := []struct {
|
||||
pattern string
|
||||
@@ -1004,51 +471,6 @@ func TestMatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyValids is a unit test which ensures that the appropriate values are
|
||||
// being returned from the isValid... functions.
|
||||
func TestMyValids(t *testing.T) {
|
||||
|
||||
tables := []struct {
|
||||
myQuery string
|
||||
indexList []int
|
||||
myIndex int
|
||||
myValIndex bool
|
||||
header []string
|
||||
err error
|
||||
}{
|
||||
{"SELECT UPPER(NULLIF(draft_year,random_name))", []int{3, 5, 6, 7, 8, 9}, 3, true, []string{"draft_year", "random_name"}, nil},
|
||||
{"SELECT UPPER(NULLIF(draft_year,xandom_name))", []int{3, 5, 6, 7, 8, 9}, 3, true, []string{"draft_year", "random_name"}, ErrMissingHeaders},
|
||||
}
|
||||
for _, table := range tables {
|
||||
options := &Options{
|
||||
HasHeader: false,
|
||||
RecordDelimiter: "\n",
|
||||
FieldDelimiter: ",",
|
||||
Comments: "",
|
||||
Name: "S3Object", // Default table name for all objects
|
||||
ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
|
||||
Compressed: "",
|
||||
Expression: "",
|
||||
OutputFieldDelimiter: ",",
|
||||
StreamSize: 20,
|
||||
HeaderOpt: true,
|
||||
}
|
||||
s3s, err := NewInput(options)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
s3s.header = table.header
|
||||
_, _, _, _, _, _, err = s3s.ParseSelect(table.myQuery)
|
||||
if err != table.err {
|
||||
t.Fatal()
|
||||
}
|
||||
myVal := isValidFunc(table.indexList, table.myIndex)
|
||||
if myVal != table.myValIndex {
|
||||
t.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMyFuncProcessing is a unit test which ensures that the appropriate values are
|
||||
// being returned from the Processing... functions.
|
||||
func TestMyFuncProcessing(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user