select: Add IS (NOT) operators (#13906)

Add `IS` and `IS NOT` as comparison operators.

This may be a bit wider than the S3 spec, but we can rather 
easily remove the forwarding.

internal/s3select/select_test.go

@@ -683,71 +683,117 @@ func TestCSVQueries(t *testing.T) {
 }
 
 func TestCSVQueries2(t *testing.T) {
-	input := `id,time,num,num2,text
+	testInput := []byte(`id,time,num,num2,text
 1,2010-01-01T,7867786,4565.908123,"a text, with comma"
 2,2017-01-02T03:04Z,-5, 0.765111,
-`
+`)
 	var testTable = []struct {
 		name       string
 		query      string
+		input      []byte
 		requestXML []byte // override request XML
 		wantResult string
 	}{
 		{
 			name:       "select-all",
+			input:      testInput,
 			query:      `SELECT * from s3object AS s WHERE id = '1'`,
 			wantResult: `{"id":"1","time":"2010-01-01T","num":"7867786","num2":"4565.908123","text":"a text, with comma"}`,
 		},
 		{
 			name:       "select-all-2",
+			input:      testInput,
 			query:      `SELECT * from s3object s WHERE id = 2`,
 			wantResult: `{"id":"2","time":"2017-01-02T03:04Z","num":"-5","num2":" 0.765111","text":""}`,
 		},
 		{
 			name:       "select-text-convert",
+			input:      testInput,
 			query:      `SELECT CAST(text AS STRING) AS text from s3object s WHERE id = 1`,
 			wantResult: `{"text":"a text, with comma"}`,
 		},
 		{
 			name:       "select-text-direct",
+			input:      testInput,
 			query:      `SELECT text from s3object s WHERE id = 1`,
 			wantResult: `{"text":"a text, with comma"}`,
 		},
 		{
 			name:       "select-time-direct",
+			input:      testInput,
 			query:      `SELECT time from s3object s WHERE id = 2`,
 			wantResult: `{"time":"2017-01-02T03:04Z"}`,
 		},
 		{
 			name:       "select-int-direct",
+			input:      testInput,
 			query:      `SELECT num from s3object s WHERE id = 2`,
 			wantResult: `{"num":"-5"}`,
 		},
 		{
 			name:       "select-float-direct",
+			input:      testInput,
 			query:      `SELECT num2 from s3object s WHERE id = 2`,
 			wantResult: `{"num2":" 0.765111"}`,
 		},
 		{
 			name:       "select-in-array",
+			input:      testInput,
 			query:      `select id from S3Object s WHERE id in [1,3]`,
 			wantResult: `{"id":"1"}`,
 		},
 		{
 			name:       "select-in-array-matchnone",
+			input:      testInput,
 			query:      `select id from S3Object s WHERE s.id in [4,3]`,
 			wantResult: ``,
 		},
 		{
 			name:       "select-float-by-val",
+			input:      testInput,
 			query:      `SELECT num2 from s3object s WHERE num2 = 0.765111`,
 			wantResult: `{"num2":" 0.765111"}`,
 		},
 		{
 			name:       "select-non_exiting_values",
+			input:      testInput,
 			query:      `SELECT _1 as first, s._100 from s3object s LIMIT 1`,
 			wantResult: `{"first":"1","_100":null}`,
 		},
+		{
+			name:       "select-is_null_noresults",
+			input:      testInput,
+			query:      `select _2 from S3object where _2 IS NULL`,
+			wantResult: ``,
+		},
+		{
+			name:  "select-is_null_results",
+			input: testInput,
+			query: `select _2 from S3object WHERE _100 IS NULL`,
+			wantResult: `{"_2":"2010-01-01T"}
+{"_2":"2017-01-02T03:04Z"}`,
+		},
+		{
+			name:  "select-is_not_null_results",
+			input: testInput,
+			query: `select _2 from S3object where _2 IS NOT NULL`,
+			wantResult: `{"_2":"2010-01-01T"}
+{"_2":"2017-01-02T03:04Z"}`,
+		},
+		{
+			name:       "select-is_not_null_noresults",
+			input:      testInput,
+			query:      `select _2 from S3object WHERE _100 IS NOT NULL`,
+			wantResult: ``,
+		},
+		{
+			name: "select-is_not_string",
+			input: []byte(`c1,c2,c3
+1,2,3
+1,,3`),
+			query:      `select * from S3object where _2 IS NOT ''`,
+			wantResult: `{"c1":"1","c2":"2","c3":"3"}`,
+		},
 	}
 
 	defRequest := `<?xml version="1.0" encoding="UTF-8"?>
@@ -782,7 +828,7 @@ func TestCSVQueries2(t *testing.T) {
 			}
 
 			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
-				return ioutil.NopCloser(bytes.NewBufferString(input)), nil
+				return ioutil.NopCloser(bytes.NewBuffer(testCase.input)), nil
 			}); err != nil {
 				t.Fatal(err)
 			}

internal/s3select/sql/parser.go

@@ -184,7 +184,7 @@ type ConditionRHS struct {
 
 // Compare represents the RHS of a comparison expression
 type Compare struct {
-	Operator string   `parser:"@( \"<>\" | \"<=\" | \">=\" | \"=\" | \"<\" | \">\" | \"!=\" )"`
+	Operator string   `parser:"@( \"<>\" | \"<=\" | \">=\" | \"=\" | \"<\" | \">\" | \"!=\" | \"IS\" \"NOT\" | \"IS\")"`
 	Operand  *Operand `parser:"  @@"`
 }
 

internal/s3select/sql/value.go

@@ -326,12 +326,14 @@ func (v *Value) negate() {
 
 // Supported comparison operators
 const (
-	opLt   = "<"
-	opLte  = "<="
-	opGt   = ">"
-	opGte  = ">="
-	opEq   = "="
-	opIneq = "!="
+	opLt    = "<"
+	opLte   = "<="
+	opGt    = ">"
+	opGte   = ">="
+	opEq    = "="
+	opIneq  = "!="
+	opIs    = "IS"
+	opIsNot = "ISNOT"
 )
 
 // InferBytesType will attempt to infer the data type of bytes.
@@ -385,6 +387,21 @@ func (v *Value) compareOp(op string, a *Value) (res bool, err error) {
 	if !isValidComparisonOperator(op) {
 		return false, errArithInvalidOperator
 	}
+	switch op {
+	case opIs:
+		if a.IsNull() {
+			return v.IsNull(), nil
+		}
+		// Forward to Equal
+		op = opEq
+	case opIsNot:
+		if a.IsNull() {
+			return !v.IsNull(), nil
+		}
+		// Forward to not equal.
+		op = opIneq
+	default:
+	}
 
 	// Check if type conversion/inference is needed - it is needed
 	// if the Value is a byte-slice.
@@ -747,12 +764,7 @@ func inferTypeAsString(v *Value) {
 
 func isValidComparisonOperator(op string) bool {
 	switch op {
-	case opLt:
-	case opLte:
-	case opGt:
-	case opGte:
-	case opEq:
-	case opIneq:
+	case opLt, opLte, opGt, opGte, opEq, opIneq, opIs, opIsNot:
 	default:
 		return false
 	}