mirror of
https://github.com/minio/minio.git
synced 2025-11-06 20:33:07 -05:00
S3 Select: Add parser support for lists. (#8329)
This commit is contained in:
Klaus Post
committed by
Harshavardhana
Harshavardhana
parent
e85df07518
commit
002ac82631
@@ -24,7 +24,10 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/v6"
|
||||
)
|
||||
|
||||
type testResponseWriter struct {
|
||||
@@ -48,6 +51,223 @@ func (w *testResponseWriter) WriteHeader(statusCode int) {
|
||||
func (w *testResponseWriter) Flush() {
|
||||
}
|
||||
|
||||
func TestJSONQueries(t *testing.T) {
|
||||
input := `{"id": 0,"title": "Test Record","desc": "Some text","synonyms": ["foo", "bar", "whatever"]}
|
||||
{"id": 1,"title": "Second Record","desc": "another text","synonyms": ["some", "synonym", "value"]}
|
||||
{"id": 2,"title": "Second Record","desc": "another text","numbers": [2, 3.0, 4]}
|
||||
{"id": 3,"title": "Second Record","desc": "another text","nested": [[2, 3.0, 4], [7, 8.5, 9]]}`
|
||||
var testTable = []struct {
|
||||
name string
|
||||
query string
|
||||
requestXML []byte
|
||||
wantResult string
|
||||
}{
|
||||
{
|
||||
name: "select-in-array-full",
|
||||
query: `SELECT * from s3object s WHERE 'bar' IN s.synonyms[*]`,
|
||||
wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}`,
|
||||
},
|
||||
{
|
||||
name: "simple-in-array",
|
||||
query: `SELECT * from s3object s WHERE s.id IN (1,3)`,
|
||||
wantResult: `{"id":1,"title":"Second Record","desc":"another text","synonyms":["some","synonym","value"]}
|
||||
{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,
|
||||
},
|
||||
{
|
||||
name: "select-in-array-single",
|
||||
query: `SELECT synonyms from s3object s WHERE 'bar' IN s.synonyms[*] `,
|
||||
wantResult: `{"synonyms":["foo","bar","whatever"]}`,
|
||||
},
|
||||
{
|
||||
name: "donatello-1",
|
||||
query: `SELECT * from s3object s WHERE 'bar' in s.synonyms`,
|
||||
wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}`,
|
||||
},
|
||||
{
|
||||
name: "donatello-2",
|
||||
query: `SELECT * from s3object s WHERE 'bar' in s.synonyms[*]`,
|
||||
wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}`,
|
||||
},
|
||||
{
|
||||
name: "donatello-3",
|
||||
query: `SELECT * from s3object s WHERE 'value' IN s.synonyms[*]`,
|
||||
wantResult: `{"id":1,"title":"Second Record","desc":"another text","synonyms":["some","synonym","value"]}`,
|
||||
},
|
||||
{
|
||||
name: "select-in-number",
|
||||
query: `SELECT * from s3object s WHERE 4 in s.numbers[*]`,
|
||||
wantResult: `{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,
|
||||
},
|
||||
{
|
||||
name: "select-in-number-float",
|
||||
query: `SELECT * from s3object s WHERE 3 in s.numbers[*]`,
|
||||
wantResult: `{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,
|
||||
},
|
||||
{
|
||||
name: "select-in-number-float-in-sql",
|
||||
query: `SELECT * from s3object s WHERE 3.0 in s.numbers[*]`,
|
||||
wantResult: `{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,
|
||||
},
|
||||
{
|
||||
name: "select-in-list-match",
|
||||
query: `SELECT * from s3object s WHERE (2,3,4) IN s.nested[*]`,
|
||||
wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,
|
||||
},
|
||||
{
|
||||
name: "select-in-nested-float",
|
||||
query: `SELECT s.nested from s3object s WHERE 8.5 IN s.nested[*][*]`,
|
||||
wantResult: `{"nested":[[2,3,4],[7,8.5,9]]}`,
|
||||
},
|
||||
{
|
||||
name: "select-in-combine-and",
|
||||
query: `SELECT s.nested from s3object s WHERE (8.5 IN s.nested[*][*]) AND (s.id > 0)`,
|
||||
wantResult: `{"nested":[[2,3,4],[7,8.5,9]]}`,
|
||||
},
|
||||
{
|
||||
name: "select-in-combine-and-no",
|
||||
query: `SELECT s.nested from s3object s WHERE (8.5 IN s.nested[*][*]) AND (s.id = 0)`,
|
||||
wantResult: ``,
|
||||
},
|
||||
{
|
||||
name: "select-in-nested-float-no-flat",
|
||||
query: `SELECT s.nested from s3object s WHERE 8.5 IN s.nested[*]`,
|
||||
wantResult: ``,
|
||||
},
|
||||
{
|
||||
name: "select-empty-field-result",
|
||||
query: `SELECT * from s3object s WHERE s.nested[0][0] = 2`,
|
||||
wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,
|
||||
},
|
||||
{
|
||||
name: "select-arrays-specific",
|
||||
query: `SELECT * from s3object s WHERE s.nested[1][0] = 7`,
|
||||
wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,
|
||||
},
|
||||
{
|
||||
name: "wrong-index-no-result",
|
||||
query: `SELECT * from s3object s WHERE s.nested[0][0] = 7`,
|
||||
wantResult: ``,
|
||||
},
|
||||
{
|
||||
name: "not-equal-result",
|
||||
query: `SELECT * from s3object s WHERE s.nested[1][0] != 7`,
|
||||
wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}
|
||||
{"id":1,"title":"Second Record","desc":"another text","synonyms":["some","synonym","value"]}
|
||||
{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,
|
||||
},
|
||||
{
|
||||
name: "indexed-list-match",
|
||||
query: `SELECT * from s3object s WHERE (7,8.5,9) IN s.nested[1]`,
|
||||
wantResult: ``,
|
||||
},
|
||||
{
|
||||
name: "indexed-list-match-equals",
|
||||
query: `SELECT * from s3object s WHERE (7,8.5,9) = s.nested[1]`,
|
||||
wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,
|
||||
},
|
||||
{
|
||||
name: "indexed-list-match-not-equals",
|
||||
query: `SELECT * from s3object s WHERE (7,8.5,9) != s.nested[1]`,
|
||||
wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}
|
||||
{"id":1,"title":"Second Record","desc":"another text","synonyms":["some","synonym","value"]}
|
||||
{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,
|
||||
},
|
||||
{
|
||||
name: "index-wildcard-in",
|
||||
query: `SELECT * from s3object s WHERE (8.5) IN s.nested[1][*]`,
|
||||
wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,
|
||||
},
|
||||
{
|
||||
name: "index-wildcard-in",
|
||||
query: `SELECT * from s3object s WHERE (8.0+0.5) IN s.nested[1][*]`,
|
||||
wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,
|
||||
},
|
||||
{
|
||||
name: "select-output-field-as-csv",
|
||||
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>SELECT s.synonyms from s3object s WHERE 'whatever' IN s.synonyms</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<JSON>
|
||||
<Type>DOCUMENT</Type>
|
||||
</JSON>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<CSV>
|
||||
</CSV>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
</SelectObjectContentRequest>`),
|
||||
wantResult: `"[""foo"",""bar"",""whatever""]"`,
|
||||
},
|
||||
}
|
||||
|
||||
defRequest := `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>%s</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<JSON>
|
||||
<Type>DOCUMENT</Type>
|
||||
</JSON>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<JSON>
|
||||
</JSON>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
</SelectObjectContentRequest>`
|
||||
|
||||
for _, testCase := range testTable {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
testReq := testCase.requestXML
|
||||
if len(testReq) == 0 {
|
||||
testReq = []byte(fmt.Sprintf(defRequest, testCase.query))
|
||||
}
|
||||
s3Select, err := NewS3Select(bytes.NewReader(testReq))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
|
||||
return ioutil.NopCloser(bytes.NewBufferString(input)), nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
w := &testResponseWriter{}
|
||||
s3Select.Evaluate(w)
|
||||
s3Select.Close()
|
||||
resp := http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: ioutil.NopCloser(bytes.NewReader(w.response)),
|
||||
ContentLength: int64(len(w.response)),
|
||||
}
|
||||
res, err := minio.NewSelectResults(&resp, "testbucket")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
got, err := ioutil.ReadAll(res)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
gotS := strings.TrimSpace(string(got))
|
||||
if !reflect.DeepEqual(gotS, testCase.wantResult) {
|
||||
t.Errorf("received response does not match with expected reply. Query: %s\ngot: %s\nwant:%s", testCase.query, gotS, testCase.wantResult)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVInput(t *testing.T) {
|
||||
var testTable = []struct {
|
||||
requestXML []byte
|
||||
|
||||
Reference in New Issue
Block a user