MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-08-02 03:54:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix: S3 Select CSV -> JSON with variable field count (#15677)

Browse Source 
When there are fewer fields than expected, output fewer fields.
This commit is contained in:
Klaus Post 2022-09-13 02:00:59 +02:00 committed by GitHub
parent 4a92134235
commit c22f3ca7a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 35 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
internal/s3select/csv/record.go
View File

@ -125,9 +125,11 @@ func (r *Record) WriteCSV(writer io.Writer, opts sql.WriteCSVOpts) error {
// WriteJSON - encodes to JSON data. // WriteJSON - encodes to JSON data.
func (r *Record) WriteJSON(writer io.Writer) error { func (r *Record) WriteJSON(writer io.Writer) error {
var kvs jstream.KVS = make([]jstream.KV, len(r.columnNames)) var kvs jstream.KVS = make([]jstream.KV, 0, len(r.columnNames))
for i := 0; i < len(r.columnNames); i++ { for i, cn := range r.columnNames {
kvs[i] = jstream.KV{Key: r.columnNames[i], Value: r.csvRecord[i]} if i < len(r.csvRecord) {
kvs = append(kvs, jstream.KV{Key: cn, Value: r.csvRecord[i]})
}
} }
return json.NewEncoder(writer).Encode(kvs) return json.NewEncoder(writer).Encode(kvs)
} }

30
internal/s3select/select_test.go
View File

@ -1547,6 +1547,36 @@ func TestCSVRanges(t *testing.T) {
<Enabled>FALSE</Enabled> <Enabled>FALSE</Enabled>
</RequestProgress> </RequestProgress>
<ScanRange></ScanRange> <ScanRange></ScanRange>
</SelectObjectContentRequest>`),
},
{
name: "var-field-count",
input: []byte(`id,time,num,num2,text
1,2010-01-01T,7867786,4565.908123
2,2017-01-02T03:04Z,-5, 0.765111,Some some
`),
// Since we are doing offset, no headers are used.
wantResult: `{"id":"1","time":"2010-01-01T","num":"7867786","num2":"4565.908123"}
{"id":"2","time":"2017-01-02T03:04Z","num":"-5","num2":" 0.765111","text":"Some some"}`,
wantErr: false,
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
<SelectObjectContentRequest>
<Expression>SELECT * from s3object</Expression>
<ExpressionType>SQL</ExpressionType>
<InputSerialization>
<CompressionType>NONE</CompressionType>
<CSV>
<FileHeaderInfo>USE</FileHeaderInfo>
<QuoteCharacter>"</QuoteCharacter>
</CSV>
</InputSerialization>
<OutputSerialization>
<JSON>
</JSON>
</OutputSerialization>
<RequestProgress>
<Enabled>FALSE</Enabled>
</RequestProgress>
</SelectObjectContentRequest>`), </SelectObjectContentRequest>`),
}, },
{ {