MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 05:34:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Use jstream to serialize records to JSON format in S3Select (#7318)

Browse Source 
- Also, switch to jstream to generate internal record representation
  from CSV/JSON readers

- This fixes a bug in which JSON output objects have their keys
  reversed from the order they are specified in the Select columns.

- Also includes a fix for tests.
This commit is contained in:
Aditya Manthramurthy
2019-03-07 00:20:10 -08:00
committed by Harshavardhana
parent f97a33a63f
commit f4879ed96d
4 changed files with 68 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
pkg/s3select/parquet/reader.go
View File

@@ -17,9 +17,11 @@
package parquet
import (
"encoding/json"
"io"
"github.com/minio/minio/pkg/s3select/json"
"github.com/bcicen/jstream"
jsonfmt "github.com/minio/minio/pkg/s3select/json"
"github.com/minio/minio/pkg/s3select/sql"
parquetgo "github.com/minio/parquet-go"
parquetgen "github.com/minio/parquet-go/gen-go/parquet"
@@ -42,42 +44,42 @@ func (r *Reader) Read() (rec sql.Record, rerr error) {
return nil, err
}
record := json.NewRecord()
kvs := jstream.KVS{}
f := func(name string, v parquetgo.Value) bool {
if v.Value == nil {
if err := record.Set(name, sql.FromNull()); err != nil {
rerr = errParquetParsingError(err)
}
return rerr == nil
kvs = append(kvs, jstream.KV{Key: name, Value: nil})
return true
}
var value *sql.Value
var value interface{}
switch v.Type {
case parquetgen.Type_BOOLEAN:
value = sql.FromBool(v.Value.(bool))
value = v.Value.(bool)
case parquetgen.Type_INT32:
value = sql.FromInt(int64(v.Value.(int32)))
value = int64(v.Value.(int32))
case parquetgen.Type_INT64:
value = sql.FromInt(int64(v.Value.(int64)))
value = int64(v.Value.(int64))
case parquetgen.Type_FLOAT:
value = sql.FromFloat(float64(v.Value.(float32)))
value = float64(v.Value.(float32))
case parquetgen.Type_DOUBLE:
value = sql.FromFloat(v.Value.(float64))
value = v.Value.(float64)
case parquetgen.Type_INT96, parquetgen.Type_BYTE_ARRAY, parquetgen.Type_FIXED_LEN_BYTE_ARRAY:
value = sql.FromString(string(v.Value.([]byte)))
value = string(v.Value.([]byte))
default:
rerr = errParquetParsingError(nil)
return false
}
if err = record.Set(name, value); err != nil {
rerr = errParquetParsingError(err)
}
return rerr == nil
kvs = append(kvs, jstream.KV{Key: name, Value: value})
return true
}
parquetRecord.Range(f)
return record, rerr
data, err := json.Marshal(kvs)
if err != nil {
return nil, err
}
return &jsonfmt.Record{Data: data}, rerr
}
// Close - closes underlaying readers.