MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-20 01:50:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

SIMDJSON S3 select input (#8401)

Browse Source
This commit is contained in:
Klaus Post
2020-02-13 14:03:52 -08:00
committed by GitHub
parent d1144c2c7e
commit e4020fb41f
16 changed files with 1116 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pkg/s3select/select.go
View File

@@ -30,7 +30,9 @@ import (
"github.com/minio/minio/pkg/s3select/csv"
"github.com/minio/minio/pkg/s3select/json"
"github.com/minio/minio/pkg/s3select/parquet"
"github.com/minio/minio/pkg/s3select/simdj"
"github.com/minio/minio/pkg/s3select/sql"
"github.com/minio/simdjson-go"
)
type recordReader interface {
@@ -317,7 +319,11 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos
}
if strings.EqualFold(s3Select.Input.JSONArgs.ContentType, "lines") {
s3Select.recordReader = json.NewPReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
if simdjson.SupportedCPU() {
s3Select.recordReader = simdj.NewReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
} else {
s3Select.recordReader = json.NewPReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
}
} else {
s3Select.recordReader = json.NewReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
}
@@ -350,7 +356,9 @@ func (s3Select *S3Select) marshal(buf *bytes.Buffer, record sql.Record) error {
if err != nil {
return err
}
buf.Truncate(buf.Len() - 1)
if buf.Bytes()[buf.Len()-1] == '\n' {
buf.Truncate(buf.Len() - 1)
}
buf.WriteString(s3Select.Output.CSVArgs.RecordDelimiter)
return nil
@@ -359,8 +367,10 @@ func (s3Select *S3Select) marshal(buf *bytes.Buffer, record sql.Record) error {
if err != nil {
return err
}
buf.Truncate(buf.Len() - 1)
// Trim trailing newline from non-simd output
if buf.Bytes()[buf.Len()-1] == '\n' {
buf.Truncate(buf.Len() - 1)
}
buf.WriteString(s3Select.Output.JSONArgs.RecordDelimiter)
return nil