MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-20 01:50:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

S3 Select: Concurrent LINES delimited json parsing (#8610)

Browse Source 
The speedup is ~5x on a 6 core CPU
This commit is contained in:
Klaus Post
2019-12-09 06:55:31 -08:00
committed by Harshavardhana
parent 3b67f629a4
commit bf3a97d3aa
3 changed files with 338 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/s3select/select.go
View File

@@ -316,7 +316,11 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos
return err
}
s3Select.recordReader = json.NewReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
if strings.EqualFold(s3Select.Input.JSONArgs.ContentType, "lines") {
s3Select.recordReader = json.NewPReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
} else {
s3Select.recordReader = json.NewReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
}
return nil
case parquetFormat:
var err error