MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-20 01:50:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

Use a buffer to collect SQL Select result rows (#7158)

Browse Source 
Batching records into a single SQL Select message in the response
leads to significant speed up as the message header overhead is made
negligible.

This change leads to a speed up of 3-5x for queries that select many
small records.
This commit is contained in:
Aditya Manthramurthy
2019-01-28 20:00:18 -08:00
committed by Harshavardhana
parent 2786055df4
commit 91c839ad28
3 changed files with 59 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/s3select/select.go
View File

@@ -49,6 +49,10 @@ const (
bzip2Type CompressionType = "bzip2"
)
const (
maxRecordSize = 1 << 20 // 1 MiB
)
// UnmarshalXML - decodes XML data.
func (c *CompressionType) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var s string
@@ -324,6 +328,11 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
return false
}
if len(data) > maxRecordSize {
writer.SendError("OverMaxRecordSize", "The length of a record in the input or result is greater than maxCharsPerRecord of 1 MB.")
return false
}
if err = writer.SendRecords(data); err != nil {
// FIXME: log this error.
err = nil
@@ -335,6 +344,12 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
for {
if s3Select.statement.LimitReached() {
if err = writer.FlushRecords(); err != nil {
// FIXME: log this error
err = nil
break
}
if err = writer.SendStats(s3Select.getProgress()); err != nil {
// FIXME: log this error.
err = nil
@@ -358,6 +373,12 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
}
}
if err = writer.FlushRecords(); err != nil {
// FIXME: log this error
err = nil
break
}
if err = writer.SendStats(s3Select.getProgress()); err != nil {
// FIXME: log this error.
err = nil
@@ -379,7 +400,6 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
if !sendRecord() {
break
}
}
}