From cef044178c6ba0f76cb9dd21184d5163115538a7 Mon Sep 17 00:00:00 2001 From: Praveen raj Mani Date: Wed, 17 Oct 2018 23:31:26 +0530 Subject: [PATCH] Treat columns with spaces inbetween [s3Select] (#6597) replace the double/single quotes with backticks for the xwb1989/sqlparser to recognise such queries. Fixes #6589 --- cmd/object-handlers.go | 10 ++++++++-- pkg/s3select/input.go | 18 ++++++++++-------- pkg/s3select/select.go | 1 - pkg/s3select/select_test.go | 1 + 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index bf7ec7d62..890bac5e8 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -70,6 +70,12 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { } } +// This function replaces "",'' with `` for the select parser +func cleanExpr(expr string) string { + r := strings.NewReplacer("\"", "`", "'", "`") + return r.Replace(expr) +} + // SelectObjectContentHandler - GET Object?select // ---------- // This implementation of the GET operation retrieves object content based @@ -255,7 +261,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r Name: "S3Object", // Default table name for all objects ReadFrom: gr, Compressed: string(selectReq.InputSerialization.CompressionType), - Expression: selectReq.Expression, + Expression: cleanExpr(selectReq.Expression), OutputFieldDelimiter: selectReq.OutputSerialization.CSV.FieldDelimiter, StreamSize: objInfo.Size, HeaderOpt: selectReq.InputSerialization.CSV.FileHeaderInfo == CSVFileHeaderInfoUse, @@ -266,7 +272,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } - _, _, _, _, _, _, err = s3s.ParseSelect(selectReq.Expression) + _, _, _, _, _, _, err = s3s.ParseSelect(options.Expression) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return diff --git a/pkg/s3select/input.go b/pkg/s3select/input.go index b9c3d4ad3..b814c2294 100644 --- a/pkg/s3select/input.go +++ b/pkg/s3select/input.go @@ -207,12 +207,6 @@ func (reader *Input) ReadRecord() []string { return row } -// convertMySQL Replaces double quote escape for column names with backtick for -// the MySQL parser -func convertMySQL(random string) string { - return strings.Replace(random, "\"", "`", len(random)) -} - // readHeader reads the header into the header variable if the header is present // as the first row of the csv func (reader *Input) readHeader() error { @@ -222,7 +216,7 @@ func (reader *Input) readHeader() error { if readErr != nil { return ErrCSVParsingError } - reader.header = reader.firstRow + reader.header = cleanHeader(reader.firstRow) reader.firstRow = nil reader.minOutputLength = len(reader.header) } else { @@ -236,6 +230,14 @@ func (reader *Input) readHeader() error { return nil } +// Replace the spaces in columnnames with underscores +func cleanHeader(columns []string) []string { + for i := 0; i < len(columns); i++ { + columns[i] = strings.Replace(columns[i], " ", "_", -1) + } + return columns +} + // createStatXML is the function which does the marshaling from the stat // structs into XML so that the progress and stat message can be sent func (reader *Input) createStatXML() (string, error) { @@ -296,7 +298,7 @@ func (reader *Input) Execute(writer io.Writer) error { continuationTimer := time.NewTimer(continuationTime) defer progressTicker.Stop() defer continuationTimer.Stop() - go reader.runSelectParser(convertMySQL(reader.options.Expression), myRow) + go reader.runSelectParser(reader.options.Expression, myRow) for { select { case row, ok := <-myRow: diff --git a/pkg/s3select/select.go b/pkg/s3select/select.go index 59b85746c..38b4e7fc5 100644 --- a/pkg/s3select/select.go +++ b/pkg/s3select/select.go @@ -50,7 +50,6 @@ func (reader *Input) runSelectParser(selectExpression string, myRow chan *Row) { // records, and the where clause. func (reader *Input) ParseSelect(sqlInput string) ([]string, string, int64, interface{}, []string, *SelectFuncs, error) { // return columnNames, alias, limitOfRecords, whereclause,coalStore, nil - stmt, err := sqlparser.Parse(sqlInput) var whereClause interface{} var alias string diff --git a/pkg/s3select/select_test.go b/pkg/s3select/select_test.go index 441aee5f0..92281664f 100644 --- a/pkg/s3select/select_test.go +++ b/pkg/s3select/select_test.go @@ -253,6 +253,7 @@ func TestMyParser(t *testing.T) { {"SELECT count(*) FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"*"}, "A", 5, []string{"count"}, []string{"col_name", "col_other", "name3", "name4"}}, {"SELECT sum(col_name),sum(col_other) FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, []string{"sum", "sum"}, []string{"col_name", "col_other"}}, {"SELECT A.col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}}, + {"SELECT A.`col name` FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}}, {"SELECT A._col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}}, {"SELECT A._col_name FROM S3OBJECT AS A WHERE randomname > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}}, {"SELECT A._col_name FROM S3OBJECT AS A WHERE A._11 > 5", ErrInvalidColumnIndex, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},