sql: Add support of escape quote in CSV (#9231)

This commit modifies csv parser, a fork of golang csv
parser to support a custom quote escape character.

The quote escape character is used to escape the quote
character when a csv field contains a quote character
as part of data.

pkg/s3select/csv/args.go

@@ -104,8 +104,15 @@ func (args *ReaderArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (er
 						return fmt.Errorf("unsupported QuoteCharacter '%v'", s)
 					}
 					args.QuoteCharacter = s
-				// Not supported yet
 				case "QuoteEscapeCharacter":
+					switch utf8.RuneCountInString(s) {
+					case 0:
+						args.QuoteEscapeCharacter = defaultQuoteEscapeCharacter
+					case 1:
+						args.QuoteEscapeCharacter = s
+					default:
+						return fmt.Errorf("unsupported QuoteEscapeCharacter '%v'", s)
+					}
 				case "Comments":
 					args.CommentCharacter = s
 				default:
@@ -115,7 +122,6 @@ func (args *ReaderArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (er
 		}
 	}
 
-	args.QuoteEscapeCharacter = args.QuoteCharacter
 	args.unmarshaled = true
 	return nil
 }
@@ -176,15 +182,21 @@ func (args *WriterArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) err
 				default:
 					return fmt.Errorf("unsupported QuoteCharacter '%v'", s)
 				}
-			// Not supported yet
 			case "QuoteEscapeCharacter":
+				switch utf8.RuneCountInString(s) {
+				case 0:
+					args.QuoteEscapeCharacter = defaultQuoteEscapeCharacter
+				case 1:
+					args.QuoteEscapeCharacter = s
+				default:
+					return fmt.Errorf("unsupported QuoteCharacter '%v'", s)
+				}
 			default:
 				return errors.New("unrecognized option")
 			}
 		}
 	}
 
-	args.QuoteEscapeCharacter = args.QuoteCharacter
 	args.unmarshaled = true
 	return nil
 }

pkg/s3select/csv/reader.go

@@ -299,6 +299,7 @@ func NewReader(readCloser io.ReadCloser, args *ReaderArgs) (*Reader, error) {
 			// Add the first rune of args.QuoteChracter
 			ret.Quote = append(ret.Quote, []rune(args.QuoteCharacter)[0])
 		}
+		ret.QuoteEscape = []rune(args.QuoteEscapeCharacter)[0]
 		ret.FieldsPerRecord = -1
 		// If LazyQuotes is true, a quote may appear in an unquoted field and a
 		// non-doubled quote may appear in a quoted field.

pkg/s3select/csv/reader_test.go

@@ -63,7 +63,13 @@ func TestRead(t *testing.T) {
 			if err != nil {
 				break
 			}
-			record.WriteCSV(&result, []rune(c.fieldDelimiter)[0], '"', false)
+			opts := sql.WriteCSVOpts{
+				FieldDelimiter: []rune(c.fieldDelimiter)[0],
+				Quote:          '"',
+				QuoteEscape:    '"',
+				AlwaysQuote:    false,
+			}
+			record.WriteCSV(&result, opts)
 			result.Truncate(result.Len() - 1)
 			result.WriteString(c.recordDelimiter)
 		}
@@ -242,8 +248,14 @@ func TestReadExtended(t *testing.T) {
 					break
 				}
 				if fields < 10 {
+					opts := sql.WriteCSVOpts{
+						FieldDelimiter: ',',
+						Quote:          '"',
+						QuoteEscape:    '"',
+						AlwaysQuote:    false,
+					}
 					// Write with fixed delimiters, newlines.
-					err := record.WriteCSV(&result, ',', '"', false)
+					err := record.WriteCSV(&result, opts)
 					if err != nil {
 						t.Error(err)
 					}
@@ -453,8 +465,15 @@ func TestReadFailures(t *testing.T) {
 				if err != nil {
 					break
 				}
+
+				opts := sql.WriteCSVOpts{
+					FieldDelimiter: ',',
+					Quote:          '"',
+					QuoteEscape:    '"',
+					AlwaysQuote:    false,
+				}
 				// Write with fixed delimiters, newlines.
-				err := record.WriteCSV(&result, ',', '"', false)
+				err := record.WriteCSV(&result, opts)
 				if err != nil {
 					t.Error(err)
 				}

pkg/s3select/csv/record.go

@@ -92,11 +92,12 @@ func (r *Record) Clone(dst sql.Record) sql.Record {
 }
 
 // WriteCSV - encodes to CSV data.
-func (r *Record) WriteCSV(writer io.Writer, fieldDelimiter rune, quote rune, alwaysQuote bool) error {
+func (r *Record) WriteCSV(writer io.Writer, opts sql.WriteCSVOpts) error {
 	w := csv.NewWriter(writer)
-	w.Comma = fieldDelimiter
-	w.AlwaysQuote = alwaysQuote
-	w.Quote = quote
+	w.Comma = opts.FieldDelimiter
+	w.AlwaysQuote = opts.AlwaysQuote
+	w.Quote = opts.Quote
+	w.QuoteEscape = opts.QuoteEscape
 	if err := w.Write(r.csvRecord); err != nil {
 		return err
 	}