select: Add ScanRange to CSV&JSON (#14546)

Implements https://docs.aws.amazon.com/AmazonS3/latest/API/API_SelectObjectContent.html#AmazonS3-SelectObjectContent-request-ScanRange

Fixes #14539

internal/s3select/select.go

@@ -212,6 +212,69 @@ type RequestProgress struct {
 	Enabled bool `xml:"Enabled"`
 }
 
+// ScanRange represents the ScanRange parameter.
+type ScanRange struct {
+	// Start if byte offset form the start off the file.
+	Start *uint64 `xml:"Start"`
+	// End is the last byte that should be returned, if Start is set,
+	// or the offset from EOF to start reading if start is not present.
+	End *uint64 `xml:"End"`
+}
+
+// Validate if the scan range is valid.
+func (s *ScanRange) Validate() error {
+	if s == nil {
+		return nil
+	}
+	if s.Start == nil && s.End == nil {
+		// This parameter is optional, but when specified, it must not be empty.
+		// Ref: https://docs.aws.amazon.com/AmazonS3/latest/API/API_SelectObjectContent.html#AmazonS3-SelectObjectContent-request-ScanRange
+		return errors.New("ScanRange: No Start or End specified")
+	}
+	if s.Start == nil || s.End == nil {
+		return nil
+	}
+	if *s.Start > *s.End {
+		return errors.New("ScanRange: Start cannot be after end")
+	}
+	return nil
+}
+
+// StartLen returns start offset plus length from range.
+func (s *ScanRange) StartLen() (start, length int64, err error) {
+	if s == nil {
+		return 0, -1, nil
+	}
+	err = s.Validate()
+	if err != nil {
+		return 0, 0, err
+	}
+
+	if s.End == nil && s.Start == nil {
+		// Not valid, but should be caught above.
+		return 0, -1, nil
+	}
+	if s.End == nil {
+		start := int64(*s.Start)
+		if start < 0 {
+			return 0, 0, errors.New("ScanRange: Start after EOF")
+		}
+		return start, -1, nil
+	}
+	if s.Start == nil {
+		// Suffix length
+		end := int64(*s.End)
+		if end < 0 {
+			return 0, 0, errors.New("ScanRange: End bigger than file")
+		}
+		// Suffix length
+		return -end, -1, nil
+	}
+	start = int64(*s.Start)
+	end := int64(*s.End)
+	return start, end - start + 1, nil
+}
+
 // S3Select - filters the contents on a simple structured query language (SQL) statement. It
 // represents elements inside <SelectRequest/> in request XML specified in detail at
 // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html.
@@ -222,6 +285,7 @@ type S3Select struct {
 	Input          InputSerialization  `xml:"InputSerialization"`
 	Output         OutputSerialization `xml:"OutputSerialization"`
 	Progress       RequestProgress     `xml:"RequestProgress"`
+	ScanRange      *ScanRange          `xml:"ScanRange"`
 
 	statement      *sql.SelectStatement
 	progressReader *progressReader
@@ -251,7 +315,9 @@ func (s3Select *S3Select) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e
 
 		return errMalformedXML(err)
 	}
-
+	if err := parsedS3Select.ScanRange.Validate(); err != nil {
+		return errInvalidScanRangeParameter(err)
+	}
 	parsedS3Select.ExpressionType = strings.ToLower(parsedS3Select.ExpressionType)
 	if parsedS3Select.ExpressionType != "sql" {
 		return errInvalidExpressionType(fmt.Errorf("invalid expression type '%v'", parsedS3Select.ExpressionType))
@@ -298,9 +364,13 @@ func (s3Select *S3Select) getProgress() (bytesScanned, bytesProcessed int64) {
 // Open - opens S3 object by using callback for SQL selection query.
 // Currently CSV, JSON and Apache Parquet formats are supported.
 func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadCloser, error)) error {
+	offset, end, err := s3Select.ScanRange.StartLen()
+	if err != nil {
+		return err
+	}
 	switch s3Select.Input.format {
 	case csvFormat:
-		rc, err := getReader(0, -1)
+		rc, err := getReader(offset, end)
 		if err != nil {
 			return err
 		}
@@ -335,7 +405,7 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos
 		s3Select.close = rc.Close
 		return nil
 	case jsonFormat:
-		rc, err := getReader(0, -1)
+		rc, err := getReader(offset, end)
 		if err != nil {
 			return err
 		}
@@ -362,6 +432,10 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos
 		if !strings.EqualFold(os.Getenv("MINIO_API_SELECT_PARQUET"), "on") {
 			return errors.New("parquet format parsing not enabled on server")
 		}
+		if offset != 0 || end != -1 {
+			// Offsets do not make sense in parquet files.
+			return errors.New("parquet format does not support offsets")
+		}
 		var err error
 		s3Select.recordReader, err = parquet.NewReader(getReader, &s3Select.Input.ParquetArgs)
 		return err