S3 select switch to new parquet library and reduce locking (#14731)

- This change switches to a new parquet library
- SelectObjectContent now takes a single lock at the beginning and holds it
during the operation. Previously the operation took a lock every time the
parquet library performed a Seek on the underlying object stream.
- Add basic support for LogicalType annotations for timestamps.

cmd/object-handlers.go

@@ -189,28 +189,17 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 	getObjectNInfo := objectAPI.GetObjectNInfo
 	if api.CacheAPI() != nil {
 		getObjectNInfo = api.CacheAPI().GetObjectNInfo
-	}
+	} else {
-
+		// Take read lock on object, here so subsequent lower-level
-	getObject := func(offset, length int64) (rc io.ReadCloser, err error) {
+		// calls do not need to.
-		isSuffixLength := false
+		lock := objectAPI.NewNSLock(bucket, object)
-		if offset < 0 {
+		lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
-			isSuffixLength = true
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
 		}
-
+		ctx = lkctx.Context()
-		if length > 0 {
+		defer lock.RUnlock(lkctx.Cancel)
-			length--
-		}
-
-		rs := &HTTPRangeSpec{
-			IsSuffixLength: isSuffixLength,
-			Start:          offset,
-			End:            offset + length,
-		}
-		if length == -1 {
-			rs.End = -1
-		}
-
-		return getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, opts)
 	}
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
@@ -241,6 +230,24 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		}
 	}
 
+	actualSize, err := objInfo.GetActualSize()
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	objectRSC := s3select.NewObjectReadSeekCloser(
+		func(offset int64) (io.ReadCloser, error) {
+			rs := &HTTPRangeSpec{
+				IsSuffixLength: false,
+				Start:          offset,
+				End:            -1,
+			}
+			return getObjectNInfo(ctx, bucket, object, rs, r.Header, noLock, opts)
+		},
+		actualSize,
+	)
+
 	s3Select, err := s3select.NewS3Select(r.Body)
 	if err != nil {
 		if serr, ok := err.(s3select.SelectError); ok {
@@ -261,7 +268,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 	}
 	defer s3Select.Close()
 
-	if err = s3Select.Open(getObject); err != nil {
+	if err = s3Select.Open(objectRSC); err != nil {
 		if serr, ok := err.(s3select.SelectError); ok {
 			encodedErrorResponse := encodeResponse(APIErrorResponse{
 				Code:       serr.ErrorCode(),
@@ -4191,23 +4198,26 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 	go func() {
 		rctx := GlobalContext
 		if !rreq.SelectParameters.IsEmpty() {
-			getObject := func(offset, length int64) (rc io.ReadCloser, err error) {
+			actualSize, err := objInfo.GetActualSize()
-				isSuffixLength := false
+			if err != nil {
-				if offset < 0 {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-					isSuffixLength = true
+				return
-				}
-
-				rs := &HTTPRangeSpec{
-					IsSuffixLength: isSuffixLength,
-					Start:          offset,
-					End:            offset + length,
-				}
-
-				return getTransitionedObjectReader(rctx, bucket, object, rs, r.Header, objInfo, ObjectOptions{
-					VersionID: objInfo.VersionID,
-				})
 			}
-			if err = rreq.SelectParameters.Open(getObject); err != nil {
+
+			objectRSC := s3select.NewObjectReadSeekCloser(
+				func(offset int64) (io.ReadCloser, error) {
+					rs := &HTTPRangeSpec{
+						IsSuffixLength: false,
+						Start:          offset,
+						End:            -1,
+					}
+					return getTransitionedObjectReader(rctx, bucket, object, rs, r.Header,
+						objInfo, ObjectOptions{VersionID: objInfo.VersionID})
+				},
+				actualSize,
+			)
+
+			if err = rreq.SelectParameters.Open(objectRSC); err != nil {
 				if serr, ok := err.(s3select.SelectError); ok {
 					encodedErrorResponse := encodeResponse(APIErrorResponse{
 						Code:       serr.ErrorCode(),

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/elastic/go-elasticsearch/v7 v7.12.0
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
+	github.com/fraugster/parquet-go v0.10.0
 	github.com/go-ldap/ldap/v3 v3.2.4
 	github.com/go-openapi/loads v0.21.1
 	github.com/go-sql-driver/mysql v1.6.0
@@ -52,7 +53,6 @@ require (
 	github.com/minio/kes v0.19.0
 	github.com/minio/madmin-go v1.3.11
 	github.com/minio/minio-go/v7 v7.0.23
-	github.com/minio/parquet-go v1.1.0
 	github.com/minio/pkg v1.1.20
 	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
@@ -200,7 +200,6 @@ require (
 	github.com/tidwall/gjson v1.14.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/tidwall/sjson v1.2.3 // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/unrolled/secure v1.10.0 // indirect

go.sum

@@ -111,6 +111,9 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y=
 github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
 github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs=
 github.com/armon/go-metrics v0.3.3 h1:a9F4rlj7EWWrbj7BYw8J8+x+ZZkJeqzNyRk8hdPF+ro=
@@ -192,8 +195,12 @@ github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDG
 github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
 github.com/coredns/coredns v1.9.0 h1:M1EF1uups4CYcQGb1z8A97mfoq4BYCw3+xCYcJkOSDc=
 github.com/coredns/coredns v1.9.0/go.mod h1:czzy6Ofs15Mzn1PXpWoplBCZxoWdGoQUInL9uPSiYME=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -202,6 +209,7 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cosnicolaou/pbzip2 v1.0.1 h1:f5Ix000Rtl9tr0Ne33wNLtljGl2nAyR4ZirJrz9qg+0=
 github.com/cosnicolaou/pbzip2 v1.0.1/go.mod h1:cE04zhBMvwMrCLhsx6aLYh9cGsU9GyFB0oo/GmO+SkY=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -263,9 +271,11 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
-github.com/frankban/quicktest v1.12.1/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
 github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
 github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
+github.com/fraugster/parquet-go v0.10.0 h1:whX91AO3dkkOnbH9MqD53DZ3rISw+Tnnj5yiqXjSv9Q=
+github.com/fraugster/parquet-go v0.10.0/go.mod h1:asQOKX0K/j+F3Xyj87kw7gKrU3yXo9M2hb8STSQKIIw=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
 github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
@@ -620,6 +630,7 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
@@ -683,12 +694,9 @@ github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054/go.mod h1:PIQHcb4uOctKyL/
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
 github.com/minio/minio-go/v7 v7.0.23 h1:NleyGQvAn9VQMU+YHVrgV4CX+EPtxPt/78lHOOTncy4=
 github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
-github.com/minio/parquet-go v1.1.0 h1:j2Fn1/h7Ts/0qzdMZU9oCUKr0IJwRTD9Hg9QJyVaN6A=
-github.com/minio/parquet-go v1.1.0/go.mod h1:nnAkbt2CG/DCQ3trcV3uyvwns4VjyoINF5vMqF5efOE=
 github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.1.20 h1:NhYtoQHw/Sl1ib/lroANFwkQspE0YyTeVR1CMPEff/A=
 github.com/minio/pkg v1.1.20/go.mod h1:Xo7LQshlxGa9shKwJ7NzQbgW4s8T/Wc1cOStR/eUiMY=
@@ -713,6 +721,7 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -785,13 +794,13 @@ github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.m
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 h1:6ob53CVz+ja2i7easAStApZJlh7sxyq3Cm7g1Di6iqA=
 github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
 github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -866,10 +875,12 @@ github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
+github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
 github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc=
 github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs=
 github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
@@ -889,11 +900,27 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo=
 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -905,8 +932,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
 github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
@@ -914,8 +939,6 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8=
-github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs=
 github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
 github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e h1:P5tyWbssToKowBPTA1/EzqPXwrZNc8ZeNPdjgpcDEoI=
 github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e/go.mod h1:g7jEyb18KPe65d9RRhGw+ThaJr5duyBH8eaFgBUor7Y=
@@ -927,6 +950,7 @@ github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//
 github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
 github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/unrolled/secure v1.10.0 h1:TBNP42z2AB+2pW9PR6vdbqhlQuv1iTeSVzK1qHjOBzA=
 github.com/unrolled/secure v1.10.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
 github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -940,6 +964,7 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
 github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
@@ -993,6 +1018,7 @@ go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
@@ -1143,6 +1169,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

internal/s3select/parquet/reader.go

@@ -18,88 +18,55 @@
 package parquet
 
 import (
-	"fmt"
+	"errors"
 	"io"
 	"time"
 
 	"github.com/bcicen/jstream"
+	parquetgo "github.com/fraugster/parquet-go"
+	parquettypes "github.com/fraugster/parquet-go/parquet"
 	jsonfmt "github.com/minio/minio/internal/s3select/json"
 	"github.com/minio/minio/internal/s3select/sql"
-	parquetgo "github.com/minio/parquet-go"
-	parquetgen "github.com/minio/parquet-go/gen-go/parquet"
 )
 
-// Reader - Parquet record reader for S3Select.
+// Reader implements reading records from parquet input.
 type Reader struct {
-	args   *ReaderArgs
+	io.Closer
-	reader *parquetgo.Reader
+	r *parquetgo.FileReader
 }
 
-// Read - reads single record.
+// NewParquetReader creates a Reader2 from a io.ReadSeekCloser.
-func (r *Reader) Read(dst sql.Record) (rec sql.Record, rerr error) {
+func NewParquetReader(rsc io.ReadSeekCloser, _ *ReaderArgs) (r *Reader, err error) {
-	defer func() {
+	fr, err := parquetgo.NewFileReader(rsc)
-		if rec := recover(); rec != nil {
-			rerr = fmt.Errorf("panic reading parquet record: %v", rec)
-		}
-	}()
-
-	parquetRecord, err := r.reader.Read()
 	if err != nil {
-		if err != io.EOF {
+		return nil, errParquetParsingError(err)
-			return nil, errParquetParsingError(err)
+	}
-		}
 
-		return nil, err
+	return &Reader{Closer: rsc, r: fr}, nil
+}
+
+func (pr *Reader) Read(dst sql.Record) (rec sql.Record, rerr error) {
+	nextRow, err := pr.r.NextRow()
+	if err != nil {
+		if err == io.EOF {
+			return nil, err
+		}
+		return nil, errParquetParsingError(err)
 	}
 
 	kvs := jstream.KVS{}
-	f := func(name string, v parquetgo.Value) bool {
+	for _, col := range pr.r.Columns() {
-		if v.Value == nil {
-			kvs = append(kvs, jstream.KV{Key: name, Value: nil})
-			return true
-		}
 
 		var value interface{}
-		switch v.Type {
+		if v, ok := nextRow[col.FlatName()]; ok {
-		case parquetgen.Type_BOOLEAN:
+			value, err = convertFromAnnotation(col.Element(), v)
-			value = v.Value.(bool)
+			if err != nil {
-		case parquetgen.Type_INT32:
+				return nil, errParquetParsingError(err)
-			value = int64(v.Value.(int32))
-			if v.Schema != nil && v.Schema.ConvertedType != nil {
-				switch *v.Schema.ConvertedType {
-				case parquetgen.ConvertedType_DATE:
-					value = sql.FormatSQLTimestamp(time.Unix(60*60*24*int64(v.Value.(int32)), 0).UTC())
-				}
 			}
-		case parquetgen.Type_INT64:
-			value = v.Value.(int64)
-			if v.Schema != nil && v.Schema.ConvertedType != nil {
-				switch *v.Schema.ConvertedType {
-				// Only UTC supported, add one NS to never be exactly midnight.
-				case parquetgen.ConvertedType_TIMESTAMP_MILLIS:
-					value = sql.FormatSQLTimestamp(time.Unix(0, 0).Add(time.Duration(v.Value.(int64)) * time.Millisecond).UTC())
-				case parquetgen.ConvertedType_TIMESTAMP_MICROS:
-					value = sql.FormatSQLTimestamp(time.Unix(0, 0).Add(time.Duration(v.Value.(int64)) * time.Microsecond).UTC())
-				}
-			}
-		case parquetgen.Type_FLOAT:
-			value = float64(v.Value.(float32))
-		case parquetgen.Type_DOUBLE:
-			value = v.Value.(float64)
-		case parquetgen.Type_INT96, parquetgen.Type_BYTE_ARRAY, parquetgen.Type_FIXED_LEN_BYTE_ARRAY:
-			value = string(v.Value.([]byte))
-		default:
-			rerr = errParquetParsingError(nil)
-			return false
 		}
-
+		kvs = append(kvs, jstream.KV{Key: col.FlatName(), Value: value})
-		kvs = append(kvs, jstream.KV{Key: name, Value: value})
-		return true
 	}
 
-	// Apply our range
-	parquetRecord.Range(f)
-
 	// Reuse destination if we can.
 	dstRec, ok := dst.(*jsonfmt.Record)
 	if !ok {
@@ -110,29 +77,67 @@ func (r *Reader) Read(dst sql.Record) (rec sql.Record, rerr error) {
 	return dstRec, nil
 }
 
-// Close - closes underlying readers.
+// convertFromAnnotation - converts values based on the Parquet column's type
-func (r *Reader) Close() error {
+// annotations. LogicalType annotations if present override the deprecated
-	return r.reader.Close()
+// ConvertedType annotations. Ref:
-}
+// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
-
+func convertFromAnnotation(se *parquettypes.SchemaElement, v interface{}) (interface{}, error) {
-// NewReader - creates new Parquet reader using readerFunc callback.
+	if se == nil {
-func NewReader(getReaderFunc func(offset, length int64) (io.ReadCloser, error), args *ReaderArgs) (r *Reader, err error) {
+		return v, nil
-	defer func() {
-		if rec := recover(); rec != nil {
-			err = fmt.Errorf("panic reading parquet header: %v", rec)
-		}
-	}()
-	reader, err := parquetgo.NewReader(getReaderFunc, nil)
-	if err != nil {
-		if err != io.EOF {
-			return nil, errParquetParsingError(err)
-		}
-
-		return nil, err
 	}
 
-	return &Reader{
+	var value interface{}
-		args:   args,
+	switch val := v.(type) {
-		reader: reader,
+	case []byte:
-	}, nil
+		// TODO: only strings are supported in s3select output (not
+		// binary arrays) - perhaps we need to check the annotation to
+		// ensure it's UTF8 encoded.
+		value = string(val)
+	case [12]byte:
+		// TODO: This is returned for the parquet INT96 type. We just
+		// treat it same as []byte (but AWS S3 treats it as a large int)
+		// - fix this later.
+		value = string(val[:])
+	case int32:
+		value = int64(val)
+		if logicalType := se.GetLogicalType(); logicalType != nil {
+			if logicalType.IsSetDATE() {
+				value = sql.FormatSQLTimestamp(time.Unix(60*60*24*int64(val), 0).UTC())
+			}
+		} else if se.GetConvertedType() == parquettypes.ConvertedType_DATE {
+			value = sql.FormatSQLTimestamp(time.Unix(60*60*24*int64(val), 0).UTC())
+		}
+	case int64:
+		value = val
+		if logicalType := se.GetLogicalType(); logicalType != nil {
+			if ts := logicalType.GetTIMESTAMP(); ts != nil {
+				var duration time.Duration
+				// Only support UTC normalized timestamps.
+				if ts.IsAdjustedToUTC {
+					switch {
+					case ts.Unit.IsSetNANOS():
+						duration = time.Duration(val) * time.Nanosecond
+					case ts.Unit.IsSetMILLIS():
+						duration = time.Duration(val) * time.Millisecond
+					case ts.Unit.IsSetMICROS():
+						duration = time.Duration(val) * time.Microsecond
+					default:
+						return nil, errors.New("Invalid LogicalType annotation found")
+					}
+					value = sql.FormatSQLTimestamp(time.Unix(0, 0).Add(duration))
+				}
+			} else if se.GetConvertedType() == parquettypes.ConvertedType_TIMESTAMP_MILLIS {
+				duration := time.Duration(val) * time.Millisecond
+				value = sql.FormatSQLTimestamp(time.Unix(0, 0).Add(duration))
+			} else if se.GetConvertedType() == parquettypes.ConvertedType_TIMESTAMP_MICROS {
+				duration := time.Duration(val) * time.Microsecond
+				value = sql.FormatSQLTimestamp(time.Unix(0, 0).Add(duration))
+			}
+		}
+	case float32:
+		value = float64(val)
+	default:
+		value = v
+	}
+	return value, nil
 }

internal/s3select/select.go

@@ -214,10 +214,10 @@ type RequestProgress struct {
 
 // ScanRange represents the ScanRange parameter.
 type ScanRange struct {
-	// Start if byte offset form the start off the file.
+	// Start is the byte offset to read from (from the start of the file).
 	Start *uint64 `xml:"Start"`
-	// End is the last byte that should be returned, if Start is set,
+	// End is the offset of the last byte that should be returned when Start
-	// or the offset from EOF to start reading if start is not present.
+	// is set, otherwise it is the offset from EOF to start reading.
 	End *uint64 `xml:"End"`
 }
 
@@ -362,21 +362,29 @@ func (s3Select *S3Select) getProgress() (bytesScanned, bytesProcessed int64) {
 
 // Open - opens S3 object by using callback for SQL selection query.
 // Currently CSV, JSON and Apache Parquet formats are supported.
-func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadCloser, error)) error {
+func (s3Select *S3Select) Open(rsc io.ReadSeekCloser) error {
-	offset, end, err := s3Select.ScanRange.StartLen()
+	offset, length, err := s3Select.ScanRange.StartLen()
 	if err != nil {
 		return err
 	}
+	seekDirection := io.SeekStart
+	if offset < 0 {
+		seekDirection = io.SeekEnd
+	}
 	switch s3Select.Input.format {
 	case csvFormat:
-		rc, err := getReader(offset, end)
+		_, err = rsc.Seek(offset, seekDirection)
 		if err != nil {
 			return err
 		}
+		var rc io.ReadCloser = rsc
+		if length != -1 {
+			rc = newLimitedReadCloser(rsc, length)
+		}
 
 		s3Select.progressReader, err = newProgressReader(rc, s3Select.Input.CompressionType)
 		if err != nil {
-			rc.Close()
+			rsc.Close()
 			return err
 		}
 
@@ -405,14 +413,18 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos
 		}
 		return nil
 	case jsonFormat:
-		rc, err := getReader(offset, end)
+		_, err = rsc.Seek(offset, seekDirection)
 		if err != nil {
 			return err
 		}
+		var rc io.ReadCloser = rsc
+		if length != -1 {
+			rc = newLimitedReadCloser(rsc, length)
+		}
 
 		s3Select.progressReader, err = newProgressReader(rc, s3Select.Input.CompressionType)
 		if err != nil {
-			rc.Close()
+			rsc.Close()
 			return err
 		}
 
@@ -431,12 +443,12 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos
 		if !strings.EqualFold(os.Getenv("MINIO_API_SELECT_PARQUET"), "on") {
 			return errors.New("parquet format parsing not enabled on server")
 		}
-		if offset != 0 || end != -1 {
+		if offset != 0 || length != -1 {
 			// Offsets do not make sense in parquet files.
 			return errors.New("parquet format does not support offsets")
 		}
 		var err error
-		s3Select.recordReader, err = parquet.NewReader(getReader, &s3Select.Input.ParquetArgs)
+		s3Select.recordReader, err = parquet.NewParquetReader(rsc, &s3Select.Input.ParquetArgs)
 		return err
 	}
 
@@ -657,3 +669,93 @@ func NewS3Select(r io.Reader) (*S3Select, error) {
 
 	return s3Select, nil
 }
+
+//////////////////
+// Helpers
+/////////////////
+
+// limitedReadCloser is like io.LimitedReader, but also implements io.Closer.
+type limitedReadCloser struct {
+	io.LimitedReader
+	io.Closer
+}
+
+func newLimitedReadCloser(r io.ReadCloser, n int64) *limitedReadCloser {
+	return &limitedReadCloser{
+		LimitedReader: io.LimitedReader{R: r, N: n},
+		Closer:        r,
+	}
+}
+
+// ObjectSegmentReaderFn is a function that returns a reader for a contiguous
+// suffix segment of an object starting at the given (non-negative) offset.
+type ObjectSegmentReaderFn func(offset int64) (io.ReadCloser, error)
+
+// ObjectReadSeekCloser implements ReadSeekCloser interface for reading objects.
+// It uses a function that returns a io.ReadCloser for the object.
+type ObjectReadSeekCloser struct {
+	segmentReader ObjectSegmentReaderFn
+
+	size   int64 // actual object size regardless of compression/encryption
+	offset int64
+	reader io.ReadCloser
+}
+
+// NewObjectReadSeekCloser creates a new ObjectReadSeekCloser.
+func NewObjectReadSeekCloser(segmentReader ObjectSegmentReaderFn, actualSize int64) *ObjectReadSeekCloser {
+	return &ObjectReadSeekCloser{
+		segmentReader: segmentReader,
+		size:          actualSize,
+		offset:        0,
+		reader:        nil,
+	}
+}
+
+// Seek call to implement io.Seeker
+func (rsc *ObjectReadSeekCloser) Seek(offset int64, whence int) (int64, error) {
+	// fmt.Printf("actual: %v offset: %v (%v) whence: %v\n", rsc.size, offset, rsc.offset, whence)
+	switch whence {
+	case io.SeekStart:
+		rsc.offset = offset
+	case io.SeekCurrent:
+		rsc.offset += offset
+	case io.SeekEnd:
+		rsc.offset = rsc.size + offset
+	}
+	if rsc.offset < 0 {
+		return rsc.offset, errors.New("seek to invalid negative offset")
+	}
+	if rsc.offset >= rsc.size {
+		return rsc.offset, errors.New("seek past end of object")
+	}
+	if rsc.reader != nil {
+		_ = rsc.reader.Close()
+		rsc.reader = nil
+	}
+	return rsc.offset, nil
+}
+
+// Read call to implement io.Reader
+func (rsc *ObjectReadSeekCloser) Read(p []byte) (n int, err error) {
+	if rsc.reader == nil {
+		rsc.reader, err = rsc.segmentReader(rsc.offset)
+		if err != nil {
+			return 0, err
+		}
+	}
+	return rsc.reader.Read(p)
+}
+
+// Close call to implement io.Closer. Calling Read/Seek after Close reopens the
+// object for reading and a subsequent Close call is required to ensure
+// resources are freed.
+func (rsc *ObjectReadSeekCloser) Close() error {
+	if rsc.reader != nil {
+		err := rsc.reader.Close()
+		if err == nil {
+			rsc.reader = nil
+		}
+		return err
+	}
+	return nil
+}

internal/s3select/select_benchmark_test.go

@@ -20,8 +20,6 @@ package s3select
 import (
 	"bytes"
 	"encoding/csv"
-	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"strconv"
@@ -112,9 +110,7 @@ func benchmarkSelect(b *testing.B, count int, query string) {
 				b.Fatal(err)
 			}
 
-			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+			if err = s3Select.Open(newBytesRSC(csvData)); err != nil {
-				return ioutil.NopCloser(bytes.NewReader(csvData)), nil
-			}); err != nil {
 				b.Fatal(err)
 			}
 

internal/s3select/select_test.go

@@ -20,7 +20,6 @@ package s3select
 import (
 	"bytes"
 	"encoding/xml"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -35,6 +34,22 @@ import (
 	"github.com/minio/simdjson-go"
 )
 
+func newStringRSC(s string) io.ReadSeekCloser {
+	return newBytesRSC([]byte(s))
+}
+
+func newBytesRSC(b []byte) io.ReadSeekCloser {
+	r := bytes.NewReader(b)
+	segmentReader := func(offset int64) (io.ReadCloser, error) {
+		_, err := r.Seek(offset, io.SeekStart)
+		if err != nil {
+			return nil, err
+		}
+		return io.NopCloser(r), nil
+	}
+	return NewObjectReadSeekCloser(segmentReader, int64(len(b)))
+}
+
 type testResponseWriter struct {
 	statusCode int
 	response   []byte
@@ -608,13 +623,11 @@ func TestJSONQueries(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+			in := input
-				in := input
+			if len(testCase.withJSON) > 0 {
-				if len(testCase.withJSON) > 0 {
+				in = testCase.withJSON
-					in = testCase.withJSON
+			}
-				}
+			if err = s3Select.Open(newStringRSC(in)); err != nil {
-				return ioutil.NopCloser(bytes.NewBufferString(in)), nil
-			}); err != nil {
 				t.Fatal(err)
 			}
 
@@ -656,13 +669,11 @@ func TestJSONQueries(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+			in := input
-				in := input
+			if len(testCase.withJSON) > 0 {
-				if len(testCase.withJSON) > 0 {
+				in = testCase.withJSON
-					in = testCase.withJSON
+			}
-				}
+			if err = s3Select.Open(newStringRSC(in)); err != nil {
-				return ioutil.NopCloser(bytes.NewBufferString(in)), nil
-			}); err != nil {
 				t.Fatal(err)
 			}
 
@@ -743,9 +754,7 @@ func TestCSVQueries(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+			if err = s3Select.Open(newStringRSC(input)); err != nil {
-				return ioutil.NopCloser(bytes.NewBufferString(input)), nil
-			}); err != nil {
 				t.Fatal(err)
 			}
 
@@ -928,9 +937,7 @@ func TestCSVQueries2(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+			if err = s3Select.Open(newBytesRSC(testCase.input)); err != nil {
-				return ioutil.NopCloser(bytes.NewBuffer(testCase.input)), nil
-			}); err != nil {
 				t.Fatal(err)
 			}
 
@@ -1074,9 +1081,7 @@ true`,
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+			if err = s3Select.Open(newStringRSC(input)); err != nil {
-				return ioutil.NopCloser(bytes.NewBufferString(input)), nil
-			}); err != nil {
 				t.Fatal(err)
 			}
 
@@ -1220,9 +1225,7 @@ func TestCSVInput(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+			if err = s3Select.Open(newBytesRSC(csvData)); err != nil {
-				return ioutil.NopCloser(bytes.NewReader(csvData)), nil
-			}); err != nil {
 				t.Fatal(err)
 			}
 
@@ -1342,9 +1345,7 @@ func TestJSONInput(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+			if err = s3Select.Open(newBytesRSC(jsonData)); err != nil {
-				return ioutil.NopCloser(bytes.NewReader(jsonData)), nil
-			}); err != nil {
 				t.Fatal(err)
 			}
 
@@ -1616,37 +1617,7 @@ func TestCSVRanges(t *testing.T) {
 				return
 			}
 
-			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+			if err = s3Select.Open(newBytesRSC(testCase.input)); err != nil {
-				in := testCase.input
-				if offset != 0 || length != -1 {
-					// Copy from SelectObjectContentHandler
-					isSuffixLength := false
-					if offset < 0 {
-						isSuffixLength = true
-					}
-
-					if length > 0 {
-						length--
-					}
-
-					rs := &httpRangeSpec{
-						IsSuffixLength: isSuffixLength,
-						Start:          offset,
-						End:            offset + length,
-					}
-					if length == -1 {
-						rs.End = -1
-					}
-					t.Log("input, offset:", offset, "length:", length, "size:", len(in))
-					offset, length, err = rs.GetOffsetLength(int64(len(in)))
-					if err != nil {
-						return nil, err
-					}
-					t.Log("rs:", *rs, "offset:", offset, "length:", length)
-					in = in[offset : offset+length]
-				}
-				return ioutil.NopCloser(bytes.NewBuffer(in)), nil
-			}); err != nil {
 				if !testCase.wantErr {
 					t.Fatal(err)
 				}
@@ -1741,27 +1712,10 @@ func TestParquetInput(t *testing.T) {
 
 	for i, testCase := range testTable {
 		t.Run(fmt.Sprint(i), func(t *testing.T) {
-			getReader := func(offset int64, length int64) (io.ReadCloser, error) {
+			testdataFile := "testdata/testdata.parquet"
-				testdataFile := "testdata/testdata.parquet"
+			file, err := os.Open(testdataFile)
-				file, err := os.Open(testdataFile)
+			if err != nil {
-				if err != nil {
+				t.Fatal(err)
-					return nil, err
-				}
-
-				fi, err := file.Stat()
-				if err != nil {
-					return nil, err
-				}
-
-				if offset < 0 {
-					offset = fi.Size() + offset
-				}
-
-				if _, err = file.Seek(offset, io.SeekStart); err != nil {
-					return nil, err
-				}
-
-				return file, nil
 			}
 
 			s3Select, err := NewS3Select(bytes.NewReader(testCase.requestXML))
@@ -1769,10 +1723,12 @@ func TestParquetInput(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(getReader); err != nil {
+			if err = s3Select.Open(file); err != nil {
 				t.Fatal(err)
 			}
 
+			fmt.Printf("R: \nE: %s\n" /* string(w.response), */, string(testCase.expectedResult))
+
 			w := &testResponseWriter{}
 			s3Select.Evaluate(w)
 			s3Select.Close()
@@ -1862,27 +1818,10 @@ func TestParquetInputSchema(t *testing.T) {
 
 	for i, testCase := range testTable {
 		t.Run(fmt.Sprint(i), func(t *testing.T) {
-			getReader := func(offset int64, length int64) (io.ReadCloser, error) {
+			testdataFile := "testdata/lineitem_shipdate.parquet"
-				testdataFile := "testdata/lineitem_shipdate.parquet"
+			file, err := os.Open(testdataFile)
-				file, err := os.Open(testdataFile)
+			if err != nil {
-				if err != nil {
+				t.Fatal(err)
-					return nil, err
-				}
-
-				fi, err := file.Stat()
-				if err != nil {
-					return nil, err
-				}
-
-				if offset < 0 {
-					offset = fi.Size() + offset
-				}
-
-				if _, err = file.Seek(offset, io.SeekStart); err != nil {
-					return nil, err
-				}
-
-				return file, nil
 			}
 
 			s3Select, err := NewS3Select(bytes.NewReader(testCase.requestXML))
@@ -1890,7 +1829,7 @@ func TestParquetInputSchema(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(getReader); err != nil {
+			if err = s3Select.Open(file); err != nil {
 				t.Fatal(err)
 			}
 
@@ -1980,27 +1919,10 @@ func TestParquetInputSchemaCSV(t *testing.T) {
 
 	for i, testCase := range testTable {
 		t.Run(fmt.Sprint(i), func(t *testing.T) {
-			getReader := func(offset int64, length int64) (io.ReadCloser, error) {
+			testdataFile := "testdata/lineitem_shipdate.parquet"
-				testdataFile := "testdata/lineitem_shipdate.parquet"
+			file, err := os.Open(testdataFile)
-				file, err := os.Open(testdataFile)
+			if err != nil {
-				if err != nil {
+				t.Fatal(err)
-					return nil, err
-				}
-
-				fi, err := file.Stat()
-				if err != nil {
-					return nil, err
-				}
-
-				if offset < 0 {
-					offset = fi.Size() + offset
-				}
-
-				if _, err = file.Seek(offset, io.SeekStart); err != nil {
-					return nil, err
-				}
-
-				return file, nil
 			}
 
 			s3Select, err := NewS3Select(bytes.NewReader(testCase.requestXML))
@@ -2008,7 +1930,7 @@ func TestParquetInputSchemaCSV(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = s3Select.Open(getReader); err != nil {
+			if err = s3Select.Open(file); err != nil {
 				t.Fatal(err)
 			}
 
@@ -2037,76 +1959,3 @@ func TestParquetInputSchemaCSV(t *testing.T) {
 		})
 	}
 }
-
-// httpRangeSpec represents a range specification as supported by S3 GET
-// object request.
-//
-// Case 1: Not present -> represented by a nil RangeSpec
-// Case 2: bytes=1-10 (absolute start and end offsets) -> RangeSpec{false, 1, 10}
-// Case 3: bytes=10- (absolute start offset with end offset unspecified) -> RangeSpec{false, 10, -1}
-// Case 4: bytes=-30 (suffix length specification) -> RangeSpec{true, -30, -1}
-type httpRangeSpec struct {
-	// Does the range spec refer to a suffix of the object?
-	IsSuffixLength bool
-
-	// Start and end offset specified in range spec
-	Start, End int64
-}
-
-func (h *httpRangeSpec) GetLength(resourceSize int64) (rangeLength int64, err error) {
-	switch {
-	case resourceSize < 0:
-		return 0, errors.New("Resource size cannot be negative")
-
-	case h == nil:
-		rangeLength = resourceSize
-
-	case h.IsSuffixLength:
-		specifiedLen := -h.Start
-		rangeLength = specifiedLen
-		if specifiedLen > resourceSize {
-			rangeLength = resourceSize
-		}
-
-	case h.Start >= resourceSize:
-		return 0, errors.New("errInvalidRange")
-
-	case h.End > -1:
-		end := h.End
-		if resourceSize <= end {
-			end = resourceSize - 1
-		}
-		rangeLength = end - h.Start + 1
-
-	case h.End == -1:
-		rangeLength = resourceSize - h.Start
-
-	default:
-		return 0, errors.New("Unexpected range specification case")
-	}
-
-	return rangeLength, nil
-}
-
-// GetOffsetLength computes the start offset and length of the range
-// given the size of the resource
-func (h *httpRangeSpec) GetOffsetLength(resourceSize int64) (start, length int64, err error) {
-	if h == nil {
-		// No range specified, implies whole object.
-		return 0, resourceSize, nil
-	}
-
-	length, err = h.GetLength(resourceSize)
-	if err != nil {
-		return 0, 0, err
-	}
-
-	start = h.Start
-	if h.IsSuffixLength {
-		start = resourceSize + h.Start
-		if start < 0 {
-			start = 0
-		}
-	}
-	return start, length, nil
-}