Handle ListObjectsV2 start-after parameter in ObjectLayer (#6078)

cmd/bucket-handlers-listobjects.go

@@ -80,17 +80,9 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// In ListObjectsV2 'continuation-token' is the marker.
-	marker := token
-	// Check if 'continuation-token' is empty.
-	if token == "" {
-		// Then we need to use 'start-after' as marker instead.
-		marker = startAfter
-	}
-
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -101,7 +93,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
 	// marshaled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
+	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return

cmd/fs-v1.go

@@ -1250,7 +1250,12 @@ func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 
 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	loi, err := fs.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	loi, err := fs.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
 	}

cmd/gateway/azure/gateway-azure.go

@@ -611,7 +611,7 @@ func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 // ListObjectsV2 - list all blobs in Azure bucket filtered by prefix
 func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
 	marker := continuationToken
-	if startAfter != "" {
+	if marker == "" {
 		marker = startAfter
 	}
 

cmd/gateway/b2/gateway-b2.go

@@ -355,12 +355,20 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 // ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
 func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
 	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
-	// fetchOwner, startAfter are not supported and unused.
+	// fetchOwner is not supported and unused.
+	marker := continuationToken
+	if marker == "" {
+		// B2's continuation token is an object name to "start at" rather than "start after"
+		// startAfter plus the lowest character B2 supports is used so that the startAfter
+		// object isn't included in the results
+		marker = startAfter + " "
+	}
+
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return loi, err
 	}
-	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, continuationToken, prefix, delimiter)
+	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
 	if lerr != nil {
 		logger.LogIf(ctx, lerr)
 		return loi, b2ToObjectError(lerr, bucket)

cmd/gateway/gcs/gateway-gcs.go

@@ -658,6 +658,10 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
 
 // ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix
 func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
+	if continuationToken == "" && startAfter != "" {
+		continuationToken = toGCSPageToken(startAfter)
+	}
+
 	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
 		Delimiter: delimiter,
 		Prefix:    prefix,

cmd/gateway/manta/gateway-manta.go

@@ -347,6 +347,13 @@ func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 		dirName = path.Join(mantaRoot, bucket, pathDir)
 	}
 
+	if marker != "" {
+		// Manta uses the marker as the key to start at rather than start after
+		// A space is appended to the marker so that the corresponding object is not
+		// included in the results
+		marker += " "
+	}
+
 	input = &storage.ListDirectoryInput{
 		DirectoryName: dirName,
 		Limit:         uint64(maxKeys),
@@ -419,6 +426,18 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 		pathBase = path.Base(prefix)
 	)
 
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	if marker != "" {
+		// Manta uses the marker as the key to start at rather than start after.
+		// A space is appended to the marker so that the corresponding object is not
+		// included in the results
+		marker += " "
+	}
+
 	if pathDir := path.Dir(prefix); pathDir == "." {
 		dirName = path.Join(mantaRoot, bucket)
 	} else {
@@ -428,7 +447,7 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 	input = &storage.ListDirectoryInput{
 		DirectoryName: dirName,
 		Limit:         uint64(maxKeys),
-		Marker:        continuationToken,
+		Marker:        marker,
 	}
 	objs, err = t.client.Dir().List(ctx, input)
 	if err != nil {

cmd/gateway/oss/gateway-oss.go

@@ -479,8 +479,11 @@ func ossListObjects(ctx context.Context, client *oss.Client, bucket, prefix, mar
 // ossListObjectsV2 lists all blobs in OSS bucket filtered by prefix.
 func ossListObjectsV2(ctx context.Context, client *oss.Client, bucket, prefix, continuationToken, delimiter string, maxKeys int,
 	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
-	// fetchOwner and startAfter are not supported and unused.
+	// fetchOwner is not supported and unused.
 	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
 
 	resultV1, err := ossListObjects(ctx, client, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {

cmd/xl-sets.go

@@ -454,7 +454,12 @@ func (s *xlSets) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo B
 
 // ListObjectsV2 lists all objects in bucket filtered by prefix
 func (s *xlSets) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	loi, err := s.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	loi, err := s.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
 	}

cmd/xl-v1-object.go

@@ -855,7 +855,12 @@ func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (er
 
 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (xl xlObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	loi, err := xl.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	loi, err := xl.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
 	}