GCS gateway allows apps to supply their own marker (#4495)

Most s3 compatible apps use object keys returned in listing as
marker. This change allows this behaviour with gateway-gcs too.

cmd/gateway-gcs.go

@@ -42,6 +42,9 @@ const (
 	// ZZZZMinioPrefix is used for metadata and multiparts. The prefix is being filtered out,
 	// hence the naming of ZZZZ (last prefix)
 	ZZZZMinioPrefix = "ZZZZ-Minio"
+	// token prefixed with GCS returned marker to differentiate
+	// from user supplied marker.
+	gcsTokenPrefix = "##minio"
 )
 
 // Check if object prefix is "ZZZZ_Minio".
@@ -303,6 +306,12 @@ func toGCSPageToken(name string) string {
 	return base64.StdEncoding.EncodeToString(b)
 }
 
+// Returns true if marker was returned by GCS, i.e prefixed with
+// ##minio by minio gcs gateway.
+func isGCSMarker(marker string) bool {
+	return strings.HasPrefix(marker, gcsTokenPrefix)
+}
+
 // ListObjects - lists all blobs in GCS bucket filtered by prefix
 func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
 	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Delimiter: delimiter, Prefix: prefix, Versions: false})
@@ -311,8 +320,26 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de
 	nextMarker := ""
 	prefixes := []string{}
 
-	// we'll set marker to continue
+	// To accommodate S3-compatible applications using
-	it.PageInfo().Token = marker
+	// ListObjectsV1 to use object keys as markers to control the
+	// listing of objects, we use the following encoding scheme to
+	// distinguish between GCS continuation tokens and application
+	// supplied markers.
+	//
+	// - NextMarker in ListObjectsV1 response is constructed by
+	//   prefixing "##minio" to the GCS continuation token,
+	//   e.g, "##minioCgRvYmoz"
+	//
+	// - Application supplied markers are used as-is to list
+	//   object keys that appear after it in the lexicographical order.
+
+	// If application is using GCS continuation token we should
+	// strip the gcsTokenPrefix we added.
+	gcsMarker := isGCSMarker(marker)
+	if gcsMarker {
+		it.PageInfo().Token = strings.TrimPrefix(marker, gcsTokenPrefix)
+	}
+
 	it.PageInfo().MaxSize = maxKeys
 
 	objects := []ObjectInfo{}
@@ -348,6 +375,10 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de
 		} else if attrs.Prefix != "" {
 			prefixes = append(prefixes, attrs.Prefix)
 			continue
+		} else if !gcsMarker && attrs.Name <= marker {
+			// if user supplied a marker don't append
+			// objects until we reach marker (and skip it).
+			continue
 		}
 
 		objects = append(objects, ObjectInfo{
@@ -364,7 +395,7 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de
 
 	return ListObjectsInfo{
 		IsTruncated: isTruncated,
-		NextMarker:  nextMarker,
+		NextMarker:  gcsTokenPrefix + nextMarker,
 		Prefixes:    prefixes,
 		Objects:     objects,
 	}, nil

cmd/gateway-gcs_test.go

@@ -164,3 +164,35 @@ func TestIsGCSPrefix(t *testing.T) {
 		}
 	}
 }
+
+// Test for isGCSMarker.
+func TestIsGCSMarker(t *testing.T) {
+	testCases := []struct {
+		marker   string
+		expected bool
+	}{
+		{
+			marker:   "##miniogcs123",
+			expected: true,
+		},
+		{
+			marker:   "##mini_notgcs123",
+			expected: false,
+		},
+		{
+			marker:   "#minioagainnotgcs123",
+			expected: false,
+		},
+		{
+			marker:   "obj1",
+			expected: false,
+		},
+	}
+
+	for i, tc := range testCases {
+		if actual := isGCSMarker(tc.marker); actual != tc.expected {
+			t.Errorf("Test %d: marker is %s, expected %v but got %v",
+				i+1, tc.marker, tc.expected, actual)
+		}
+	}
+}

cmd/gateway-handlers.go

@@ -741,15 +741,11 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Extract all the litsObjectsV1 query params to their native values.
+	// Extract all the listObjectsV1 query params to their native
+	// values.  N B We delegate validation of params to respective
+	// gateway backends.
 	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
 
-	// Validate all the query params before beginning to serve the request.
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
-
 	listObjects := objectAPI.ListObjects
 	if reqAuthType == authTypeAnonymous {
 		listObjects = objectAPI.AnonListObjects