Improve ListObject Compatibility (#18099)

Do not error out when a provided marker is before or after the prefix, but instead just ignore it if before and return an empty list when after.

Fixes #18093

cmd/api-errors.go

@@ -2236,8 +2236,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrSlowDownWrite
 	case InsufficientReadQuorum:
 		apiErr = ErrSlowDownRead
-	case InvalidMarkerPrefixCombination:
-		apiErr = ErrNotImplemented
 	case InvalidUploadIDKeyCombination:
 		apiErr = ErrNotImplemented
 	case MalformedUploadID:

cmd/api-errors_test.go

@@ -42,7 +42,6 @@ var toAPIErrorTests = []struct {
 	{err: InvalidPart{}, errCode: ErrInvalidPart},
 	{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
 	{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
-	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
 	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
 	{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
 	{err: PartTooSmall{}, errCode: ErrEntityTooSmall},

cmd/metacache-server-pool.go

@@ -60,6 +60,10 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	if err := checkListObjsArgs(ctx, o.Bucket, o.Prefix, o.Marker, z); err != nil {
 		return entries, err
 	}
+	// Marker points to before the prefix, just ignore it.
+	if o.Marker < o.Prefix {
+		o.Marker = ""
+	}
 
 	// Marker is set validate pre-condition.
 	if o.Marker != "" && o.Prefix != "" {

cmd/object-api-errors.go

@@ -348,15 +348,6 @@ func (e InvalidUploadIDKeyCombination) Error() string {
 	return fmt.Sprintf("Invalid combination of uploadID marker '%s' and marker '%s'", e.UploadIDMarker, e.KeyMarker)
 }
 
-// InvalidMarkerPrefixCombination - invalid marker and prefix combination.
-type InvalidMarkerPrefixCombination struct {
-	Marker, Prefix string
-}
-
-func (e InvalidMarkerPrefixCombination) Error() string {
-	return fmt.Sprintf("Invalid combination of marker '%s' and prefix '%s'", e.Marker, e.Prefix)
-}
-
 // BucketPolicyNotFound - no bucket policy found.
 type BucketPolicyNotFound GenericError
 

cmd/object-api-input-checks.go

@@ -78,17 +78,6 @@ func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string, obj g
 			Object: prefix,
 		}
 	}
-	// Verify if marker has prefix.
-	if marker != "" && !HasPrefix(marker, prefix) {
-		logger.LogIf(ctx, InvalidMarkerPrefixCombination{
-			Marker: marker,
-			Prefix: prefix,
-		})
-		return InvalidMarkerPrefixCombination{
-			Marker: marker,
-			Prefix: prefix,
-		}
-	}
 	return nil
 }
 

cmd/object-api-listobjects_test.go

@@ -393,7 +393,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 	resultCases := []ListObjectsInfo{
 		// ListObjectsResult-0.
 		// Testing for listing all objects in the bucket, (testCase 20,21,22).
-		{
+		0: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -409,7 +409,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-1.
 		// Used for asserting the truncated case, (testCase 23).
-		{
+		1: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -421,7 +421,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-2.
 		// (TestCase 24).
-		{
+		2: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -432,7 +432,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-3.
 		// (TestCase 25).
-		{
+		3: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -443,7 +443,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-4.
 		// Again used for truncated case.
 		// (TestCase 26).
-		{
+		4: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -452,7 +452,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-5.
 		// Used for Asserting prefixes.
 		// Used for test case with prefix "new", (testCase 27-29).
-		{
+		5: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -463,7 +463,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-6.
 		// Used for Asserting prefixes.
 		// Used for test case with prefix = "obj", (testCase 30).
-		{
+		6: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj0"},
@@ -474,7 +474,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-7.
 		// Used for Asserting prefixes and truncation.
 		// Used for test case with prefix = "new" and maxKeys = 1, (testCase 31).
-		{
+		7: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -483,7 +483,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-8.
 		// Used for Asserting prefixes.
 		// Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32).
-		{
+		8: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "obj0"},
@@ -493,7 +493,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-9.
 		// Used for asserting the case with marker, but without prefix.
 		// marker is set to "newPrefix0" in the testCase, (testCase 33).
-		{
+		9: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix1"},
@@ -505,7 +505,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-10.
 		// marker is set to "newPrefix1" in the testCase, (testCase 34).
-		{
+		10: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newzen/zen/recurse/again/again/again/pics"},
@@ -516,7 +516,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-11.
 		// marker is set to "obj0" in the testCase, (testCase 35).
-		{
+		11: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -525,7 +525,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-12.
 		// Marker is set to "obj1" in the testCase, (testCase 36).
-		{
+		12: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj2"},
@@ -533,7 +533,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-13.
 		// Marker is set to "man" in the testCase, (testCase37).
-		{
+		13: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -546,7 +546,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-14.
 		// Marker is set to "Abc" in the testCase, (testCase 39).
-		{
+		14: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -562,7 +562,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-15.
 		// Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40).
-		{
+		15: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
@@ -576,7 +576,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-16.
 		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41).
-		{
+		16: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -591,7 +591,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// Used for asserting the case with marker, without prefix but with truncation.
 		// Marker =  "newPrefix0" & maxKeys = 3 in the testCase, (testCase42).
 		// Output truncated to 3 values.
-		{
+		17: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix1"},
@@ -602,7 +602,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-18.
 		// Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43).
 		// Output truncated to 1 value.
-		{
+		18: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "newzen/zen/recurse/again/again/again/pics"},
@@ -611,7 +611,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-19.
 		// Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44).
 		// Output truncated to 1 value.
-		{
+		19: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -619,7 +619,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-20.
 		// Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45).
-		{
+		20: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -628,7 +628,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-21.
 		// Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46).
-		{
+		21: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj2"},
@@ -636,7 +636,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-22.
 		// Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47).
-		{
+		22: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix1"},
@@ -645,7 +645,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-23.
 		// Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55).
-		{
+		23: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia/India/India-summer-photos-1"},
@@ -655,7 +655,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 
 		// ListObjectsResult-24.
 		// Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56).
-		{
+		24: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -666,7 +666,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 
 		// ListObjectsResult-25.
 		// Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57).
-		{
+		25: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -675,7 +675,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-26.
 		// prefix = "new" and delimiter is set in the testCase.(testCase 58).
-		{
+		26: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -685,7 +685,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-27.
 		// Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59).
-		{
+		27: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia/India/India-summer-photos-1"},
@@ -694,7 +694,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-28.
 		// Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60).
-		{
+		28: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -707,7 +707,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-29.
 		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimiter set, (testCase 61).
-		{
+		29: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -720,12 +720,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-30.
 		// Prefix and Delimiter is set to '/', (testCase 62).
-		{
+		30: {
 			IsTruncated: false,
 			Objects:     []ObjectInfo{},
 		},
 		// ListObjectsResult-31 Empty directory, recursive listing
-		{
+		31: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -734,7 +734,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			},
 		},
 		// ListObjectsResult-32 Empty directory, non recursive listing
-		{
+		32: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -743,7 +743,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			Prefixes: []string{"temporary/"},
 		},
 		// ListObjectsResult-33 Listing empty directory only
-		{
+		33: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "temporary/0/"},
@@ -752,12 +752,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-34:
 		//    * Listing with marker > last object should return empty
 		//    * Listing an object with a trailing slash and '/' delimiter
-		{
+		34: {
 			IsTruncated: false,
 			Objects:     []ObjectInfo{},
 		},
 		// ListObjectsResult-35 list with custom uncommon delimiter
-		{
+		35: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "file1/receipt.json"},
@@ -765,12 +765,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			Prefixes: []string{"file1/guidSplunk"},
 		},
 		// ListObjectsResult-36 list with nextmarker prefix and maxKeys set to 1.
-		{
+		36: {
 			IsTruncated: true,
 			Prefixes:    []string{"dir/day_id=2017-10-10/"},
 		},
 		// ListObjectsResult-37 list with prefix match 2 levels deep
-		{
+		37: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "foo/201910/1112"},
@@ -778,7 +778,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			},
 		},
 		// ListObjectsResult-38 list with prefix match 1 level deep
-		{
+		38: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "foo/201910/1112"},
@@ -788,14 +788,14 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			},
 		},
 		// ListObjectsResult-39 list with prefix match 1 level deep
-		{
+		39: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "201910/foo/bar/xl.meta/1.txt"},
 			},
 		},
 		// ListObjectsResult-40
-		{
+		40: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "aaa"},
@@ -829,9 +829,10 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
 		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
 		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
-		// Testing for failure cases with both perfix and marker (11).
+		// If marker is *after* the last possible object from the prefix it should return an empty list.
-		// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true},
-		{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
+		// If the marker is *before* the first possible object from the prefix it should return the first object.
+		{"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true},
 		// Setting a non-existing directory to be prefix (12-13).
 		{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
 		{"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
@@ -1573,9 +1574,8 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
 		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
 		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
 		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
-		// Testing for failure cases with both perfix and marker (9).
+		// If marker is *after* the last possible object from the prefix it should return an empty list.
-		// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true},
-		{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
 		// Setting a non-existing directory to be prefix (10-11).
 		{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
 		{"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
@@ -1671,6 +1671,7 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
 		{testBuckets[4], "file1/", "", "guidSplunk", 1000, resultCases[35], nil, true},
 		// Test listing at prefix with expected prefix markers
 		{testBuckets[5], "dir/", "", SlashSeparator, 1, resultCases[36], nil, true},
+		{"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true},
 	}
 
 	for i, testCase := range testCases {

cmd/object-api-multipart_test.go

@@ -1051,12 +1051,11 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 		// Valid, existing bucket, delimiter not supported, returns empty values (Test number 8-9).
 		{bucketNames[0], "", "", "", "*", 0, ListMultipartsInfo{Delimiter: "*"}, nil, true},
 		{bucketNames[0], "", "", "", "-", 0, ListMultipartsInfo{Delimiter: "-"}, nil, true},
-		// Testing for failure cases with both perfix and marker (Test number 10).
+		// If marker is *after* the last possible object from the prefix it should return an empty list.
-		// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
 		{
-			bucketNames[0], "asia", "europe-object", "", "", 0,
+			bucketNames[0], "Asia", "europe-object", "", "", 0,
-			ListMultipartsInfo{},
+			ListMultipartsInfo{KeyMarker: "europe-object", Prefix: "Asia", IsTruncated: false},
-			fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false,
+			nil, true,
 		},
 		// Setting an invalid combination of uploadIDMarker and Marker (Test number 11-12).
 		{