diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 2bec6a701..98117f405 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -2236,8 +2236,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) { apiErr = ErrSlowDownWrite case InsufficientReadQuorum: apiErr = ErrSlowDownRead - case InvalidMarkerPrefixCombination: - apiErr = ErrNotImplemented case InvalidUploadIDKeyCombination: apiErr = ErrNotImplemented case MalformedUploadID: diff --git a/cmd/api-errors_test.go b/cmd/api-errors_test.go index a0fae3f75..d57817397 100644 --- a/cmd/api-errors_test.go +++ b/cmd/api-errors_test.go @@ -42,7 +42,6 @@ var toAPIErrorTests = []struct { {err: InvalidPart{}, errCode: ErrInvalidPart}, {err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead}, {err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite}, - {err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented}, {err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented}, {err: MalformedUploadID{}, errCode: ErrNoSuchUpload}, {err: PartTooSmall{}, errCode: ErrEntityTooSmall}, diff --git a/cmd/metacache-server-pool.go b/cmd/metacache-server-pool.go index 61ef5cc76..f454393b8 100644 --- a/cmd/metacache-server-pool.go +++ b/cmd/metacache-server-pool.go @@ -60,6 +60,10 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) ( if err := checkListObjsArgs(ctx, o.Bucket, o.Prefix, o.Marker, z); err != nil { return entries, err } + // Marker points to before the prefix, just ignore it. + if o.Marker < o.Prefix { + o.Marker = "" + } // Marker is set validate pre-condition. if o.Marker != "" && o.Prefix != "" { diff --git a/cmd/object-api-errors.go b/cmd/object-api-errors.go index 6e67d0490..e36887627 100644 --- a/cmd/object-api-errors.go +++ b/cmd/object-api-errors.go @@ -348,15 +348,6 @@ func (e InvalidUploadIDKeyCombination) Error() string { return fmt.Sprintf("Invalid combination of uploadID marker '%s' and marker '%s'", e.UploadIDMarker, e.KeyMarker) } -// InvalidMarkerPrefixCombination - invalid marker and prefix combination. -type InvalidMarkerPrefixCombination struct { - Marker, Prefix string -} - -func (e InvalidMarkerPrefixCombination) Error() string { - return fmt.Sprintf("Invalid combination of marker '%s' and prefix '%s'", e.Marker, e.Prefix) -} - // BucketPolicyNotFound - no bucket policy found. type BucketPolicyNotFound GenericError diff --git a/cmd/object-api-input-checks.go b/cmd/object-api-input-checks.go index e790c05c9..d530827d0 100644 --- a/cmd/object-api-input-checks.go +++ b/cmd/object-api-input-checks.go @@ -78,17 +78,6 @@ func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string, obj g Object: prefix, } } - // Verify if marker has prefix. - if marker != "" && !HasPrefix(marker, prefix) { - logger.LogIf(ctx, InvalidMarkerPrefixCombination{ - Marker: marker, - Prefix: prefix, - }) - return InvalidMarkerPrefixCombination{ - Marker: marker, - Prefix: prefix, - } - } return nil } diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go index 797ef6204..73dc162ec 100644 --- a/cmd/object-api-listobjects_test.go +++ b/cmd/object-api-listobjects_test.go @@ -393,7 +393,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v resultCases := []ListObjectsInfo{ // ListObjectsResult-0. // Testing for listing all objects in the bucket, (testCase 20,21,22). - { + 0: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "Asia-maps.png"}, @@ -409,7 +409,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-1. // Used for asserting the truncated case, (testCase 23). - { + 1: { IsTruncated: true, Objects: []ObjectInfo{ {Name: "Asia-maps.png"}, @@ -421,7 +421,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-2. // (TestCase 24). - { + 2: { IsTruncated: true, Objects: []ObjectInfo{ {Name: "Asia-maps.png"}, @@ -432,7 +432,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-3. // (TestCase 25). - { + 3: { IsTruncated: true, Objects: []ObjectInfo{ {Name: "Asia-maps.png"}, @@ -443,7 +443,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-4. // Again used for truncated case. // (TestCase 26). - { + 4: { IsTruncated: true, Objects: []ObjectInfo{ {Name: "Asia-maps.png"}, @@ -452,7 +452,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-5. // Used for Asserting prefixes. // Used for test case with prefix "new", (testCase 27-29). - { + 5: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "newPrefix0"}, @@ -463,7 +463,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-6. // Used for Asserting prefixes. // Used for test case with prefix = "obj", (testCase 30). - { + 6: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "obj0"}, @@ -474,7 +474,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-7. // Used for Asserting prefixes and truncation. // Used for test case with prefix = "new" and maxKeys = 1, (testCase 31). - { + 7: { IsTruncated: true, Objects: []ObjectInfo{ {Name: "newPrefix0"}, @@ -483,7 +483,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-8. // Used for Asserting prefixes. // Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32). - { + 8: { IsTruncated: true, Objects: []ObjectInfo{ {Name: "obj0"}, @@ -493,7 +493,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-9. // Used for asserting the case with marker, but without prefix. // marker is set to "newPrefix0" in the testCase, (testCase 33). - { + 9: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "newPrefix1"}, @@ -505,7 +505,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-10. // marker is set to "newPrefix1" in the testCase, (testCase 34). - { + 10: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "newzen/zen/recurse/again/again/again/pics"}, @@ -516,7 +516,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-11. // marker is set to "obj0" in the testCase, (testCase 35). - { + 11: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "obj1"}, @@ -525,7 +525,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-12. // Marker is set to "obj1" in the testCase, (testCase 36). - { + 12: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "obj2"}, @@ -533,7 +533,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-13. // Marker is set to "man" in the testCase, (testCase37). - { + 13: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "newPrefix0"}, @@ -546,7 +546,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-14. // Marker is set to "Abc" in the testCase, (testCase 39). - { + 14: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "Asia-maps.png"}, @@ -562,7 +562,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-15. // Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40). - { + 15: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, @@ -576,7 +576,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-16. // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41). - { + 16: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "newPrefix0"}, @@ -591,7 +591,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // Used for asserting the case with marker, without prefix but with truncation. // Marker = "newPrefix0" & maxKeys = 3 in the testCase, (testCase42). // Output truncated to 3 values. - { + 17: { IsTruncated: true, Objects: []ObjectInfo{ {Name: "newPrefix1"}, @@ -602,7 +602,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-18. // Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43). // Output truncated to 1 value. - { + 18: { IsTruncated: true, Objects: []ObjectInfo{ {Name: "newzen/zen/recurse/again/again/again/pics"}, @@ -611,7 +611,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-19. // Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44). // Output truncated to 1 value. - { + 19: { IsTruncated: true, Objects: []ObjectInfo{ {Name: "obj1"}, @@ -619,7 +619,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-20. // Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45). - { + 20: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "obj1"}, @@ -628,7 +628,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-21. // Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46). - { + 21: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "obj2"}, @@ -636,7 +636,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-22. // Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47). - { + 22: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "newPrefix1"}, @@ -645,7 +645,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-23. // Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55). - { + 23: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "Asia/India/India-summer-photos-1"}, @@ -655,7 +655,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-24. // Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56). - { + 24: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "Asia-maps.png"}, @@ -666,7 +666,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-25. // Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57). - { + 25: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "Asia-maps.png"}, @@ -675,7 +675,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-26. // prefix = "new" and delimiter is set in the testCase.(testCase 58). - { + 26: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "newPrefix0"}, @@ -685,7 +685,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-27. // Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59). - { + 27: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "Asia/India/India-summer-photos-1"}, @@ -694,7 +694,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-28. // Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60). - { + 28: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "newPrefix0"}, @@ -707,7 +707,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-29. // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimiter set, (testCase 61). - { + 29: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "newPrefix0"}, @@ -720,12 +720,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, // ListObjectsResult-30. // Prefix and Delimiter is set to '/', (testCase 62). - { + 30: { IsTruncated: false, Objects: []ObjectInfo{}, }, // ListObjectsResult-31 Empty directory, recursive listing - { + 31: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "obj1"}, @@ -734,7 +734,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, }, // ListObjectsResult-32 Empty directory, non recursive listing - { + 32: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "obj1"}, @@ -743,7 +743,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v Prefixes: []string{"temporary/"}, }, // ListObjectsResult-33 Listing empty directory only - { + 33: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "temporary/0/"}, @@ -752,12 +752,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v // ListObjectsResult-34: // * Listing with marker > last object should return empty // * Listing an object with a trailing slash and '/' delimiter - { + 34: { IsTruncated: false, Objects: []ObjectInfo{}, }, // ListObjectsResult-35 list with custom uncommon delimiter - { + 35: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "file1/receipt.json"}, @@ -765,12 +765,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v Prefixes: []string{"file1/guidSplunk"}, }, // ListObjectsResult-36 list with nextmarker prefix and maxKeys set to 1. - { + 36: { IsTruncated: true, Prefixes: []string{"dir/day_id=2017-10-10/"}, }, // ListObjectsResult-37 list with prefix match 2 levels deep - { + 37: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "foo/201910/1112"}, @@ -778,7 +778,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, }, // ListObjectsResult-38 list with prefix match 1 level deep - { + 38: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "foo/201910/1112"}, @@ -788,14 +788,14 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v }, }, // ListObjectsResult-39 list with prefix match 1 level deep - { + 39: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "201910/foo/bar/xl.meta/1.txt"}, }, }, // ListObjectsResult-40 - { + 40: { IsTruncated: false, Objects: []ObjectInfo{ {Name: "aaa"}, @@ -829,9 +829,10 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v {"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, {"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, {"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, - // Testing for failure cases with both perfix and marker (11). - // The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix). - {"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false}, + // If marker is *after* the last possible object from the prefix it should return an empty list. + {"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true}, + // If the marker is *before* the first possible object from the prefix it should return the first object. + {"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true}, // Setting a non-existing directory to be prefix (12-13). {"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true}, {"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true}, @@ -1573,9 +1574,8 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand {"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, {"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, {"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, - // Testing for failure cases with both perfix and marker (9). - // The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix). - {"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false}, + // If marker is *after* the last possible object from the prefix it should return an empty list. + {"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true}, // Setting a non-existing directory to be prefix (10-11). {"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true}, {"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true}, @@ -1671,6 +1671,7 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand {testBuckets[4], "file1/", "", "guidSplunk", 1000, resultCases[35], nil, true}, // Test listing at prefix with expected prefix markers {testBuckets[5], "dir/", "", SlashSeparator, 1, resultCases[36], nil, true}, + {"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true}, } for i, testCase := range testCases { diff --git a/cmd/object-api-multipart_test.go b/cmd/object-api-multipart_test.go index 4680c3d2e..c79436eb1 100644 --- a/cmd/object-api-multipart_test.go +++ b/cmd/object-api-multipart_test.go @@ -1051,12 +1051,11 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan // Valid, existing bucket, delimiter not supported, returns empty values (Test number 8-9). {bucketNames[0], "", "", "", "*", 0, ListMultipartsInfo{Delimiter: "*"}, nil, true}, {bucketNames[0], "", "", "", "-", 0, ListMultipartsInfo{Delimiter: "-"}, nil, true}, - // Testing for failure cases with both perfix and marker (Test number 10). - // The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix). + // If marker is *after* the last possible object from the prefix it should return an empty list. { - bucketNames[0], "asia", "europe-object", "", "", 0, - ListMultipartsInfo{}, - fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false, + bucketNames[0], "Asia", "europe-object", "", "", 0, + ListMultipartsInfo{KeyMarker: "europe-object", Prefix: "Asia", IsTruncated: false}, + nil, true, }, // Setting an invalid combination of uploadIDMarker and Marker (Test number 11-12). {