MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve ListObject Compatibility (#18099) 

Do not error out when a provided marker is before or after the prefix, but instead just ignore it if before and return an empty list when after.

Fixes #18093
This commit is contained in:
Klaus Post 2023-09-25 08:13:08 -07:00 committed by GitHub
parent 57f84a8b4c
commit 21e8e071d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 56 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/api-errors.go
View File

@ -2236,8 +2236,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrSlowDownWrite
case InsufficientReadQuorum:
apiErr = ErrSlowDownRead
case InvalidMarkerPrefixCombination:
apiErr = ErrNotImplemented
case InvalidUploadIDKeyCombination:
apiErr = ErrNotImplemented
case MalformedUploadID:

1
cmd/api-errors_test.go
View File

@ -42,7 +42,6 @@ var toAPIErrorTests = []struct {
{err: InvalidPart{}, errCode: ErrInvalidPart},
{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
{err: PartTooSmall{}, errCode: ErrEntityTooSmall},

4
cmd/metacache-server-pool.go
View File

@ -60,6 +60,10 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
if err := checkListObjsArgs(ctx, o.Bucket, o.Prefix, o.Marker, z); err != nil {
return entries, err
}
// Marker points to before the prefix, just ignore it.
if o.Marker < o.Prefix {
o.Marker = ""
}
// Marker is set validate pre-condition.
if o.Marker != "" && o.Prefix != "" {

9
cmd/object-api-errors.go
View File

@ -348,15 +348,6 @@ func (e InvalidUploadIDKeyCombination) Error() string {
return fmt.Sprintf("Invalid combination of uploadID marker '%s' and marker '%s'", e.UploadIDMarker, e.KeyMarker)
}
// InvalidMarkerPrefixCombination - invalid marker and prefix combination.
type InvalidMarkerPrefixCombination struct {
Marker, Prefix string
}
func (e InvalidMarkerPrefixCombination) Error() string {
return fmt.Sprintf("Invalid combination of marker '%s' and prefix '%s'", e.Marker, e.Prefix)
}
// BucketPolicyNotFound - no bucket policy found.
type BucketPolicyNotFound GenericError

11
cmd/object-api-input-checks.go
View File

@ -78,17 +78,6 @@ func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string, obj g
Object: prefix,
}
}
// Verify if marker has prefix.
if marker != "" && !HasPrefix(marker, prefix) {
logger.LogIf(ctx, InvalidMarkerPrefixCombination{
Marker: marker,
Prefix: prefix,
})
return InvalidMarkerPrefixCombination{
Marker: marker,
Prefix: prefix,
}
}
return nil
}

95
cmd/object-api-listobjects_test.go
View File

@ -393,7 +393,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
resultCases := []ListObjectsInfo{
// ListObjectsResult-0.
// Testing for listing all objects in the bucket, (testCase 20,21,22).
{
0: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "Asia-maps.png"},
@ -409,7 +409,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-1.
// Used for asserting the truncated case, (testCase 23).
{
1: {
IsTruncated: true,
Objects: []ObjectInfo{
{Name: "Asia-maps.png"},
@ -421,7 +421,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-2.
// (TestCase 24).
{
2: {
IsTruncated: true,
Objects: []ObjectInfo{
{Name: "Asia-maps.png"},
@ -432,7 +432,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-3.
// (TestCase 25).
{
3: {
IsTruncated: true,
Objects: []ObjectInfo{
{Name: "Asia-maps.png"},
@ -443,7 +443,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-4.
// Again used for truncated case.
// (TestCase 26).
{
4: {
IsTruncated: true,
Objects: []ObjectInfo{
{Name: "Asia-maps.png"},
@ -452,7 +452,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-5.
// Used for Asserting prefixes.
// Used for test case with prefix "new", (testCase 27-29).
{
5: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "newPrefix0"},
@ -463,7 +463,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-6.
// Used for Asserting prefixes.
// Used for test case with prefix = "obj", (testCase 30).
{
6: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "obj0"},
@ -474,7 +474,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-7.
// Used for Asserting prefixes and truncation.
// Used for test case with prefix = "new" and maxKeys = 1, (testCase 31).
{
7: {
IsTruncated: true,
Objects: []ObjectInfo{
{Name: "newPrefix0"},
@ -483,7 +483,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-8.
// Used for Asserting prefixes.
// Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32).
{
8: {
IsTruncated: true,
Objects: []ObjectInfo{
{Name: "obj0"},
@ -493,7 +493,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-9.
// Used for asserting the case with marker, but without prefix.
// marker is set to "newPrefix0" in the testCase, (testCase 33).
{
9: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "newPrefix1"},
@ -505,7 +505,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-10.
// marker is set to "newPrefix1" in the testCase, (testCase 34).
{
10: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "newzen/zen/recurse/again/again/again/pics"},
@ -516,7 +516,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-11.
// marker is set to "obj0" in the testCase, (testCase 35).
{
11: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "obj1"},
@ -525,7 +525,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-12.
// Marker is set to "obj1" in the testCase, (testCase 36).
{
12: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "obj2"},
@ -533,7 +533,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-13.
// Marker is set to "man" in the testCase, (testCase37).
{
13: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "newPrefix0"},
@ -546,7 +546,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-14.
// Marker is set to "Abc" in the testCase, (testCase 39).
{
14: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "Asia-maps.png"},
@ -562,7 +562,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-15.
// Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40).
{
15: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
@ -576,7 +576,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-16.
// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41).
{
16: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "newPrefix0"},
@ -591,7 +591,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// Used for asserting the case with marker, without prefix but with truncation.
// Marker = "newPrefix0" & maxKeys = 3 in the testCase, (testCase42).
// Output truncated to 3 values.
{
17: {
IsTruncated: true,
Objects: []ObjectInfo{
{Name: "newPrefix1"},
@ -602,7 +602,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-18.
// Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43).
// Output truncated to 1 value.
{
18: {
IsTruncated: true,
Objects: []ObjectInfo{
{Name: "newzen/zen/recurse/again/again/again/pics"},
@ -611,7 +611,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-19.
// Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44).
// Output truncated to 1 value.
{
19: {
IsTruncated: true,
Objects: []ObjectInfo{
{Name: "obj1"},
@ -619,7 +619,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-20.
// Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45).
{
20: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "obj1"},
@ -628,7 +628,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-21.
// Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46).
{
21: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "obj2"},
@ -636,7 +636,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-22.
// Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47).
{
22: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "newPrefix1"},
@ -645,7 +645,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-23.
// Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55).
{
23: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "Asia/India/India-summer-photos-1"},
@ -655,7 +655,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-24.
// Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56).
{
24: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "Asia-maps.png"},
@ -666,7 +666,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-25.
// Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57).
{
25: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "Asia-maps.png"},
@ -675,7 +675,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-26.
// prefix = "new" and delimiter is set in the testCase.(testCase 58).
{
26: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "newPrefix0"},
@ -685,7 +685,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-27.
// Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59).
{
27: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "Asia/India/India-summer-photos-1"},
@ -694,7 +694,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-28.
// Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60).
{
28: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "newPrefix0"},
@ -707,7 +707,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-29.
// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimiter set, (testCase 61).
{
29: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "newPrefix0"},
@ -720,12 +720,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
// ListObjectsResult-30.
// Prefix and Delimiter is set to '/', (testCase 62).
{
30: {
IsTruncated: false,
Objects: []ObjectInfo{},
},
// ListObjectsResult-31 Empty directory, recursive listing
{
31: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "obj1"},
@ -734,7 +734,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
},
// ListObjectsResult-32 Empty directory, non recursive listing
{
32: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "obj1"},
@ -743,7 +743,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
Prefixes: []string{"temporary/"},
},
// ListObjectsResult-33 Listing empty directory only
{
33: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "temporary/0/"},
@ -752,12 +752,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
// ListObjectsResult-34:
// * Listing with marker > last object should return empty
// * Listing an object with a trailing slash and '/' delimiter
{
34: {
IsTruncated: false,
Objects: []ObjectInfo{},
},
// ListObjectsResult-35 list with custom uncommon delimiter
{
35: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "file1/receipt.json"},
@ -765,12 +765,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
Prefixes: []string{"file1/guidSplunk"},
},
// ListObjectsResult-36 list with nextmarker prefix and maxKeys set to 1.
{
36: {
IsTruncated: true,
Prefixes: []string{"dir/day_id=2017-10-10/"},
},
// ListObjectsResult-37 list with prefix match 2 levels deep
{
37: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "foo/201910/1112"},
@ -778,7 +778,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
},
// ListObjectsResult-38 list with prefix match 1 level deep
{
38: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "foo/201910/1112"},
@ -788,14 +788,14 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
},
},
// ListObjectsResult-39 list with prefix match 1 level deep
{
39: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "201910/foo/bar/xl.meta/1.txt"},
},
},
// ListObjectsResult-40
{
40: {
IsTruncated: false,
Objects: []ObjectInfo{
{Name: "aaa"},
@ -829,9 +829,10 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
// Testing for failure cases with both perfix and marker (11).
// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
// If marker is *after* the last possible object from the prefix it should return an empty list.
{"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true},
// If the marker is *before* the first possible object from the prefix it should return the first object.
{"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true},
// Setting a non-existing directory to be prefix (12-13).
{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
{"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
@ -1573,9 +1574,8 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
// Testing for failure cases with both perfix and marker (9).
// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
// If marker is *after* the last possible object from the prefix it should return an empty list.
{"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true},
// Setting a non-existing directory to be prefix (10-11).
{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
{"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
@ -1671,6 +1671,7 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
{testBuckets[4], "file1/", "", "guidSplunk", 1000, resultCases[35], nil, true},
// Test listing at prefix with expected prefix markers
{testBuckets[5], "dir/", "", SlashSeparator, 1, resultCases[36], nil, true},
{"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true},
}
for i, testCase := range testCases {

9
cmd/object-api-multipart_test.go
View File

@ -1051,12 +1051,11 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
// Valid, existing bucket, delimiter not supported, returns empty values (Test number 8-9).
{bucketNames[0], "", "", "", "*", 0, ListMultipartsInfo{Delimiter: "*"}, nil, true},
{bucketNames[0], "", "", "", "-", 0, ListMultipartsInfo{Delimiter: "-"}, nil, true},
// Testing for failure cases with both perfix and marker (Test number 10).
// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
// If marker is *after* the last possible object from the prefix it should return an empty list.
{
bucketNames[0], "asia", "europe-object", "", "", 0,
ListMultipartsInfo{},
fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false,
bucketNames[0], "Asia", "europe-object", "", "", 0,
ListMultipartsInfo{KeyMarker: "europe-object", Prefix: "Asia", IsTruncated: false},
nil, true,
},
// Setting an invalid combination of uploadIDMarker and Marker (Test number 11-12).
{