mirror of
https://github.com/minio/minio.git
synced 2025-04-03 19:30:29 -04:00
fix: missing entries on first list resume (#13627)
On first list resume or when specifying a custom markers entries could be missed in rare cases. Do conservative truncation of entries when forwarding. Replaces #13619
This commit is contained in:
parent
d008e90d50
commit
c897b6a82d
@ -116,6 +116,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
|
|||||||
forward := ""
|
forward := ""
|
||||||
if len(opts.ForwardTo) > 0 && strings.HasPrefix(opts.ForwardTo, current) {
|
if len(opts.ForwardTo) > 0 && strings.HasPrefix(opts.ForwardTo, current) {
|
||||||
forward = strings.TrimPrefix(opts.ForwardTo, current)
|
forward = strings.TrimPrefix(opts.ForwardTo, current)
|
||||||
|
// Trim further directories and trailing slash.
|
||||||
if idx := strings.IndexByte(forward, '/'); idx > 0 {
|
if idx := strings.IndexByte(forward, '/'); idx > 0 {
|
||||||
forward = forward[:idx]
|
forward = forward[:idx]
|
||||||
}
|
}
|
||||||
@ -163,7 +164,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
|
|||||||
entries[i] = entry
|
entries[i] = entry
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Trim slash, maybe compiler is clever?
|
// Trim slash, since we don't know if this is folder or object.
|
||||||
entries[i] = entries[i][:len(entry)-1]
|
entries[i] = entries[i][:len(entry)-1]
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@ -214,9 +215,12 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
|
|||||||
dirStack := make([]string, 0, 5)
|
dirStack := make([]string, 0, 5)
|
||||||
prefix = "" // Remove prefix after first level as we have already filtered the list.
|
prefix = "" // Remove prefix after first level as we have already filtered the list.
|
||||||
if len(forward) > 0 {
|
if len(forward) > 0 {
|
||||||
idx := sort.SearchStrings(entries, forward)
|
// Conservative forwarding. Entries may be either objects or prefixes.
|
||||||
if idx > 0 {
|
for i, entry := range entries {
|
||||||
entries = entries[idx:]
|
if entry >= forward || strings.HasPrefix(forward, entry) {
|
||||||
|
entries = entries[i:]
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1378,6 +1378,152 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Wrapper for calling ListObjects continuation tests for both Erasure multiple disks and single node setup.
|
||||||
|
func TestListObjectsContinuation(t *testing.T) {
|
||||||
|
ExecObjectLayerTest(t, testListObjectsContinuation)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unit test for ListObjects in general.
|
||||||
|
func testListObjectsContinuation(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
|
||||||
|
t, _ := t1.(*testing.T)
|
||||||
|
testBuckets := []string{
|
||||||
|
// This bucket is used for testing ListObject operations.
|
||||||
|
"test-bucket-list-object-continuation-1",
|
||||||
|
"test-bucket-list-object-continuation-2",
|
||||||
|
}
|
||||||
|
for _, bucket := range testBuckets {
|
||||||
|
err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var err error
|
||||||
|
testObjects := []struct {
|
||||||
|
parentBucket string
|
||||||
|
name string
|
||||||
|
content string
|
||||||
|
meta map[string]string
|
||||||
|
}{
|
||||||
|
{testBuckets[0], "a/1.txt", "contentstring", nil},
|
||||||
|
{testBuckets[0], "a-1.txt", "contentstring", nil},
|
||||||
|
{testBuckets[0], "a.txt", "contentstring", nil},
|
||||||
|
{testBuckets[0], "apache2-doc/1.txt", "contentstring", nil},
|
||||||
|
{testBuckets[0], "apache2/1.txt", "contentstring", nil},
|
||||||
|
{testBuckets[0], "apache2/-sub/2.txt", "contentstring", nil},
|
||||||
|
{testBuckets[1], "azerty/1.txt", "contentstring", nil},
|
||||||
|
{testBuckets[1], "apache2-doc/1.txt", "contentstring", nil},
|
||||||
|
{testBuckets[1], "apache2/1.txt", "contentstring", nil},
|
||||||
|
}
|
||||||
|
for _, object := range testObjects {
|
||||||
|
md5Bytes := md5.Sum([]byte(object.content))
|
||||||
|
_, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
|
||||||
|
int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{UserDefined: object.meta})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Formualting the result data set to be expected from ListObjects call inside the tests,
|
||||||
|
// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
|
||||||
|
|
||||||
|
resultCases := []ListObjectsInfo{
|
||||||
|
{
|
||||||
|
Objects: []ObjectInfo{
|
||||||
|
{Name: "a-1.txt"},
|
||||||
|
{Name: "a.txt"},
|
||||||
|
{Name: "a/1.txt"},
|
||||||
|
{Name: "apache2-doc/1.txt"},
|
||||||
|
{Name: "apache2/-sub/2.txt"},
|
||||||
|
{Name: "apache2/1.txt"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Objects: []ObjectInfo{
|
||||||
|
{Name: "apache2-doc/1.txt"},
|
||||||
|
{Name: "apache2/1.txt"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Prefixes: []string{"apache2-doc/", "apache2/", "azerty/"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
testCases := []struct {
|
||||||
|
// Inputs to ListObjects.
|
||||||
|
bucketName string
|
||||||
|
prefix string
|
||||||
|
delimiter string
|
||||||
|
page int
|
||||||
|
// Expected output of ListObjects.
|
||||||
|
result ListObjectsInfo
|
||||||
|
}{
|
||||||
|
{testBuckets[0], "", "", 1, resultCases[0]},
|
||||||
|
{testBuckets[0], "a", "", 1, resultCases[0]},
|
||||||
|
{testBuckets[1], "apache", "", 1, resultCases[1]},
|
||||||
|
{testBuckets[1], "", "/", 1, resultCases[2]},
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, testCase := range testCases {
|
||||||
|
testCase := testCase
|
||||||
|
t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) {
|
||||||
|
var foundObjects []ObjectInfo
|
||||||
|
var foundPrefixes []string
|
||||||
|
var marker = ""
|
||||||
|
for {
|
||||||
|
result, err := obj.ListObjects(context.Background(), testCase.bucketName,
|
||||||
|
testCase.prefix, marker, testCase.delimiter, testCase.page)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
|
||||||
|
}
|
||||||
|
foundObjects = append(foundObjects, result.Objects...)
|
||||||
|
foundPrefixes = append(foundPrefixes, result.Prefixes...)
|
||||||
|
if !result.IsTruncated {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
marker = result.NextMarker
|
||||||
|
if len(result.Objects) > 0 {
|
||||||
|
// Discard marker, so it cannot resume listing.
|
||||||
|
marker = result.Objects[len(result.Objects)-1].Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(testCase.result.Objects) != len(foundObjects) {
|
||||||
|
t.Logf("want: %v", objInfoNames(testCase.result.Objects))
|
||||||
|
t.Logf("got: %v", objInfoNames(foundObjects))
|
||||||
|
t.Errorf("Test %d: %s: Expected number of objects in the result to be '%d', but found '%d' objects instead",
|
||||||
|
i+1, instanceType, len(testCase.result.Objects), len(foundObjects))
|
||||||
|
}
|
||||||
|
for j := 0; j < len(testCase.result.Objects); j++ {
|
||||||
|
if j >= len(foundObjects) {
|
||||||
|
t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.result.Objects[j].Name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if testCase.result.Objects[j].Name != foundObjects[j].Name {
|
||||||
|
t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, foundObjects[j].Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(testCase.result.Prefixes) != len(foundPrefixes) {
|
||||||
|
t.Logf("want: %v", testCase.result.Prefixes)
|
||||||
|
t.Logf("got: %v", foundPrefixes)
|
||||||
|
t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead",
|
||||||
|
i+1, instanceType, len(testCase.result.Prefixes), len(foundPrefixes))
|
||||||
|
}
|
||||||
|
for j := 0; j < len(testCase.result.Prefixes); j++ {
|
||||||
|
if j >= len(foundPrefixes) {
|
||||||
|
t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.result.Prefixes[j])
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if testCase.result.Prefixes[j] != foundPrefixes[j] {
|
||||||
|
t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Prefixes[j], foundPrefixes[j])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Initialize FS backend for the benchmark.
|
// Initialize FS backend for the benchmark.
|
||||||
func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) {
|
func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) {
|
||||||
var err error
|
var err error
|
||||||
|
Loading…
x
Reference in New Issue
Block a user