MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

request only 1 best version for latest version resolution (#14625)

Browse Source 
ListObjects, ListObjectsV2 calls are being heavily taxed when
there are many versions on objects left over from a previous
release or ILM was never setup to clean them up. Instead
of being absolutely correct at resolving the exact latest
version of an object, we simply rely on the top most 1
version and resolve the rest.

Once we have obtained the top most "1" version for
ListObject, ListObjectsV2 call we break out.
This commit is contained in:
Harshavardhana
2022-03-25 08:50:07 -07:00
committed by GitHub
parent 401958938d
commit f046f557fa
4 changed files with 98 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
cmd/xl-storage-format-v2_test.go
View File

@@ -401,6 +401,55 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
}
}
func Benchmark_mergeXLV2Versions(b *testing.B) {
data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
if err != nil {
b.Fatal(err)
}
dec, _ := zstd.NewReader(nil)
data, err = dec.DecodeAll(data, nil)
if err != nil {
b.Fatal(err)
}
var xl xlMetaV2
if err = xl.LoadOrConvert(data); err != nil {
b.Fatal(err)
}
vers := make([][]xlMetaV2ShallowVersion, 16)
for i := range vers {
vers[i] = xl.versions
}
b.Run("requested-none", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(855) // number of versions...
for i := 0; i < b.N; i++ {
mergeXLV2Versions(8, false, 0, vers...)
}
})
b.Run("requested-v1", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(855) // number of versions...
for i := 0; i < b.N; i++ {
mergeXLV2Versions(8, false, 1, vers...)
}
})
b.Run("requested-v2", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(855) // number of versions...
for i := 0; i < b.N; i++ {
mergeXLV2Versions(8, false, 1, vers...)
}
})
}
func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
if err != nil {
@@ -424,6 +473,7 @@ func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
}
}
})
b.Run("indexed", func(b *testing.B) {
var xl xlMetaV2
err = xl.Load(data)
@@ -550,7 +600,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
for i := range vers {
t.Run(fmt.Sprintf("non-strict-q%d", i), func(t *testing.T) {
merged := mergeXLV2Versions(i, false, vers...)
merged := mergeXLV2Versions(i, false, 0, vers...)
if len(merged) == 0 {
t.Error("Did not get any results")
return
@@ -562,7 +612,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
}
})
t.Run(fmt.Sprintf("strict-q%d", i), func(t *testing.T) {
merged := mergeXLV2Versions(i, true, vers...)
merged := mergeXLV2Versions(i, true, 0, vers...)
if len(merged) == 0 {
t.Error("Did not get any results")
return
@@ -584,7 +634,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
}
vMod = append(vMod, newVers)
}
merged := mergeXLV2Versions(i, false, vMod...)
merged := mergeXLV2Versions(i, false, 0, vMod...)
if len(merged) == 0 {
t.Error("Did not get any results")
return
@@ -606,7 +656,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
}
vMod = append(vMod, newVers)
}
merged := mergeXLV2Versions(i, false, vMod...)
merged := mergeXLV2Versions(i, false, 0, vMod...)
if len(merged) == 0 && i < 2 {
t.Error("Did not get any results")
return
@@ -632,7 +682,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
}
vMod = append(vMod, newVers)
}
merged := mergeXLV2Versions(i, false, vMod...)
merged := mergeXLV2Versions(i, false, 0, vMod...)
if len(merged) == 0 {
t.Error("Did not get any results")
return
@@ -654,7 +704,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
}
vMod = append(vMod, newVers)
}
merged := mergeXLV2Versions(i, false, vMod...)
merged := mergeXLV2Versions(i, false, 0, vMod...)
if len(merged) == 0 && i < 2 {
t.Error("Did not get any results")
return
@@ -680,7 +730,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
}
vMod = append(vMod, newVers)
}
merged := mergeXLV2Versions(i, true, vMod...)
merged := mergeXLV2Versions(i, true, 0, vMod...)
if len(merged) == 0 && i < 2 {
t.Error("Did not get any results")
return
@@ -706,7 +756,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
}
vMod = append(vMod, newVers)
}
merged := mergeXLV2Versions(i, true, vMod...)
merged := mergeXLV2Versions(i, true, 0, vMod...)
if len(merged) == 0 && i < 2 {
t.Error("Did not get any results")
return
@@ -732,7 +782,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
}
vMod = append(vMod, newVers)
}
merged := mergeXLV2Versions(i, true, vMod...)
merged := mergeXLV2Versions(i, true, 0, vMod...)
if len(merged) == 0 && i < 2 {
t.Error("Did not get any results")
return
@@ -758,7 +808,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
}
vMod = append(vMod, newVers)
}
merged := mergeXLV2Versions(i, true, vMod...)
merged := mergeXLV2Versions(i, true, 0, vMod...)
if len(merged) == 0 && i < 2 {
t.Error("Did not get any results")
return