mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
request only 1 best version for latest version resolution (#14625)
ListObjects, ListObjectsV2 calls are being heavily taxed when there are many versions on objects left over from a previous release or ILM was never setup to clean them up. Instead of being absolutely correct at resolving the exact latest version of an object, we simply rely on the top most 1 version and resolve the rest. Once we have obtained the top most "1" version for ListObject, ListObjectsV2 call we break out.
This commit is contained in:
Harshavardhana
GitHub
parent
401958938d
commit
f046f557fa
@ -281,10 +281,16 @@ func (m metaCacheEntries) shallowClone() metaCacheEntries {
|
||||
}
|
||||
|
||||
type metadataResolutionParams struct {
|
||||
dirQuorum int // Number if disks needed for a directory to 'exist'.
|
||||
objQuorum int // Number of disks needed for an object to 'exist'.
|
||||
bucket string // Name of the bucket. Used for generating cached fileinfo.
|
||||
strict bool // Versions must match exactly, including all metadata.
|
||||
dirQuorum int // Number if disks needed for a directory to 'exist'.
|
||||
objQuorum int // Number of disks needed for an object to 'exist'.
|
||||
|
||||
// An optimization request only an 'n' amount of versions from xl.meta
|
||||
// to avoid resolving all versions to figure out the latest 'version'
|
||||
// for ListObjects, ListObjectsV2
|
||||
requestedVersions int
|
||||
|
||||
bucket string // Name of the bucket. Used for generating cached fileinfo.
|
||||
strict bool // Versions must match exactly, including all metadata.
|
||||
|
||||
// Reusable slice for resolution
|
||||
candidates [][]xlMetaV2ShallowVersion
|
||||
@ -372,7 +378,7 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
|
||||
reusable: true,
|
||||
cached: &xlMetaV2{metaV: selected.cached.metaV},
|
||||
}
|
||||
selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.candidates...)
|
||||
selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.requestedVersions, r.candidates...)
|
||||
if len(selected.cached.versions) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
@ -571,6 +571,13 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
|
||||
bucket: o.Bucket,
|
||||
}
|
||||
|
||||
// Maximum versions requested for "latest" object
|
||||
// resolution on versioned buckets, this is to be only
|
||||
// used when o.Versioned is false
|
||||
if !o.Versioned {
|
||||
resolver.requestedVersions = 1
|
||||
}
|
||||
|
||||
ctxDone := ctx.Done()
|
||||
return listPathRaw(ctx, listPathRawOptions{
|
||||
disks: disks,
|
||||
|
||||
@ -1691,7 +1691,7 @@ func (x xlMetaV2) ListVersions(volume, path string) ([]FileInfo, error) {
|
||||
// Quorum must be the minimum number of matching metadata files.
|
||||
// Quorum should be > 1 and <= len(versions).
|
||||
// If strict is set to false, entries that match type
|
||||
func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) {
|
||||
func mergeXLV2Versions(quorum int, strict bool, requestedVersions int, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) {
|
||||
if quorum <= 0 {
|
||||
quorum = 1
|
||||
}
|
||||
@ -1708,6 +1708,8 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
|
||||
// Shallow copy input
|
||||
versions = append(make([][]xlMetaV2ShallowVersion, 0, len(versions)), versions...)
|
||||
|
||||
var nVersions int // captures all non-free versions
|
||||
|
||||
// Our result
|
||||
merged = make([]xlMetaV2ShallowVersion, 0, len(versions[0]))
|
||||
tops := make([]xlMetaV2ShallowVersion, len(versions))
|
||||
@ -1744,6 +1746,12 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
|
||||
latest = tops[0]
|
||||
latestCount = len(tops)
|
||||
merged = append(merged, latest)
|
||||
|
||||
// Calculate latest 'n' non-free versions.
|
||||
if !latest.header.FreeVersion() {
|
||||
nVersions++
|
||||
}
|
||||
|
||||
} else {
|
||||
// Find latest.
|
||||
for i, ver := range tops {
|
||||
@ -1807,6 +1815,11 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
|
||||
}
|
||||
if latestCount >= quorum {
|
||||
merged = append(merged, latest)
|
||||
|
||||
// Calculate latest 'n' non-free versions.
|
||||
if !latest.header.FreeVersion() {
|
||||
nVersions++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1840,7 +1853,13 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if requestedVersions > 0 && requestedVersions == nVersions {
|
||||
merged = append(merged, versions[0]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Sanity check. Enable if duplicates show up.
|
||||
if false {
|
||||
found := make(map[[16]byte]struct{})
|
||||
|
||||
@ -401,6 +401,55 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func Benchmark_mergeXLV2Versions(b *testing.B) {
|
||||
data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
dec, _ := zstd.NewReader(nil)
|
||||
data, err = dec.DecodeAll(data, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
var xl xlMetaV2
|
||||
if err = xl.LoadOrConvert(data); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
vers := make([][]xlMetaV2ShallowVersion, 16)
|
||||
for i := range vers {
|
||||
vers[i] = xl.versions
|
||||
}
|
||||
|
||||
b.Run("requested-none", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
b.SetBytes(855) // number of versions...
|
||||
for i := 0; i < b.N; i++ {
|
||||
mergeXLV2Versions(8, false, 0, vers...)
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("requested-v1", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
b.SetBytes(855) // number of versions...
|
||||
for i := 0; i < b.N; i++ {
|
||||
mergeXLV2Versions(8, false, 1, vers...)
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("requested-v2", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
b.SetBytes(855) // number of versions...
|
||||
for i := 0; i < b.N; i++ {
|
||||
mergeXLV2Versions(8, false, 1, vers...)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
|
||||
data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
|
||||
if err != nil {
|
||||
@ -424,6 +473,7 @@ func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
b.Run("indexed", func(b *testing.B) {
|
||||
var xl xlMetaV2
|
||||
err = xl.Load(data)
|
||||
@ -550,7 +600,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
|
||||
for i := range vers {
|
||||
t.Run(fmt.Sprintf("non-strict-q%d", i), func(t *testing.T) {
|
||||
merged := mergeXLV2Versions(i, false, vers...)
|
||||
merged := mergeXLV2Versions(i, false, 0, vers...)
|
||||
if len(merged) == 0 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
@ -562,7 +612,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
}
|
||||
})
|
||||
t.Run(fmt.Sprintf("strict-q%d", i), func(t *testing.T) {
|
||||
merged := mergeXLV2Versions(i, true, vers...)
|
||||
merged := mergeXLV2Versions(i, true, 0, vers...)
|
||||
if len(merged) == 0 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
@ -584,7 +634,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
}
|
||||
vMod = append(vMod, newVers)
|
||||
}
|
||||
merged := mergeXLV2Versions(i, false, vMod...)
|
||||
merged := mergeXLV2Versions(i, false, 0, vMod...)
|
||||
if len(merged) == 0 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
@ -606,7 +656,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
}
|
||||
vMod = append(vMod, newVers)
|
||||
}
|
||||
merged := mergeXLV2Versions(i, false, vMod...)
|
||||
merged := mergeXLV2Versions(i, false, 0, vMod...)
|
||||
if len(merged) == 0 && i < 2 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
@ -632,7 +682,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
}
|
||||
vMod = append(vMod, newVers)
|
||||
}
|
||||
merged := mergeXLV2Versions(i, false, vMod...)
|
||||
merged := mergeXLV2Versions(i, false, 0, vMod...)
|
||||
if len(merged) == 0 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
@ -654,7 +704,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
}
|
||||
vMod = append(vMod, newVers)
|
||||
}
|
||||
merged := mergeXLV2Versions(i, false, vMod...)
|
||||
merged := mergeXLV2Versions(i, false, 0, vMod...)
|
||||
if len(merged) == 0 && i < 2 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
@ -680,7 +730,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
}
|
||||
vMod = append(vMod, newVers)
|
||||
}
|
||||
merged := mergeXLV2Versions(i, true, vMod...)
|
||||
merged := mergeXLV2Versions(i, true, 0, vMod...)
|
||||
if len(merged) == 0 && i < 2 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
@ -706,7 +756,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
}
|
||||
vMod = append(vMod, newVers)
|
||||
}
|
||||
merged := mergeXLV2Versions(i, true, vMod...)
|
||||
merged := mergeXLV2Versions(i, true, 0, vMod...)
|
||||
if len(merged) == 0 && i < 2 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
@ -732,7 +782,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
}
|
||||
vMod = append(vMod, newVers)
|
||||
}
|
||||
merged := mergeXLV2Versions(i, true, vMod...)
|
||||
merged := mergeXLV2Versions(i, true, 0, vMod...)
|
||||
if len(merged) == 0 && i < 2 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
@ -758,7 +808,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
|
||||
}
|
||||
vMod = append(vMod, newVers)
|
||||
}
|
||||
merged := mergeXLV2Versions(i, true, vMod...)
|
||||
merged := mergeXLV2Versions(i, true, 0, vMod...)
|
||||
if len(merged) == 0 && i < 2 {
|
||||
t.Error("Did not get any results")
|
||||
return
|
||||
|
||||
Loading…
Reference in New Issue
Block a user