request only 1 best version for latest version resolution (#14625)

ListObjects, ListObjectsV2 calls are being heavily taxed when
there are many versions on objects left over from a previous
release or ILM was never setup to clean them up. Instead
of being absolutely correct at resolving the exact latest
version of an object, we simply rely on the top most 1
version and resolve the rest.

Once we have obtained the top most "1" version for
ListObject, ListObjectsV2 call we break out.
This commit is contained in:
Harshavardhana
2022-03-25 08:50:07 -07:00
committed by GitHub
parent 401958938d
commit f046f557fa
4 changed files with 98 additions and 16 deletions

View File

@@ -1691,7 +1691,7 @@ func (x xlMetaV2) ListVersions(volume, path string) ([]FileInfo, error) {
// Quorum must be the minimum number of matching metadata files.
// Quorum should be > 1 and <= len(versions).
// If strict is set to false, entries that match type
func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) {
func mergeXLV2Versions(quorum int, strict bool, requestedVersions int, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) {
if quorum <= 0 {
quorum = 1
}
@@ -1708,6 +1708,8 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
// Shallow copy input
versions = append(make([][]xlMetaV2ShallowVersion, 0, len(versions)), versions...)
var nVersions int // captures all non-free versions
// Our result
merged = make([]xlMetaV2ShallowVersion, 0, len(versions[0]))
tops := make([]xlMetaV2ShallowVersion, len(versions))
@@ -1744,6 +1746,12 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
latest = tops[0]
latestCount = len(tops)
merged = append(merged, latest)
// Calculate latest 'n' non-free versions.
if !latest.header.FreeVersion() {
nVersions++
}
} else {
// Find latest.
for i, ver := range tops {
@@ -1807,6 +1815,11 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
}
if latestCount >= quorum {
merged = append(merged, latest)
// Calculate latest 'n' non-free versions.
if !latest.header.FreeVersion() {
nVersions++
}
}
}
@@ -1840,7 +1853,13 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
break
}
}
if requestedVersions > 0 && requestedVersions == nVersions {
merged = append(merged, versions[0]...)
break
}
}
// Sanity check. Enable if duplicates show up.
if false {
found := make(map[[16]byte]struct{})