MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-04-03 19:30:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

Allow compaction at bucket top level. ()

Browse Source 
If more than 1M folders (objects or prefixes) are found at the top level in a bucket allow it to be compacted.

While very suboptimal structure we should limit memory usage at some point.
This commit is contained in:
Klaus Post 2022-07-11 07:59:03 -07:00 committed by GitHub
parent 913e977c8d
commit 37a6b2da67
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 12 additions and 9 deletions

9
cmd/data-scanner.go

@ -53,6 +53,7 @@ const (
dataScannerCompactLeastObject = 500 // Compact when there is less than this many objects in a branch. dataScannerCompactLeastObject = 500 // Compact when there is less than this many objects in a branch.
dataScannerCompactAtChildren = 10000 // Compact when there are this many children in a branch. dataScannerCompactAtChildren = 10000 // Compact when there are this many children in a branch.
dataScannerCompactAtFolders = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder. dataScannerCompactAtFolders = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder.
dataScannerForceCompactAtFolders = 1_000_000 // Compact when this many subfolders in a single folder (even top level).
dataScannerStartDelay = 1 * time.Minute // Time to wait on startup and between cycles. dataScannerStartDelay = 1 * time.Minute // Time to wait on startup and between cycles.
healDeleteDangling = true healDeleteDangling = true
@ -566,9 +567,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
} }
// If we have many subfolders, compact ourself. // If we have many subfolders, compact ourself.
if !into.Compacted && shouldCompact := f.newCache.Info.Name != folder.name &&
f.newCache.Info.Name != folder.name && len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders { len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
if !into.Compacted && shouldCompact {
into.Compacted = true into.Compacted = true
newFolders = append(newFolders, existingFolders...) newFolders = append(newFolders, existingFolders...)
existingFolders = nil existingFolders = nil