MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-08 21:24:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Stop async listing earlier (#13160)

Browse Source 
Stop async listing if we have not heard back from the client for 3 minutes.

This will stop spending resources on async listings when they are unlikely to get used. 
If the client returns a new listing will be started on the second request.

Stop saving cache metadata to disk. It is cleared on restarts anyway. Removes all 
load/save functionality
This commit is contained in:
Klaus Post
2021-09-08 11:06:45 -07:00
committed by GitHub
parent 951b1e6a7a
commit 3c2efd9cf3
8 changed files with 48 additions and 467 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/metacache.go
View File

@@ -39,6 +39,9 @@ const (
// Time in which the initiator of a scan must have reported back.
metacacheMaxRunningAge = time.Minute
// Max time between client calls before dropping an async cache listing.
metacacheMaxClientWait = 3 * time.Minute
// metacacheBlockSize is the number of file/directory entries to have in each block.
metacacheBlockSize = 5000
@@ -82,8 +85,9 @@ func (m *metacache) worthKeeping() bool {
case !cache.finished() && time.Since(cache.lastUpdate) > metacacheMaxRunningAge:
// Not finished and update for metacacheMaxRunningAge, discard it.
return false
case cache.finished() && time.Since(cache.lastHandout) > 30*time.Minute:
// Keep only for 30 minutes.
case cache.finished() && time.Since(cache.lastHandout) > 5*metacacheMaxClientWait:
// Keep for 15 minutes after we last saw the client.
// Since the cache is finished keeping it a bit longer doesn't hurt us.
return false
case cache.status == scanStateError || cache.status == scanStateNone:
// Remove failed listings after 5 minutes.
@@ -113,6 +117,9 @@ func baseDirFromPrefix(prefix string) string {
func (m *metacache) update(update metacache) {
m.lastUpdate = UTCNow()
if m.lastHandout.After(m.lastHandout) {
m.lastHandout = UTCNow()
}
if m.status == scanStateStarted && update.status == scanStateSuccess {
m.ended = UTCNow()
}
@@ -121,7 +128,8 @@ func (m *metacache) update(update metacache) {
m.status = update.status
}
if m.status == scanStateStarted && time.Since(m.lastHandout) > 15*time.Minute {
if m.status == scanStateStarted && time.Since(m.lastHandout) > metacacheMaxClientWait {
// Drop if client hasn't been seen for 3 minutes.
m.status = scanStateError
m.error = "client not seen"
}