MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-21 10:16:03 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add cache eviction low and high watermarks (#8958)

Browse Source 
To allow better control the cache eviction process.

Introduce MINIO_CACHE_WATERMARK_LOW and 
MINIO_CACHE_WATERMARK_HIGH env. variables to specify 
when to stop/start cache eviction process. 

Deprecate MINIO_CACHE_EXPIRY environment variable. Cache 
gc sweeps at 30 minute intervals whenever high watermark is
reached to clear least recently accessed entries in the cache
until sufficient space is cleared to reach the low watermark.

Garbage collection uses an adaptive file scoring approach based
on last access time, with greater weights assigned to larger
objects and those with more hits to find the candidates for eviction.

Thanks to @klauspost for this file scoring algorithm

Co-authored-by: Klaus Post <klauspost@minio.io>
This commit is contained in:
poornas
2020-02-23 05:33:39 -08:00
committed by GitHub
parent 51a9d1bdb7
commit 224b4f13b8
18 changed files with 585 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
cmd/disk-cache-utils_test.go
View File

@@ -29,16 +29,16 @@ func TestGetCacheControlOpts(t *testing.T) {
testCases := []struct {
cacheControlHeaderVal string
expiryHeaderVal time.Time
expectedCacheControl cacheControl
expectedCacheControl *cacheControl
expectedErr bool
}{
{"", timeSentinel, cacheControl{}, false},
{"max-age=2592000, public", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
{"max-age=2592000, no-store", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
{"must-revalidate, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
{"s-maxAge=2500, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
{"s-maxAge=2500, max-age=600", expiry, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
{"s-maxAge=2500, max-age=600s", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
{"", timeSentinel, nil, false},
{"max-age=2592000, public", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
{"max-age=2592000, no-store", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
{"must-revalidate, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
{"s-maxAge=2500, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
{"s-maxAge=2500, max-age=600s", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
}
for _, testCase := range testCases {
@@ -49,7 +49,7 @@ func TestGetCacheControlOpts(t *testing.T) {
m["expires"] = testCase.expiryHeaderVal.String()
}
c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})
if testCase.expectedErr && (c != cacheControl{}) {
if testCase.expectedErr && (c != nil) {
t.Errorf("expected err, got <nil>")
}
if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
@@ -83,3 +83,90 @@ func TestIsMetadataSame(t *testing.T) {
}
}
}
func TestNewFileScorer(t *testing.T) {
fs, err := newFileScorer(1000, time.Now().Unix(), 10)
if err != nil {
t.Fatal(err)
}
if len(fs.fileNames()) != 0 {
t.Fatal("non zero files??")
}
now := time.Now()
fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
fs.addFile("older", now.Add(-time.Hour), 1000, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"older"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.reset()
fs.addFile("bigger", now.Add(-time.Minute), 2000, 10)
fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"bigger"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.reset()
fs.addFile("less", now.Add(-time.Minute), 1000, 5)
fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.reset()
fs.addFile("small", now.Add(-time.Minute), 200, 10)
fs.addFile("medium", now.Add(-time.Minute), 300, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"medium", "small"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.addFile("large", now.Add(-time.Minute), 700, 10)
fs.addFile("xsmol", now.Add(-time.Minute), 7, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"large", "medium"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.reset()
fs.addFile("less", now.Add(-time.Minute), 500, 5)
fs.addFile("recent", now.Add(-time.Minute), 500, 10)
if !fs.adjustSaveBytes(-500) {
t.Fatal("we should still need more bytes, got false")
}
// We should only need 500 bytes now.
if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
t.Fatal("unexpected file list", fs.queueString())
}
if fs.adjustSaveBytes(-500) {
t.Fatal("we shouldn't need any more bytes, got true")
}
fs, err = newFileScorer(1000, time.Now().Unix(), 10)
if err != nil {
t.Fatal(err)
}
fs.addFile("bigger", now.Add(-time.Minute), 50, 10)
// sorting should be consistent after adjusting savebytes.
fs.adjustSaveBytes(-800)
fs.addFile("smaller", now.Add(-time.Minute), 40, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"bigger", "smaller"}) {
t.Fatal("unexpected file list", fs.queueString())
}
}
func TestBytesToClear(t *testing.T) {
testCases := []struct {
total int64
free int64
quotaPct uint64
watermarkLow uint64
expected uint64
}{
{1000, 800, 40, 90, 0},
{1000, 200, 40, 90, 400},
{1000, 400, 40, 90, 240},
{1000, 600, 40, 90, 40},
{1000, 600, 40, 70, 120},
{1000, 1000, 90, 70, 0},
{1000, 0, 90, 70, 370},
}
for i, tc := range testCases {
toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow)
if tc.expected != toClear {
t.Errorf("test %d expected %v, got %v", i, tc.expected, toClear)
}
}
}