MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-24 11:37:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

use optimal memory while purging cache (#9426)

Browse Source 
re-implement the cache purging routine to
avoid using ioutil.ReadDir which can lead
to high allocations when there are cache
directories with lots of content, or
when cache is installed in memory constrainted
environments.

Instead rely on a callback function where we
are not using memory no-more than 8KiB per
cycle.

Precursor for this change refer #9425, original
issue pointed by Caleb Case <caleb@storj.io>
This commit is contained in:
Harshavardhana
2020-04-23 12:26:13 -07:00
committed by GitHub
parent ac5061df2c
commit 957ecb1b64
4 changed files with 188 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
cmd/disk-cache-backend.go
View File

@@ -21,10 +21,10 @@ import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
@@ -228,6 +228,10 @@ func (c *diskCache) toClear() uint64 {
return bytesToClear(int64(di.Total), int64(di.Free), uint64(c.quotaPct), uint64(c.lowWatermark))
}
var (
errDoneForNow = errors.New("done for now")
)
// Purge cache entries that were not accessed.
func (c *diskCache) purge(ctx context.Context) {
if c.diskUsageLow() {
@@ -263,25 +267,24 @@ func (c *diskCache) purge(ctx context.Context) {
}
return fm
}
objDirs, err := ioutil.ReadDir(c.dir)
if err != nil {
log.Fatal(err)
}
for _, obj := range objDirs {
if obj.Name() == minioMetaBucket {
continue
filterFn := func(name string, typ os.FileMode) error {
if name == minioMetaBucket {
// Proceed to next file.
return nil
}
cacheDir := pathJoin(c.dir, obj.Name())
cacheDir := pathJoin(c.dir, name)
meta, _, numHits, err := c.statCachedMeta(ctx, cacheDir)
if err != nil {
// delete any partially filled cache entry left behind.
removeAll(cacheDir)
continue
// Proceed to next file.
return nil
}
// stat all cached file ranges and cacheDataFile.
cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, name))
objInfo := meta.ToObjectInfo("", "")
cc := cacheControlOpts(objInfo)
for fname, fi := range cachedFiles {
@@ -291,9 +294,11 @@ func (c *diskCache) purge(ctx context.Context) {
logger.LogIf(ctx, err)
}
scorer.adjustSaveBytes(-fi.Size())
// break early if sufficient disk space reclaimed.
if c.diskUsageLow() {
return
// if we found disk usage is already low, we return nil filtering is complete.
return errDoneForNow
}
}
continue
@@ -305,12 +310,24 @@ func (c *diskCache) purge(ctx context.Context) {
if err != nil || (fi.ModTime().Before(expiry) && len(cachedFiles) == 0) {
removeAll(cacheDir)
scorer.adjustSaveBytes(-fi.Size())
continue
// Proceed to next file.
return nil
}
// if we found disk usage is already low, we return nil filtering is complete.
if c.diskUsageLow() {
return
return errDoneForNow
}
// Proceed to next file.
return nil
}
if err := readDirFilterFn(c.dir, filterFn); err != nil {
logger.LogIf(ctx, err)
return
}
for _, path := range scorer.fileNames() {
removeAll(path)
slashIdx := strings.LastIndex(path, SlashSeparator)