MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add cache eviction low and high watermarks (#8958) 

To allow better control the cache eviction process.

Introduce MINIO_CACHE_WATERMARK_LOW and 
MINIO_CACHE_WATERMARK_HIGH env. variables to specify 
when to stop/start cache eviction process. 

Deprecate MINIO_CACHE_EXPIRY environment variable. Cache 
gc sweeps at 30 minute intervals whenever high watermark is
reached to clear least recently accessed entries in the cache
until sufficient space is cleared to reach the low watermark.

Garbage collection uses an adaptive file scoring approach based
on last access time, with greater weights assigned to larger
objects and those with more hits to find the candidates for eviction.

Thanks to @klauspost for this file scoring algorithm

Co-authored-by: Klaus Post <klauspost@minio.io>
This commit is contained in:
poornas 2020-02-23 05:33:39 -08:00 committed by GitHub
parent 51a9d1bdb7
commit 224b4f13b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 585 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/config/cache/config.go vendored
View File

@ -35,6 +35,8 @@ type Config struct {
Quota int `json:"quota"`
Exclude []string `json:"exclude"`
After int `json:"after"`
WatermarkLow int `json:"watermark_low"`
WatermarkHigh int `json:"watermark_high"`
}
// UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@ -64,6 +66,15 @@ func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
if _cfg.After < 0 {
return errors.New("cache after value should not be less than 0")
}
if _cfg.WatermarkLow < 0 || _cfg.WatermarkLow > 100 {
return errors.New("config low watermark value should be between 0 and 100")
}
if _cfg.WatermarkHigh < 0 || _cfg.WatermarkHigh > 100 {
return errors.New("config high watermark value should be between 0 and 100")
}
if _cfg.WatermarkLow > 0 && (_cfg.WatermarkLow >= _cfg.WatermarkHigh) {
return errors.New("config low watermark value should be less than high watermark")
}
return nil
}

12
cmd/config/cache/help.go vendored
View File

@ -56,5 +56,17 @@ var (
Optional: true,
Type: "number",
},
config.HelpKV{
Key: WatermarkLow,
Description: `% of cache use at which to stop cache eviction`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: WatermarkHigh,
Description: `% of cache use at which to start cache eviction`,
Optional: true,
Type: "number",
},
}
)

43
cmd/config/cache/lookup.go vendored
View File

@ -32,6 +32,8 @@ const (
MaxUse = "maxuse"
Quota = "quota"
After = "after"
WatermarkLow = "watermark_low"
WatermarkHigh = "watermark_high"
EnvCacheDrives = "MINIO_CACHE_DRIVES"
EnvCacheExclude = "MINIO_CACHE_EXCLUDE"
@ -39,12 +41,16 @@ const (
EnvCacheMaxUse = "MINIO_CACHE_MAXUSE"
EnvCacheQuota = "MINIO_CACHE_QUOTA"
EnvCacheAfter = "MINIO_CACHE_AFTER"
EnvCacheWatermarkLow = "MINIO_CACHE_WATERMARK_LOW"
EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
DefaultExpiry = "90"
DefaultQuota = "80"
DefaultAfter = "0"
DefaultWaterMarkLow = "70"
DefaultWaterMarkHigh = "80"
)
// DefaultKVS - default KV settings for caching.
@ -70,6 +76,14 @@ var (
Key: After,
Value: DefaultAfter,
},
config.KV{
Key: WatermarkLow,
Value: DefaultWaterMarkLow,
},
config.KV{
Key: WatermarkHigh,
Value: DefaultWaterMarkHigh,
},
}
)
@ -87,7 +101,6 @@ func Enabled(kvs config.KVS) bool {
// variables and merge them with provided CacheConfiguration.
func LookupConfig(kvs config.KVS) (Config, error) {
cfg := Config{}
if err := config.CheckValidKeys(config.CacheSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
@ -154,5 +167,33 @@ func LookupConfig(kvs config.KVS) (Config, error) {
}
}
if lowWMStr := env.Get(EnvCacheWatermarkLow, kvs.Get(WatermarkLow)); lowWMStr != "" {
cfg.WatermarkLow, err = strconv.Atoi(lowWMStr)
if err != nil {
return cfg, config.ErrInvalidCacheWatermarkLow(err)
}
// WatermarkLow should be a valid percentage.
if cfg.WatermarkLow < 0 || cfg.WatermarkLow > 100 {
err := errors.New("config min watermark value should be between 0 and 100")
return cfg, config.ErrInvalidCacheWatermarkLow(err)
}
}
if highWMStr := env.Get(EnvCacheWatermarkHigh, kvs.Get(WatermarkHigh)); highWMStr != "" {
cfg.WatermarkHigh, err = strconv.Atoi(highWMStr)
if err != nil {
return cfg, config.ErrInvalidCacheWatermarkHigh(err)
}
// MaxWatermark should be a valid percentage.
if cfg.WatermarkHigh < 0 || cfg.WatermarkHigh > 100 {
err := errors.New("config high watermark value should be between 0 and 100")
return cfg, config.ErrInvalidCacheWatermarkHigh(err)
}
}
if cfg.WatermarkLow > cfg.WatermarkHigh {
err := errors.New("config high watermark value should be greater than low watermark value")
return cfg, config.ErrInvalidCacheWatermarkHigh(err)
}
return cfg, nil
}

12
cmd/config/errors.go
View File

@ -72,6 +72,18 @@ var (
"MINIO_CACHE_AFTER: Valid cache after value must be 0 or greater",
)
ErrInvalidCacheWatermarkLow = newErrFn(
"Invalid cache low watermark value",
"Please check the passed value",
"MINIO_CACHE_WATERMARK_LOW: Valid cache low watermark value must be between 0-100",
)
ErrInvalidCacheWatermarkHigh = newErrFn(
"Invalid cache high watermark value",
"Please check the passed value",
"MINIO_CACHE_WATERMARK_HIGH: Valid cache high watermark value must be between 0-100",
)
ErrInvalidCacheEncryptionKey = newErrFn(
"Invalid cache encryption master key value",
"Please check the passed value",

156
cmd/disk-cache-backend.go
View File

@ -27,6 +27,7 @@ import (
"log"
"net/http"
"os"
"strings"
"sync"
"time"
@ -37,6 +38,7 @@ import (
"github.com/minio/minio/pkg/disk"
"github.com/minio/sio"
"github.com/ncw/directio"
"go.uber.org/atomic"
)
const (
@ -44,7 +46,7 @@ const (
cacheMetaJSONFile = "cache.json"
cacheDataFile = "part.1"
cacheMetaVersion = "1.0.0"
cacheExpiryDays = time.Duration(90 * time.Hour * 24) // defaults to 90 days
// SSECacheEncrypted is the metadata key indicating that the object
// is a cache entry encrypted with cache KMS master key in globalCacheKMS.
SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache"
@ -126,15 +128,15 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
type diskCache struct {
dir string // caching directory
quotaPct int // max usage in %
expiry int // cache expiry in days
// mark false if drive is offline
online bool
// mutex to protect updates to online variable
onlineMutex *sync.RWMutex
// purge() listens on this channel to start the cache-purge process
purgeChan chan struct{}
pool sync.Pool
after int // minimum accesses before an object is cached.
lowWatermark int
highWatermark int
gcCounter atomic.Uint64
// nsMutex namespace lock
nsMutex *nsLockMap
// Object functions pointing to the corresponding functions of backend implementation.
@ -142,16 +144,16 @@ type diskCache struct {
}
// Inits the disk cache dir if it is not initialized already.
func newDiskCache(dir string, expiry int, quotaPct, after int) (*diskCache, error) {
func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
if err := os.MkdirAll(dir, 0777); err != nil {
return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
}
cache := diskCache{
dir: dir,
expiry: expiry,
quotaPct: quotaPct,
after: after,
purgeChan: make(chan struct{}),
lowWatermark: lowWatermark,
highWatermark: highWatermark,
online: true,
onlineMutex: &sync.RWMutex{},
pool: sync.Pool{
@ -168,12 +170,12 @@ func newDiskCache(dir string, expiry int, quotaPct, after int) (*diskCache, erro
return &cache, nil
}
// Returns if the disk usage is low.
// Disk usage is low if usage is < 80% of cacheMaxDiskUsagePct
// Ex. for a 100GB disk, if maxUsage is configured as 70% then cacheMaxDiskUsagePct is 70G
// hence disk usage is low if the disk usage is less than 56G (because 80% of 70G is 56G)
// diskUsageLow() returns true if disk usage falls below the low watermark w.r.t configured cache quota.
// Ex. for a 100GB disk, if quota is configured as 70% and watermark_low = 80% and
// watermark_high = 90% then garbage collection starts when 63% of disk is used and
// stops when disk usage drops to 56%
func (c *diskCache) diskUsageLow() bool {
minUsage := c.quotaPct * 80 / 100
gcStopPct := c.quotaPct * c.lowWatermark / 100
di, err := disk.GetInfo(c.dir)
if err != nil {
reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
@ -182,21 +184,22 @@ func (c *diskCache) diskUsageLow() bool {
return false
}
usedPercent := (di.Total - di.Free) * 100 / di.Total
return int(usedPercent) < minUsage
return int(usedPercent) < gcStopPct
}
// Return if the disk usage is high.
// Disk usage is high if disk used is > cacheMaxDiskUsagePct
// Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
// gc starts if high water mark reached.
func (c *diskCache) diskUsageHigh() bool {
gcTriggerPct := c.quotaPct * c.highWatermark / 100
di, err := disk.GetInfo(c.dir)
if err != nil {
reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
ctx := logger.SetReqInfo(context.Background(), reqInfo)
logger.LogIf(ctx, err)
return true
return false
}
usedPercent := (di.Total - di.Free) * 100 / di.Total
return int(usedPercent) > c.quotaPct
return int(usedPercent) >= gcTriggerPct
}
// Returns if size space can be allocated without exceeding
@ -213,12 +216,42 @@ func (c *diskCache) diskAvailable(size int64) bool {
return int(usedPercent) < c.quotaPct
}
// toClear returns how many bytes should be cleared to reach the low watermark quota.
// returns 0 if below quota.
func (c *diskCache) toClear() uint64 {
di, err := disk.GetInfo(c.dir)
if err != nil {
reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
ctx := logger.SetReqInfo(context.Background(), reqInfo)
logger.LogIf(ctx, err)
return 0
}
return bytesToClear(int64(di.Total), int64(di.Free), uint64(c.quotaPct), uint64(c.lowWatermark))
}
// Purge cache entries that were not accessed.
func (c *diskCache) purge() {
func (c *diskCache) purge(ctx context.Context, doneCh <-chan struct{}) {
if c.diskUsageLow() {
return
}
toFree := c.toClear()
if toFree == 0 {
return
}
// expiry for cleaning up old cache.json files that
// need to be cleaned up.
expiry := UTCNow().Add(-cacheExpiryDays)
// defaulting max hits count to 100
scorer, err := newFileScorer(int64(toFree), time.Now().Unix(), 100)
if err != nil {
logger.LogIf(ctx, err)
return
}
// this function returns FileInfo for cached range files and cache data file.
fiStatFn := func(ranges map[string]string, dataFile, pathPrefix string) map[string]os.FileInfo {
fm := make(map[string]os.FileInfo)
fname := pathJoin(pathPrefix, cacheDataFile)
fname := pathJoin(pathPrefix, dataFile)
if fi, err := os.Stat(fname); err == nil {
fm[fname] = fi
}
@ -231,18 +264,6 @@ func (c *diskCache) purge() {
}
return fm
}
ctx := context.Background()
for {
olderThan := c.expiry * 24
for !c.diskUsageLow() {
// delete unaccessed objects older than expiry duration
expiry := UTCNow().Add(time.Hour * time.Duration(-1*olderThan))
olderThan /= 2
if olderThan < 1 {
break
}
deletedCount := 0
objDirs, err := ioutil.ReadDir(c.dir)
if err != nil {
log.Fatal(err)
@ -252,42 +273,64 @@ func (c *diskCache) purge() {
if obj.Name() == minioMetaBucket {
continue
}
meta, _, _, err := c.statCachedMeta(context.Background(), pathJoin(c.dir, obj.Name()))
cacheDir := pathJoin(c.dir, obj.Name())
meta, _, numHits, err := c.statCachedMeta(ctx, cacheDir)
if err != nil {
// delete any partially filled cache entry left behind.
removeAll(pathJoin(c.dir, obj.Name()))
removeAll(cacheDir)
continue
}
// stat all cached file ranges and cacheDataFile.
fis := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
objInfo := meta.ToObjectInfo("", "")
cc := cacheControlOpts(objInfo)
for fname, fi := range fis {
if atime.Get(fi).Before(expiry) ||
cc.isStale(objInfo.ModTime) {
for fname, fi := range cachedFiles {
if cc != nil {
if cc.isStale(objInfo.ModTime) {
if err = removeAll(fname); err != nil {
logger.LogIf(ctx, err)
}
deletedCount++
scorer.adjustSaveBytes(-fi.Size())
// break early if sufficient disk space reclaimed.
if !c.diskUsageLow() {
break
if c.diskUsageLow() {
return
}
}
continue
}
scorer.addFile(fname, atime.Get(fi), fi.Size(), numHits)
}
// clean up stale cache.json files for objects that never got cached but access count was maintained in cache.json
fi, err := os.Stat(pathJoin(cacheDir, cacheMetaJSONFile))
if err != nil || (fi.ModTime().Before(expiry) && len(cachedFiles) == 0) {
removeAll(cacheDir)
scorer.adjustSaveBytes(-fi.Size())
continue
}
if c.diskUsageLow() {
return
}
}
for _, path := range scorer.fileNames() {
removeAll(path)
slashIdx := strings.LastIndex(path, SlashSeparator)
pathPrefix := path[0:slashIdx]
fname := path[slashIdx+1:]
if fname == cacheDataFile {
removeAll(pathPrefix)
}
}
}
func (c *diskCache) incGCCounter() {
c.gcCounter.Add(uint64(1))
}
if deletedCount == 0 {
break
}
}
for {
<-c.purgeChan
if c.diskUsageHigh() {
break
}
}
func (c *diskCache) resetGCCounter() {
c.gcCounter.Store(uint64(0))
}
func (c *diskCache) gcCount() uint64 {
return c.gcCounter.Load()
}
// sets cache drive status
@ -378,6 +421,9 @@ func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HT
if !ok {
return oi, rngInfo, numHits, ObjectNotFound{Bucket: bucket, Object: object}
}
if _, err = os.Stat(pathJoin(cacheObjPath, rngFile)); err != nil {
return oi, rngInfo, numHits, ObjectNotFound{Bucket: bucket, Object: object}
}
rngInfo = RangeInfo{Range: rng, File: rngFile, Size: int64(actualRngSize)}
err = decryptCacheObjectETag(&oi)
@ -568,10 +614,8 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
// Caches the object to disk
func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
if c.diskUsageHigh() {
select {
case c.purgeChan <- struct{}{}:
default:
}
c.incGCCounter()
io.Copy(ioutil.Discard, data)
return errDiskFull
}
cachePath := getCacheSHADir(c.dir, bucket, object)
@ -622,11 +666,11 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
if IsErr(err, baseErrs...) {
c.setOnline(false)
}
if err != nil {
removeAll(cachePath)
return err
}
if actualSize != uint64(n) {
removeAll(cachePath)
return IncompleteBody{}

180
cmd/disk-cache-utils.go
View File

@ -17,8 +17,12 @@
package cmd
import (
"container/list"
"encoding/hex"
"errors"
"fmt"
"io"
"math"
"os"
"path"
"strconv"
@ -57,13 +61,8 @@ type cacheControl struct {
noCache bool
}
func (c cacheControl) isEmpty() bool {
return c == cacheControl{}
}
func (c cacheControl) isStale(modTime time.Time) bool {
if c.isEmpty() {
func (c *cacheControl) isStale(modTime time.Time) bool {
if c == nil {
return false
}
// response will never be stale if only-if-cached is set
@ -100,7 +99,8 @@ func (c cacheControl) isStale(modTime time.Time) bool {
}
// returns struct with cache-control settings from user metadata.
func cacheControlOpts(o ObjectInfo) (c cacheControl) {
func cacheControlOpts(o ObjectInfo) *cacheControl {
c := cacheControl{}
m := o.UserDefined
if o.Expires != timeSentinel {
c.expiry = o.Expires
@ -114,7 +114,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
}
if headerVal == "" {
return
return nil
}
headerVal = strings.ToLower(headerVal)
headerVal = strings.TrimSpace(headerVal)
@ -146,7 +146,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
p[0] == "max-stale" {
i, err := strconv.Atoi(p[1])
if err != nil {
return cacheControl{}
return nil
}
if p[0] == "max-age" {
c.maxAge = i
@ -162,7 +162,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
}
}
}
return c
return &c
}
// backendDownError returns true if err is due to backend failure or faulty disk if in server mode
@ -283,3 +283,161 @@ func isMetadataSame(m1, m2 map[string]string) bool {
}
return true
}
type fileScorer struct {
saveBytes int64
now int64
maxHits int
// 1/size for consistent score.
sizeMult float64
// queue is a linked list of files we want to delete.
// The list is kept sorted according to score, highest at top, lowest at bottom.
queue list.List
queuedBytes int64
}
type queuedFile struct {
name string
size int64
score float64
}
// newFileScorer allows to collect files to save a specific number of bytes.
// Each file is assigned a score based on its age, size and number of hits.
// A list of files is maintained
func newFileScorer(saveBytes int64, now int64, maxHits int) (*fileScorer, error) {
if saveBytes <= 0 {
return nil, errors.New("newFileScorer: saveBytes <= 0")
}
if now < 0 {
return nil, errors.New("newFileScorer: now < 0")
}
if maxHits <= 0 {
return nil, errors.New("newFileScorer: maxHits <= 0")
}
f := fileScorer{saveBytes: saveBytes, maxHits: maxHits, now: now, sizeMult: 1 / float64(saveBytes)}
f.queue.Init()
return &f, nil
}
func (f *fileScorer) addFile(name string, lastAccess time.Time, size int64, hits int) {
// Calculate how much we want to delete this object.
file := queuedFile{
name: name,
size: size,
}
score := float64(f.now - lastAccess.Unix())
// Size as fraction of how much we want to save, 0->1.
szWeight := math.Max(0, (math.Min(1, float64(size)*f.sizeMult)))
// 0 at f.maxHits, 1 at 0.
hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
// If we still haven't saved enough, just add the file
if f.queuedBytes < f.saveBytes {
f.insertFile(file)
f.trimQueue()
return
}
// If we score less than the worst, don't insert.
worstE := f.queue.Back()
if worstE != nil && file.score < worstE.Value.(queuedFile).score {
return
}
f.insertFile(file)
f.trimQueue()
}
// adjustSaveBytes allows to adjust the number of bytes to save.
// This can be used to adjust the count on the fly.
// Returns true if there still is a need to delete files (saveBytes >0),
// false if no more bytes needs to be saved.
func (f *fileScorer) adjustSaveBytes(n int64) bool {
f.saveBytes += n
if f.saveBytes <= 0 {
f.queue.Init()
f.saveBytes = 0
return false
}
if n < 0 {
f.trimQueue()
}
return true
}
// insertFile will insert a file into the list, sorted by its score.
func (f *fileScorer) insertFile(file queuedFile) {
e := f.queue.Front()
for e != nil {
v := e.Value.(queuedFile)
if v.score < file.score {
break
}
e = e.Next()
}
f.queuedBytes += file.size
// We reached the end.
if e == nil {
f.queue.PushBack(file)
return
}
f.queue.InsertBefore(file, e)
}
// trimQueue will trim the back of queue and still keep below wantSave.
func (f *fileScorer) trimQueue() {
for {
e := f.queue.Back()
if e == nil {
return
}
v := e.Value.(queuedFile)
if f.queuedBytes-v.size < f.saveBytes {
return
}
f.queue.Remove(e)
f.queuedBytes -= v.size
}
}
// fileNames returns all queued file names.
func (f *fileScorer) fileNames() []string {
res := make([]string, 0, f.queue.Len())
e := f.queue.Front()
for e != nil {
res = append(res, e.Value.(queuedFile).name)
e = e.Next()
}
return res
}
func (f *fileScorer) reset() {
f.queue.Init()
f.queuedBytes = 0
}
func (f *fileScorer) queueString() string {
var res strings.Builder
e := f.queue.Front()
i := 0
for e != nil {
v := e.Value.(queuedFile)
if i > 0 {
res.WriteByte('\n')
}
res.WriteString(fmt.Sprintf("%03d: %s (score: %.3f, bytes: %d)", i, v.name, v.score, v.size))
i++
e = e.Next()
}
return res.String()
}
// bytesToClear() returns the number of bytes to clear to reach low watermark
// w.r.t quota given disk total and free space, quota in % allocated to cache
// and low watermark % w.r.t allowed quota.
func bytesToClear(total, free int64, quotaPct, lowWatermark uint64) uint64 {
used := (total - free)
quotaAllowed := total * (int64)(quotaPct) / 100
lowWMUsage := (total * (int64)(lowWatermark*quotaPct) / (100 * 100))
return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
}

105
cmd/disk-cache-utils_test.go
View File

@ -29,16 +29,16 @@ func TestGetCacheControlOpts(t *testing.T) {
testCases := []struct {
cacheControlHeaderVal string
expiryHeaderVal time.Time
expectedCacheControl cacheControl
expectedCacheControl *cacheControl
expectedErr bool
}{
{"", timeSentinel, cacheControl{}, false},
{"max-age=2592000, public", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
{"max-age=2592000, no-store", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
{"must-revalidate, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
{"s-maxAge=2500, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
{"s-maxAge=2500, max-age=600", expiry, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
{"s-maxAge=2500, max-age=600s", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
{"", timeSentinel, nil, false},
{"max-age=2592000, public", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
{"max-age=2592000, no-store", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
{"must-revalidate, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
{"s-maxAge=2500, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
{"s-maxAge=2500, max-age=600s", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
}
for _, testCase := range testCases {
@ -49,7 +49,7 @@ func TestGetCacheControlOpts(t *testing.T) {
m["expires"] = testCase.expiryHeaderVal.String()
}
c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})
if testCase.expectedErr && (c != cacheControl{}) {
if testCase.expectedErr && (c != nil) {
t.Errorf("expected err, got <nil>")
}
if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
@ -83,3 +83,90 @@ func TestIsMetadataSame(t *testing.T) {
}
}
}
func TestNewFileScorer(t *testing.T) {
fs, err := newFileScorer(1000, time.Now().Unix(), 10)
if err != nil {
t.Fatal(err)
}
if len(fs.fileNames()) != 0 {
t.Fatal("non zero files??")
}
now := time.Now()
fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
fs.addFile("older", now.Add(-time.Hour), 1000, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"older"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.reset()
fs.addFile("bigger", now.Add(-time.Minute), 2000, 10)
fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"bigger"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.reset()
fs.addFile("less", now.Add(-time.Minute), 1000, 5)
fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.reset()
fs.addFile("small", now.Add(-time.Minute), 200, 10)
fs.addFile("medium", now.Add(-time.Minute), 300, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"medium", "small"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.addFile("large", now.Add(-time.Minute), 700, 10)
fs.addFile("xsmol", now.Add(-time.Minute), 7, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"large", "medium"}) {
t.Fatal("unexpected file list", fs.queueString())
}
fs.reset()
fs.addFile("less", now.Add(-time.Minute), 500, 5)
fs.addFile("recent", now.Add(-time.Minute), 500, 10)
if !fs.adjustSaveBytes(-500) {
t.Fatal("we should still need more bytes, got false")
}
// We should only need 500 bytes now.
if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
t.Fatal("unexpected file list", fs.queueString())
}
if fs.adjustSaveBytes(-500) {
t.Fatal("we shouldn't need any more bytes, got true")
}
fs, err = newFileScorer(1000, time.Now().Unix(), 10)
if err != nil {
t.Fatal(err)
}
fs.addFile("bigger", now.Add(-time.Minute), 50, 10)
// sorting should be consistent after adjusting savebytes.
fs.adjustSaveBytes(-800)
fs.addFile("smaller", now.Add(-time.Minute), 40, 10)
if !reflect.DeepEqual(fs.fileNames(), []string{"bigger", "smaller"}) {
t.Fatal("unexpected file list", fs.queueString())
}
}
func TestBytesToClear(t *testing.T) {
testCases := []struct {
total int64
free int64
quotaPct uint64
watermarkLow uint64
expected uint64
}{
{1000, 800, 40, 90, 0},
{1000, 200, 40, 90, 400},
{1000, 400, 40, 90, 240},
{1000, 600, 40, 90, 40},
{1000, 600, 40, 70, 120},
{1000, 1000, 90, 70, 0},
{1000, 0, 90, 70, 370},
}
for i, tc := range testCases {
toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow)
if tc.expected != toClear {
t.Errorf("test %d expected %v, got %v", i, tc.expected, toClear)
}
}
}

61
cmd/disk-cache.go
View File

@ -39,6 +39,7 @@ import (
const (
cacheBlkSize = int64(1 * 1024 * 1024)
cacheGCInterval = time.Minute * 30
)
// CacheStorageInfo - represents total, free capacity of
@ -174,7 +175,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
if c.isCacheExclude(bucket, object) || c.skipCache() {
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
}
var cc cacheControl
var cc *cacheControl
var cacheObjSize int64
// fetch diskCache if object is currently cached or nearest available cache drive
dcache, err := c.getCacheToLoc(ctx, bucket, object)
@ -191,8 +192,8 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
}
}
cc = cacheControlOpts(cacheReader.ObjInfo)
if (!cc.isEmpty() && !cc.isStale(cacheReader.ObjInfo.ModTime)) ||
cc.onlyIfCached {
if cc != nil && (!cc.isStale(cacheReader.ObjInfo.ModTime) ||
cc.onlyIfCached) {
// This is a cache hit, mark it so
bytesServed := cacheReader.ObjInfo.Size
if rs != nil {
@ -259,11 +260,8 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
c.cacheStats.incMiss()
// Since we got here, we are serving the request from backend,
// and also adding the object to the cache.
if !dcache.diskUsageLow() {
select {
case dcache.purgeChan <- struct{}{}:
default:
}
if dcache.diskUsageHigh() {
dcache.incGCCounter()
}
bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
@ -330,12 +328,12 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
if err != nil {
return getObjectInfoFn(ctx, bucket, object, opts)
}
var cc cacheControl
var cc *cacheControl
// if cache control setting is valid, avoid HEAD operation to backend
cachedObjInfo, _, cerr := dcache.Stat(ctx, bucket, object)
if cerr == nil {
cc = cacheControlOpts(cachedObjInfo)
if !cc.isStale(cachedObjInfo.ModTime) {
if cc == nil || (cc != nil && !cc.isStale(cachedObjInfo.ModTime)) {
// This is a cache hit, mark it so
c.cacheStats.incHit()
return cachedObjInfo, nil
@ -522,15 +520,10 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
if quota == 0 {
quota = config.Quota
}
cache, err := newDiskCache(dir, config.Expiry, quota, config.After)
cache, err := newDiskCache(dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh)
if err != nil {
return nil, false, err
}
// Start the purging go-routine for entries that have expired if no migration in progress
if !migrating {
go cache.purge()
}
caches = append(caches, cache)
}
return caches, migrating, nil
@ -577,13 +570,12 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
}
errCnt := 0
for index, err := range g.Wait() {
for _, err := range g.Wait() {
if err != nil {
errCnt++
logger.LogIf(ctx, err)
continue
}
go c.cache[index].purge()
}
if errCnt > 0 {
@ -697,5 +689,38 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
if migrateSw {
go c.migrateCacheFromV1toV2(ctx)
}
go c.gc(ctx, GlobalServiceDoneCh)
return c, nil
}
func (c *cacheObjects) gc(ctx context.Context, doneCh chan struct{}) {
ticker := time.NewTicker(cacheGCInterval)
var gcLock sync.Mutex
defer ticker.Stop()
for {
select {
case <-doneCh:
return
case <-ticker.C:
if c.migrating {
continue
}
var wg sync.WaitGroup
for _, dcache := range c.cache {
if dcache.gcCount() == 0 {
continue
}
gcLock.Lock()
wg.Add(1)
go func(d *diskCache, l *sync.Mutex) {
defer wg.Done()
d.resetGCCounter()
d.purge(ctx, doneCh)
l.Unlock()
}(dcache, &gcLock)
}
wg.Wait()
}
}
}

16
cmd/disk-cache_test.go
View File

@ -27,15 +27,15 @@ import (
)
// Initialize cache objects.
func initCacheObjects(disk string, cacheMaxUse, cacheAfter int) (*diskCache, error) {
return newDiskCache(disk, 80, cacheMaxUse, cacheAfter)
func initCacheObjects(disk string, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh int) (*diskCache, error) {
return newDiskCache(disk, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh)
}
// inits diskCache struct for nDisks
func initDiskCaches(drives []string, cacheMaxUse, cacheAfter int, t *testing.T) ([]*diskCache, error) {
func initDiskCaches(drives []string, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh int, t *testing.T) ([]*diskCache, error) {
var cb []*diskCache
for _, d := range drives {
obj, err := initCacheObjects(d, cacheMaxUse, cacheAfter)
obj, err := initCacheObjects(d, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh)
if err != nil {
return nil, err
}
@ -70,7 +70,7 @@ func TestGetCachedLoc(t *testing.T) {
if err != nil {
t.Fatal(err)
}
d, err := initDiskCaches(fsDirs, 100, 1, t)
d, err := initDiskCaches(fsDirs, 100, 1, 80, 90, t)
if err != nil {
t.Fatal(err)
}
@ -109,7 +109,7 @@ func TestGetCacheMaxUse(t *testing.T) {
if err != nil {
t.Fatal(err)
}
d, err := initDiskCaches(fsDirs, 80, 1, t)
d, err := initDiskCaches(fsDirs, 80, 1, 80, 90, t)
if err != nil {
t.Fatal(err)
}
@ -180,7 +180,7 @@ func TestDiskCache(t *testing.T) {
if err != nil {
t.Fatal(err)
}
d, err := initDiskCaches(fsDirs, 100, 0, t)
d, err := initDiskCaches(fsDirs, 100, 0, 80, 90, t)
if err != nil {
t.Fatal(err)
}
@ -258,7 +258,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
if err != nil {
t.Fatal(err)
}
d, err := initDiskCaches(fsDirs, 80, 0, t)
d, err := initDiskCaches(fsDirs, 80, 0, 80, 90, t)
if err != nil {
t.Fatal(err)
}

6
cmd/gateway/azure/gateway-azure.go
View File

@ -94,8 +94,10 @@ EXAMPLES:
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
{{.Prompt}} {{.HelpName}}
`

7
cmd/gateway/b2/gateway-b2.go
View File

@ -69,8 +69,11 @@ EXAMPLES:
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
{{.Prompt}} {{.HelpName}}
`
minio.RegisterGatewayCommand(cli.Command{

6
cmd/gateway/gcs/gateway-gcs.go
View File

@ -124,8 +124,10 @@ EXAMPLES:
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
{{.Prompt}} {{.HelpName}} mygcsprojectid
`

6
cmd/gateway/hdfs/gateway-hdfs.go
View File

@ -75,8 +75,10 @@ EXAMPLES:
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
{{.Prompt}} {{.HelpName}} hdfs://namenode:8200
`

7
cmd/gateway/nas/gateway-nas.go
View File

@ -52,8 +52,11 @@ EXAMPLES:
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
{{.Prompt}} {{.HelpName}} /shared/nasvol
`

6
cmd/gateway/oss/gateway-oss.go
View File

@ -71,8 +71,10 @@ EXAMPLES:
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
{{.Prompt}} {{.HelpName}}
`

6
cmd/gateway/s3/gateway-s3.go
View File

@ -66,8 +66,10 @@ EXAMPLES:
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
{{.Prompt}} {{.HelpName}}
`

16
docs/disk-caching/DESIGN.md
View File

@ -11,30 +11,32 @@ minio gateway <name> -h
CACHE:
MINIO_CACHE_DRIVES: List of mounted cache drives or directories delimited by ","
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ","
MINIO_CACHE_EXPIRY: Cache expiry duration in days
MINIO_CACHE_QUOTA: Maximum permitted usage of the cache in percentage (0-100).
MINIO_CACHE_AFTER: Minimum number of access before caching an object.
MINIO_CACHE_WATERMARK_LOW: % of cache quota at which cache eviction stops
MINIO_CACHE_WATERMARK_HIGH: % of cache quota at which cache eviction starts
...
...
7. Start MinIO gateway to s3 with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/export1 ... /mnt/export24',
exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension
with expiry up to 40 days. Cache only those objects accessed atleast 3 times.
exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension. Cache only those objects accessed atleast 3 times. Garbage collection triggers in at high water mark (i.e. cache disk usage reaches 90% of cache quota) or at 72% and evicts oldest objects by access time until low watermark is reached ( 70% of cache quota) , i.e. 63% of disk usage.
$ export MINIO_CACHE_DRIVES="/mnt/drive1,/mnt/drive2,/mnt/export{1..24}"
$ export MINIO_CACHE_EXCLUDE="mybucket/*,*.pdf"
$ export MINIO_CACHE_EXPIRY=40
$ export MINIO_CACHE_QUOTA=80
$ export MINIO_CACHE_AFTER=3
$ export MINIO_CACHE_WATERMARK_LOW=70
$ export MINIO_CACHE_WATERMARK_HIGH=90
$ minio gateway s3
```
## Assumptions
- Disk cache size defaults to 80% of your drive capacity.
- Disk cache quota defaults to 80% of your drive capacity.
- The cache drives are required to be a filesystem mount point with [`atime`](http://kerolasa.github.io/filetimes.html) support to be enabled on the drive. Alternatively writable directories with atime support can be specified in MINIO_CACHE_DRIVES
- Expiration of each cached entry takes user provided expiry as a hint, and defaults to 90 days if not provided.
- Garbage collection sweep of the expired cache entries happens whenever cache usage is > 80% of drive capacity, GC continues until sufficient disk space is reclaimed.
- Garbage collection sweep happens whenever cache disk usage reaches high watermark with respect to the configured cache quota , GC evicts least recently accessed objects until cache low watermark is reached with respect to the configured cache quota. Garbage collection runs a cache eviction sweep at 30 minute intervals.
- An object is only cached when drive has sufficient disk space.
## Behavior

8
docs/disk-caching/README.md
View File

@ -13,17 +13,19 @@ Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quicksta
### 2. Run MinIO gateway with cache
Disk caching can be enabled by setting the `cache` environment variables for MinIO gateway . `cache` environment variables takes the mounted drive(s) or directory paths, cache expiry duration (in days) and any wildcard patterns to exclude from being cached.
Disk caching can be enabled by setting the `cache` environment variables for MinIO gateway . `cache` environment variables takes the mounted drive(s) or directory paths, any wildcard patterns to exclude from being cached,low and high watermarks for garbage collection and the minimum accesses before caching an object.
Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, with expiry up to 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a s3 gateway setup. Objects are cached if they have been accessed three times or more.Cache max usage is restricted to 80% of disk capacity in this example.
Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension on a s3 gateway setup. Objects are cached if they have been accessed three times or more.Cache max usage is restricted to 80% of disk capacity in this example. Garbage collection is triggered when high watermark is reached - i.e. at 72% of cache disk usage and clears least recently accessed entries until the disk usage drops to low watermark - i.e. cache disk usage drops to 56% (70% of 80% quota)
```bash
export MINIO_CACHE="on"
export MINIO_CACHE_DRIVES="/mnt/drive1,/mnt/drive2,/mnt/cache{1...3}"
export MINIO_CACHE_EXPIRY=90
export MINIO_CACHE_EXCLUDE="*.pdf,mybucket/*"
export MINIO_CACHE_QUOTA=80
export MINIO_CACHE_AFTER=3
export MINIO_CACHE_WATERMARK_LOW=70
export MINIO_CACHE_WATERMARK_HIGH=90
minio gateway s3
```