Add cache eviction low and high watermarks (#8958)

To allow better control the cache eviction process.

Introduce MINIO_CACHE_WATERMARK_LOW and 
MINIO_CACHE_WATERMARK_HIGH env. variables to specify 
when to stop/start cache eviction process. 

Deprecate MINIO_CACHE_EXPIRY environment variable. Cache 
gc sweeps at 30 minute intervals whenever high watermark is
reached to clear least recently accessed entries in the cache
until sufficient space is cleared to reach the low watermark.

Garbage collection uses an adaptive file scoring approach based
on last access time, with greater weights assigned to larger
objects and those with more hits to find the candidates for eviction.

Thanks to @klauspost for this file scoring algorithm

Co-authored-by: Klaus Post <klauspost@minio.io>

cmd/disk-cache-utils.go

@@ -17,8 +17,12 @@
 package cmd
 
 import (
+	"container/list"
 	"encoding/hex"
+	"errors"
+	"fmt"
 	"io"
+	"math"
 	"os"
 	"path"
 	"strconv"
@@ -57,13 +61,8 @@ type cacheControl struct {
 	noCache      bool
 }
 
-func (c cacheControl) isEmpty() bool {
-	return c == cacheControl{}
-
-}
-
-func (c cacheControl) isStale(modTime time.Time) bool {
-	if c.isEmpty() {
+func (c *cacheControl) isStale(modTime time.Time) bool {
+	if c == nil {
 		return false
 	}
 	// response will never be stale if only-if-cached is set
@@ -100,7 +99,8 @@ func (c cacheControl) isStale(modTime time.Time) bool {
 }
 
 // returns struct with cache-control settings from user metadata.
-func cacheControlOpts(o ObjectInfo) (c cacheControl) {
+func cacheControlOpts(o ObjectInfo) *cacheControl {
+	c := cacheControl{}
 	m := o.UserDefined
 	if o.Expires != timeSentinel {
 		c.expiry = o.Expires
@@ -114,7 +114,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 
 	}
 	if headerVal == "" {
-		return
+		return nil
 	}
 	headerVal = strings.ToLower(headerVal)
 	headerVal = strings.TrimSpace(headerVal)
@@ -146,7 +146,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 			p[0] == "max-stale" {
 			i, err := strconv.Atoi(p[1])
 			if err != nil {
-				return cacheControl{}
+				return nil
 			}
 			if p[0] == "max-age" {
 				c.maxAge = i
@@ -162,7 +162,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 			}
 		}
 	}
-	return c
+	return &c
 }
 
 // backendDownError returns true if err is due to backend failure or faulty disk if in server mode
@@ -283,3 +283,161 @@ func isMetadataSame(m1, m2 map[string]string) bool {
 	}
 	return true
 }
+
+type fileScorer struct {
+	saveBytes int64
+	now       int64
+	maxHits   int
+	// 1/size for consistent score.
+	sizeMult float64
+
+	// queue is a linked list of files we want to delete.
+	// The list is kept sorted according to score, highest at top, lowest at bottom.
+	queue       list.List
+	queuedBytes int64
+}
+
+type queuedFile struct {
+	name  string
+	size  int64
+	score float64
+}
+
+// newFileScorer allows to collect files to save a specific number of bytes.
+// Each file is assigned a score based on its age, size and number of hits.
+// A list of files is maintained
+func newFileScorer(saveBytes int64, now int64, maxHits int) (*fileScorer, error) {
+	if saveBytes <= 0 {
+		return nil, errors.New("newFileScorer: saveBytes <= 0")
+	}
+	if now < 0 {
+		return nil, errors.New("newFileScorer: now < 0")
+	}
+	if maxHits <= 0 {
+		return nil, errors.New("newFileScorer: maxHits <= 0")
+	}
+	f := fileScorer{saveBytes: saveBytes, maxHits: maxHits, now: now, sizeMult: 1 / float64(saveBytes)}
+	f.queue.Init()
+	return &f, nil
+}
+
+func (f *fileScorer) addFile(name string, lastAccess time.Time, size int64, hits int) {
+	// Calculate how much we want to delete this object.
+	file := queuedFile{
+		name: name,
+		size: size,
+	}
+	score := float64(f.now - lastAccess.Unix())
+	// Size as fraction of how much we want to save, 0->1.
+	szWeight := math.Max(0, (math.Min(1, float64(size)*f.sizeMult)))
+	// 0 at f.maxHits, 1 at 0.
+	hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
+	file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
+	// If we still haven't saved enough, just add the file
+	if f.queuedBytes < f.saveBytes {
+		f.insertFile(file)
+		f.trimQueue()
+		return
+	}
+	// If we score less than the worst, don't insert.
+	worstE := f.queue.Back()
+	if worstE != nil && file.score < worstE.Value.(queuedFile).score {
+		return
+	}
+	f.insertFile(file)
+	f.trimQueue()
+}
+
+// adjustSaveBytes allows to adjust the number of bytes to save.
+// This can be used to adjust the count on the fly.
+// Returns true if there still is a need to delete files (saveBytes >0),
+// false if no more bytes needs to be saved.
+func (f *fileScorer) adjustSaveBytes(n int64) bool {
+	f.saveBytes += n
+	if f.saveBytes <= 0 {
+		f.queue.Init()
+		f.saveBytes = 0
+		return false
+	}
+	if n < 0 {
+		f.trimQueue()
+	}
+	return true
+}
+
+// insertFile will insert a file into the list, sorted by its score.
+func (f *fileScorer) insertFile(file queuedFile) {
+	e := f.queue.Front()
+	for e != nil {
+		v := e.Value.(queuedFile)
+		if v.score < file.score {
+			break
+		}
+		e = e.Next()
+	}
+	f.queuedBytes += file.size
+	// We reached the end.
+	if e == nil {
+		f.queue.PushBack(file)
+		return
+	}
+	f.queue.InsertBefore(file, e)
+}
+
+// trimQueue will trim the back of queue and still keep below wantSave.
+func (f *fileScorer) trimQueue() {
+	for {
+		e := f.queue.Back()
+		if e == nil {
+			return
+		}
+		v := e.Value.(queuedFile)
+		if f.queuedBytes-v.size < f.saveBytes {
+			return
+		}
+		f.queue.Remove(e)
+		f.queuedBytes -= v.size
+	}
+}
+
+// fileNames returns all queued file names.
+func (f *fileScorer) fileNames() []string {
+	res := make([]string, 0, f.queue.Len())
+	e := f.queue.Front()
+	for e != nil {
+		res = append(res, e.Value.(queuedFile).name)
+		e = e.Next()
+	}
+	return res
+}
+
+func (f *fileScorer) reset() {
+	f.queue.Init()
+	f.queuedBytes = 0
+}
+
+func (f *fileScorer) queueString() string {
+	var res strings.Builder
+	e := f.queue.Front()
+	i := 0
+	for e != nil {
+		v := e.Value.(queuedFile)
+		if i > 0 {
+			res.WriteByte('\n')
+		}
+		res.WriteString(fmt.Sprintf("%03d: %s (score: %.3f, bytes: %d)", i, v.name, v.score, v.size))
+		i++
+		e = e.Next()
+	}
+	return res.String()
+}
+
+// bytesToClear() returns the number of bytes to clear to reach low watermark
+// w.r.t quota given disk total and free space, quota in % allocated to cache
+// and low watermark % w.r.t allowed quota.
+func bytesToClear(total, free int64, quotaPct, lowWatermark uint64) uint64 {
+	used := (total - free)
+	quotaAllowed := total * (int64)(quotaPct) / 100
+	lowWMUsage := (total * (int64)(lowWatermark*quotaPct) / (100 * 100))
+	return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
+}