fix: remove persistence layer for metacache store in memory (#11538)

store the cache in-memory instead of disks to avoid large
write amplifications for list heavy workloads, store in
memory instead and let it auto expire.
This commit is contained in:
Harshavardhana
2021-02-24 15:51:41 -08:00
committed by GitHub
parent b912e9ab41
commit b23659927c
14 changed files with 683 additions and 605 deletions

View File

@@ -29,6 +29,7 @@ import (
"time"
"github.com/dchest/siphash"
"github.com/dustin/go-humanize"
"github.com/google/uuid"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio-go/v7/pkg/tags"
@@ -37,6 +38,7 @@ import (
"github.com/minio/minio/pkg/console"
"github.com/minio/minio/pkg/dsync"
"github.com/minio/minio/pkg/madmin"
"github.com/minio/minio/pkg/objcache"
"github.com/minio/minio/pkg/sync/errgroup"
)
@@ -366,6 +368,11 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
// setCount * setDriveCount with each memory upto blockSizeV1.
bp := bpool.NewBytePoolCap(n, blockSizeV1, blockSizeV1*2)
mcache, err := objcache.New(1*humanize.GiByte, objcache.DefaultExpiry)
if err != nil {
return nil, err
}
for i := 0; i < setCount; i++ {
s.erasureDisks[i] = make([]StorageAPI, setDriveCount)
}
@@ -412,6 +419,7 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
getEndpoints: s.GetEndpoints(i),
nsMutex: mutex,
bp: bp,
metaCache: mcache,
mrfOpCh: make(chan partialOperation, 10000),
}
}