MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-23 21:55:53 -05:00
Code Issues Packages Projects Releases Wiki Activity

remove FIFO bucket quota, use ILM expiration instead (#14206)

Browse Source
This commit is contained in:
Harshavardhana 2022-01-31 11:07:04 -08:00 committed by GitHub
parent b5d35c7e09
commit dbd05d6e82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 147 additions and 721 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
cmd/bucket-quota.go
View File

@ -24,8 +24,6 @@ import (
"time"
"github.com/minio/madmin-go"
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/logger"
)
// BucketQuotaSys - map of bucket and quota configuration.
@ -51,6 +49,35 @@ func NewBucketQuotaSys() *BucketQuotaSys {
return &BucketQuotaSys{}
}
// Init initialize bucket quota.
func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
sys.bucketStorageCache.Once.Do(func() {
sys.bucketStorageCache.TTL = 1 * time.Second
sys.bucketStorageCache.Update = func() (interface{}, error) {
ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
defer done()
return loadDataUsageFromBackend(ctx, objAPI)
}
})
}
// GetBucketUsageInfo return bucket usage info for a given bucket
func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
v, err := sys.bucketStorageCache.Get()
if err != nil {
return BucketUsageInfo{}, err
}
dui, ok := v.(DataUsageInfo)
if !ok {
return BucketUsageInfo{}, fmt.Errorf("internal error: Unexpected DUI data type: %T", v)
}
bui := dui.BucketsUsage[bucket]
return bui, nil
}
// parseBucketQuota parses BucketQuota from json
func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota, err error) {
quotaCfg = &madmin.BucketQuota{}
@ -63,45 +90,23 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
return
}
func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
objAPI := newObjectLayerFn()
if objAPI == nil {
return errServerNotInitialized
func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string, size int64) error {
if size < 0 {
return nil
}
sys.bucketStorageCache.Once.Do(func() {
sys.bucketStorageCache.TTL = 1 * time.Second
sys.bucketStorageCache.Update = func() (interface{}, error) {
ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
defer done()
return loadDataUsageFromBackend(ctx, objAPI)
}
})
q, err := sys.Get(bucket)
if err != nil {
return err
}
if q != nil && q.Type == madmin.HardQuota && q.Quota > 0 {
v, err := sys.bucketStorageCache.Get()
bui, err := sys.GetBucketUsageInfo(bucket)
if err != nil {
return err
}
dui, ok := v.(DataUsageInfo)
if !ok {
return fmt.Errorf("internal error: Unexpected DUI data type: %T", v)
}
bui, ok := dui.BucketsUsage[bucket]
if !ok {
// bucket not found, cannot enforce quota
// call will fail anyways later.
return nil
}
if (bui.Size + uint64(size)) >= q.Quota {
if bui.Size > 0 && ((bui.Size + uint64(size)) >= q.Quota) {
return BucketQuotaExceeded{Bucket: bucket}
}
}
@ -109,118 +114,9 @@ func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64)
return nil
}
func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
if size < 0 {
func enforceBucketQuotaHard(ctx context.Context, bucket string, size int64) error {
if globalBucketQuotaSys == nil {
return nil
}
return globalBucketQuotaSys.check(ctx, bucket, size)
}
// enforceFIFOQuota deletes objects in FIFO order until sufficient objects
// have been deleted so as to bring bucket usage within quota.
func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui BucketUsageInfo) {
// Check if the current bucket has quota restrictions, if not skip it
cfg, err := globalBucketQuotaSys.Get(bucket)
if err != nil {
return
}
if cfg.Type != madmin.FIFOQuota {
return
}
var toFree uint64
if bui.Size > cfg.Quota && cfg.Quota > 0 {
toFree = bui.Size - cfg.Quota
}
if toFree <= 0 {
return
}
// Allocate new results channel to receive ObjectInfo.
objInfoCh := make(chan ObjectInfo)
versioned := globalBucketVersioningSys.Enabled(bucket)
// Walk through all objects
if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkVersions: versioned}); err != nil {
logger.LogIf(ctx, err)
return
}
// reuse the fileScorer used by disk cache to score entries by
// ModTime to find the oldest objects in bucket to delete. In
// the context of bucket quota enforcement - number of hits are
// irrelevant.
scorer, err := newFileScorer(toFree, UTCNow().Unix(), 1)
if err != nil {
logger.LogIf(ctx, err)
return
}
rcfg, _ := globalBucketObjectLockSys.Get(bucket)
for obj := range objInfoCh {
// skip objects currently under retention
if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
continue
}
scorer.addFileWithObjInfo(obj, 1)
}
// If we saw less than quota we are good.
if scorer.seenBytes <= cfg.Quota {
return
}
// Calculate how much we want to delete now.
toFreeNow := scorer.seenBytes - cfg.Quota
// We were less over quota than we thought. Adjust so we delete less.
// If we are more over, leave it for the next run to pick up.
if toFreeNow < toFree {
if !scorer.adjustSaveBytes(int64(toFreeNow) - int64(toFree)) {
// We got below or at quota.
return
}
}
var objects []ObjectToDelete
numKeys := len(scorer.fileObjInfos())
for i, obj := range scorer.fileObjInfos() {
objects = append(objects, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: obj.Name,
VersionID: obj.VersionID,
},
})
if len(objects) < maxDeleteList && (i < numKeys-1) {
// skip deletion until maxDeleteList or end of slice
continue
}
if len(objects) == 0 {
break
}
// Deletes a list of objects.
_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
Versioned: versioned,
})
for j := range deleteErrs {
if deleteErrs[j] != nil {
logger.LogIf(ctx, deleteErrs[j])
continue
}
// Notify object deleted event.
sendEvent(eventArgs{
EventName: event.ObjectRemovedDelete,
BucketName: bucket,
Object: obj,
Host: "Internal: [FIFO-QUOTA-EXPIRY]",
})
}
objects = nil
}
return globalBucketQuotaSys.enforceQuotaHard(ctx, bucket, size)
}

29
cmd/data-usage-cache.go
View File

@ -798,35 +798,6 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
return dst
}
// bucketUsageInfo returns the buckets usage info.
// If not found all values returned are zero values.
func (d *dataUsageCache) bucketUsageInfo(bucket string) BucketUsageInfo {
e := d.find(bucket)
if e == nil {
return BucketUsageInfo{}
}
flat := d.flatten(*e)
bui := BucketUsageInfo{
Size: uint64(flat.Size),
ObjectsCount: flat.Objects,
ObjectSizesHistogram: flat.ObjSizes.toMap(),
}
if flat.ReplicationStats != nil {
bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
for arn, stat := range flat.ReplicationStats.Targets {
bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
ReplicationPendingSize: stat.PendingSize,
ReplicatedSize: stat.ReplicatedSize,
ReplicationFailedSize: stat.FailedSize,
ReplicationPendingCount: stat.PendingCount,
ReplicationFailedCount: stat.FailedCount,
}
}
}
return bui
}
// sizeRecursive returns the path as a flattened entry.
func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
root := d.find(path)

16
cmd/disk-cache-utils.go
View File

@ -477,22 +477,6 @@ func (f *fileScorer) trimQueue() {
}
}
// fileObjInfos returns all queued file object infos
func (f *fileScorer) fileObjInfos() []ObjectInfo {
res := make([]ObjectInfo, 0, f.queue.Len())
e := f.queue.Front()
for e != nil {
qfile := e.Value.(queuedFile)
res = append(res, ObjectInfo{
Name: qfile.name,
Size: int64(qfile.size),
VersionID: qfile.versionID,
})
e = e.Next()
}
return res
}
func (f *fileScorer) purgeFunc(p func(qfile queuedFile)) {
e := f.queue.Front()
for e != nil {

6
cmd/erasure-server-pool.go
View File

@ -619,12 +619,6 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
return
case v := <-updateCloser:
update()
// Enforce quotas when all is done.
if firstErr == nil {
for _, b := range allBuckets {
enforceFIFOQuotaBucket(ctx, z, b.Name, allMerged.bucketUsageInfo(b.Name))
}
}
close(v)
return
case <-updateTicker.C:

2
cmd/fs-v1.go
View File

@ -329,7 +329,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
logger.LogIf(ctx, totalCache.save(ctx, fs, dataUsageCacheName))
cloned := totalCache.clone()
updates <- cloned.dui(dataUsageRoot, buckets)
enforceFIFOQuotaBucket(ctx, fs, b.Name, cloned.bucketUsageInfo(b.Name))
}
return nil

1
cmd/object-api-interface.go
View File

@ -46,7 +46,6 @@ type ObjectOptions struct {
ServerSideEncryption encrypt.ServerSide
VersionSuspended bool // indicates if the bucket was previously versioned but is currently suspended.
Versioned bool // indicates if the bucket is versioned
WalkVersions bool // indicates if the we are interested in walking versions
VersionID string // Specifies the versionID which needs to be overwritten or read
MTime time.Time // Is only set in POST/PUT operations
Expires time.Time // Is only used in POST/PUT operations

10
cmd/object-handlers.go
View File

@ -1106,7 +1106,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
length := actualSize
if !cpSrcDstSame {
if err := enforceBucketQuota(ctx, dstBucket, actualSize); err != nil {
if err := enforceBucketQuotaHard(ctx, dstBucket, actualSize); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@ -1659,7 +1659,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
}
}
if err := enforceBucketQuota(ctx, bucket, size); err != nil {
if err := enforceBucketQuotaHard(ctx, bucket, size); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@ -1996,7 +1996,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
return
}
if err := enforceBucketQuota(ctx, bucket, size); err != nil {
if err := enforceBucketQuotaHard(ctx, bucket, size); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@ -2465,7 +2465,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
return
}
if err := enforceBucketQuota(ctx, dstBucket, actualPartSize); err != nil {
if err := enforceBucketQuotaHard(ctx, dstBucket, actualPartSize); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@ -2754,7 +2754,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
}
}
if err := enforceBucketQuota(ctx, bucket, size); err != nil {
if err := enforceBucketQuotaHard(ctx, bucket, size); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}

3
cmd/server-main.go
View File

@ -577,6 +577,9 @@ func serverMain(ctx *cli.Context) {
// Initialize site replication manager.
globalSiteReplicationSys.Init(GlobalContext, newObject)
// Initialize quota manager.
globalBucketQuotaSys.Init(newObject)
// Initialize users credentials and policies in background right after config has initialized.
go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval)

11
docs/bucket/quota/README.md
View File

@ -2,10 +2,7 @@
![quota](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/quota/bucketquota.png)
Buckets can be configured to have one of two types of quota configuration - FIFO and Hard quota.
- `Hard` quota disallows writes to the bucket after configured quota limit is reached.
- `FIFO` quota automatically deletes oldest content until bucket usage falls within configured limit while permitting writes.
Buckets can be configured to have `Hard` quota - it disallows writes to the bucket after configured quota limit is reached.
> NOTE: Bucket quotas are not supported under gateway or standalone single disk deployments.
@ -21,12 +18,6 @@ Buckets can be configured to have one of two types of quota configuration - FIFO
$ mc admin bucket quota myminio/mybucket --hard 1gb
```
### Set FIFO quota of 5GB for a bucket "mybucket" on MinIO to allow automatic deletion of older content to ensure bucket usage remains within 5GB
```sh
$ mc admin bucket quota myminio/mybucket --fifo 5gb
```
### Verify the quota configured on `mybucket` on MinIO
```sh

47
go.mod
View File

@ -44,11 +44,11 @@ require (
github.com/lib/pq v1.9.0
github.com/miekg/dns v1.1.43
github.com/minio/cli v1.22.0
github.com/minio/console v0.14.1
github.com/minio/console v0.14.2-0.20220131171006-1e00111b0074
github.com/minio/csvparser v1.0.0
github.com/minio/highwayhash v1.0.2
github.com/minio/kes v0.14.0
github.com/minio/madmin-go v1.2.9
github.com/minio/madmin-go v1.3.0
github.com/minio/minio-go/v7 v7.0.21
github.com/minio/parquet-go v1.1.0
github.com/minio/pkg v1.1.15
@ -80,8 +80,8 @@ require (
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
github.com/yargevad/filepathx v1.0.0
github.com/zeebo/xxh3 v1.0.0
go.etcd.io/etcd/api/v3 v3.5.0
go.etcd.io/etcd/client/v3 v3.5.0
go.etcd.io/etcd/api/v3 v3.5.1
go.etcd.io/etcd/client/v3 v3.5.1
go.uber.org/atomic v1.9.0
go.uber.org/zap v1.19.1
golang.org/x/crypto v0.0.0-20220128200615-198e4374d7ed
@ -101,11 +101,14 @@ require (
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
github.com/briandowns/spinner v1.16.0 // indirect
github.com/charmbracelet/bubbles v0.10.0 // indirect
github.com/charmbracelet/bubbletea v0.19.3 // indirect
github.com/charmbracelet/lipgloss v0.4.1-0.20220128151721-7ec01f75421d // indirect
github.com/containerd/console v1.0.2 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/eapache/go-resiliency v1.2.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
@ -115,17 +118,17 @@ require (
github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
github.com/go-logr/logr v0.4.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/analysis v0.20.1 // indirect
github.com/go-openapi/errors v0.20.1 // indirect
github.com/go-openapi/analysis v0.21.2 // indirect
github.com/go-openapi/errors v0.20.2 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-openapi/runtime v0.21.0 // indirect
github.com/go-openapi/runtime v0.21.1 // indirect
github.com/go-openapi/spec v0.20.4 // indirect
github.com/go-openapi/strfmt v0.21.1 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/go-openapi/validate v0.20.3 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/goccy/go-json v0.7.9 // indirect
github.com/goccy/go-json v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
@ -134,6 +137,7 @@ require (
github.com/golang/snappy v0.0.3 // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/googleapis/gnostic v0.5.1 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
@ -151,11 +155,12 @@ require (
github.com/lestrrat-go/blackmagic v1.0.0 // indirect
github.com/lestrrat-go/httpcc v1.0.0 // indirect
github.com/lestrrat-go/iter v1.0.1 // indirect
github.com/lestrrat-go/jwx v1.2.7 // indirect
github.com/lestrrat-go/jwx v1.2.14 // indirect
github.com/lestrrat-go/option v1.0.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-colorable v0.1.10 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-ieproxy v0.0.1 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
@ -165,29 +170,33 @@ require (
github.com/minio/colorjson v1.0.1 // indirect
github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf // indirect
github.com/minio/filepath v1.0.0 // indirect
github.com/minio/mc v0.0.0-20211207230606-23a05f5a17f2 // indirect
github.com/minio/mc v0.0.0-20220129172038-3f23cdd921ae // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/operator v0.0.0-20220110040724-a5d59a342b7f // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.10.0 // indirect
github.com/nats-io/jwt/v2 v2.0.2 // indirect
github.com/nats-io/nats-streaming-server v0.21.2 // indirect
github.com/nats-io/nkeys v0.3.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pkg/profile v1.6.0 // indirect
github.com/pkg/xattr v0.4.3 // indirect
github.com/pkg/xattr v0.4.4 // indirect
github.com/posener/complete v1.2.3 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
github.com/prometheus/common v0.31.1 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rjeczalik/notify v0.9.2 // indirect
github.com/rs/xid v1.3.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/tidwall/gjson v1.11.0 // indirect
github.com/tidwall/gjson v1.12.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/sjson v1.2.3 // indirect
@ -196,7 +205,7 @@ require (
github.com/unrolled/secure v1.0.9 // indirect
github.com/xdg/stringprep v1.0.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
go.mongodb.org/mongo-driver v1.7.5 // indirect
go.opencensus.io v0.23.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
@ -205,8 +214,8 @@ require (
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9 // indirect
google.golang.org/grpc v1.41.0 // indirect
google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
google.golang.org/grpc v1.43.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect

565
go.sum
View File

File diff suppressed because it is too large Load Diff