diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index eb4642371..60e3c09f3 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -512,7 +512,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, opts := ObjectOptions{ VersionID: object.VersionID, Versioned: vc.PrefixEnabled(object.ObjectName), - VersionSuspended: vc.PrefixSuspended(object.ObjectName), + VersionSuspended: vc.Suspended(), } if replicateDeletes || object.VersionID != "" && hasLockEnabled || !globalTierConfigMgr.Empty() { @@ -578,8 +578,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, deleteList := toNames(objectsToDelete) dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{ - PrefixEnabledFn: vc.PrefixEnabled, - PrefixSuspendedFn: vc.PrefixSuspended, + PrefixEnabledFn: vc.PrefixEnabled, + VersionSuspended: vc.Suspended(), }) for i := range errs { @@ -635,22 +635,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, continue } - if replicateDeletes { - if dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending { - dv := DeletedObjectReplicationInfo{ - DeletedObject: dobj, - Bucket: bucket, - } - scheduleReplicationDelete(ctx, dv, objectAPI) + if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) { + dv := DeletedObjectReplicationInfo{ + DeletedObject: dobj, + Bucket: bucket, } - } - - } - - // Notify deleted event for objects. - for _, dobj := range deletedObjects { - if dobj.ObjectName == "" { - continue + scheduleReplicationDelete(ctx, dv, objectAPI) } eventName := event.ObjectRemovedDelete diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index 33c2a6405..731fb7678 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -271,7 +271,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet } // Skip replication if this object's prefix is excluded from being // versioned. - if delOpts.VersionSuspended { + if !delOpts.Versioned { return } opts := replication.ObjectOpts{ diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 58bd8b290..5a0639fe1 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1189,9 +1189,6 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec if objects[i].VersionID == "" { // MinIO extension to bucket version configuration suspended := opts.VersionSuspended - if opts.PrefixSuspendedFn != nil { - suspended = opts.PrefixSuspendedFn(objects[i].ObjectName) - } versioned := opts.Versioned if opts.PrefixEnabledFn != nil { versioned = opts.PrefixEnabledFn(objects[i].ObjectName) @@ -1204,7 +1201,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec // Versioning suspended means that we add a `null` version // delete marker, if not add a new version for this delete // marker. - if opts.Versioned { + if versioned { vr.VersionID = mustGetUUID() } } diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 991ab5070..10c5c5a58 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -77,8 +77,7 @@ type ObjectOptions struct { Mutate bool WalkAscending bool // return Walk results in ascending order of versions - PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix - PrefixSuspendedFn func(prefix string) bool // function which returns true if versioning is suspended on prefix + PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix } // ExpirationOptions represents object options for object expiration at objectLayer. diff --git a/cmd/object-api-options.go b/cmd/object-api-options.go index 5fb75fdfc..7c2c83fc5 100644 --- a/cmd/object-api-options.go +++ b/cmd/object-api-options.go @@ -166,7 +166,7 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts return opts, err } opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object) - opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(bucket, object) + opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket) delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker)) if delMarker != "" { switch delMarker { diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go index 8b503e84f..d02fe5451 100644 --- a/cmd/object-handlers-common.go +++ b/cmd/object-handlers-common.go @@ -275,8 +275,8 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD } vc, _ := globalBucketVersioningSys.Get(bucket) deletedObjs, errs := o.DeleteObjects(ctx, bucket, toDel, ObjectOptions{ - PrefixEnabledFn: vc.PrefixEnabled, - PrefixSuspendedFn: vc.PrefixSuspended, + PrefixEnabledFn: vc.PrefixEnabled, + VersionSuspended: vc.Suspended(), }) var logged bool for i, err := range errs { diff --git a/docs/bucket/versioning/README.md b/docs/bucket/versioning/README.md index 7c1aa44d2..6b4a1e0c6 100644 --- a/docs/bucket/versioning/README.md +++ b/docs/bucket/versioning/README.md @@ -65,6 +65,8 @@ Similarly to suspend versioning set the configuration with Status set to `Suspen ## MinIO extension to Bucket Versioning ### Motivation +**PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on https://subnet.min.io.** + Spark/Hadoop workloads which use Hadoop MR Committer v1/v2 algorithm upload objects to a temporary prefix in a bucket. These objects are 'renamed' to a different prefix on Job commit. Object storage admins are forced to configure separate ILM policies to expire these objects and their versions to reclaim space. ### Solution @@ -76,19 +78,23 @@ To exclude objects under a list of prefix (glob) patterns from being versioned, true - app1-jobs/*/_temporary/ + */_temporary - app2-jobs/*/_magic/ + */__magic + + + */_staging ``` -Note: Objects matching these prefixes will behave as though versioning were suspended. These objects **will not** be replicated if replication is configured. - -Only users with explicit permissions or the root credential can configure the versioning state of any bucket. +### Features +- Objects matching these prefixes will behave as though versioning were suspended. These objects **will not** be replicated if bucket has replication configured. +- Objects matching these prefixes will also not leave delete markers, dramatically reduces namespace pollution while keeping the benefits of replication. +- Users with explicit permissions or the root credential can configure the versioning state of any bucket. ## Examples of enabling bucket versioning using MinIO Java SDK diff --git a/internal/bucket/versioning/versioning.go b/internal/bucket/versioning/versioning.go index 4069ab233..a986cc82e 100644 --- a/internal/bucket/versioning/versioning.go +++ b/internal/bucket/versioning/versioning.go @@ -39,7 +39,6 @@ const ( var ( errExcludedPrefixNotSupported = Errorf("excluded prefixes extension supported only when versioning is enabled") errTooManyExcludedPrefixes = Errorf("too many excluded prefixes") - errInvalidPrefixPattern = Errorf("invalid prefix pattern") ) // ExcludedPrefix - holds individual prefixes excluded from being versioned. @@ -73,11 +72,6 @@ func (v Versioning) Validate() error { if len(v.ExcludedPrefixes) > maxExcludedPrefixes { return errTooManyExcludedPrefixes } - for _, sprefix := range v.ExcludedPrefixes { - if !strings.HasSuffix(sprefix.Prefix, "/") { - return errInvalidPrefixPattern - } - } case Suspended: if len(v.ExcludedPrefixes) > 0 { diff --git a/internal/bucket/versioning/versioning_test.go b/internal/bucket/versioning/versioning_test.go index 8f34bbb29..718da5c02 100644 --- a/internal/bucket/versioning/versioning_test.go +++ b/internal/bucket/versioning/versioning_test.go @@ -112,15 +112,6 @@ func TestParseConfig(t *testing.T) { excludedPrefixes: []string{"path/to/my/workload/_staging/", "path/to/my/workload/_temporary/"}, excludeFolders: true, }, - { - input: ` - Enabled - - path/to/my/workload/_staging - - `, - err: errInvalidPrefixPattern, - }, } for i, tc := range testcases {