feat: allow expiration of all versions via ILM Expiration action (#17521)

Following extension allows users to specify immediate purge of
all versions as soon as the latest version of this object has
expired.

```
<LifecycleConfiguration>
    <Rule>
        <ID>ClassADocRule</ID>
        <Filter>
           <Prefix>classA/</Prefix>
        </Filter>
        <Status>Enabled</Status>
        <Expiration>
             <Days>3650</Days>
	     <ExpiredObjectAllVersions>true</ExpiredObjectAllVersions>
        </Expiration>
    </Rule>
    ...
```

cmd/data-scanner.go

@@ -1187,6 +1187,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
 		opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
 	}
+	if lcEvent.Action.DeleteAll() {
+		opts.DeletePrefix = true
+	}
 
 	obj, err := objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts)
 	if err != nil {

cmd/erasure-server-pool.go

@@ -993,13 +993,10 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return objInfo, err
 	}
 
-	if opts.DeletePrefix {
-		err := z.deletePrefix(ctx, bucket, object)
-		return ObjectInfo{}, err
+	if !opts.DeletePrefix { // DeletePrefix handles dir object encoding differently.
+		object = encodeDirObject(object)
 	}
 
-	object = encodeDirObject(object)
-
 	// Acquire a write lock before deleting the object.
 	lk := z.NewNSLock(bucket, object)
 	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
@@ -1009,6 +1006,10 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	ctx = lkctx.Context()
 	defer lk.Unlock(lkctx)
 
+	if opts.DeletePrefix {
+		return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
+	}
+
 	gopts := opts
 	gopts.NoLock = true
 	pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts)

docs/bucket/lifecycle/README.md

@@ -129,9 +129,31 @@ of objects under the prefix `user-uploads/` as soon as there are more than `N` n
     ]
 }
 ```
-
 Note: This rule has an implicit zero NoncurrentDays, which makes the expiry of those 'extra' noncurrent versions immediate.
 
+#### 3.2.b Automatic removal of all versions (MinIO only extension)
+
+This is available only on MinIO as an extension to the Expiration feature. The following rule makes it possible to remove all versions of an object under 
+the prefix `user-uploads/` as soon as the latest object satisfies the expiration criteria.
+
+```
+{
+    "Rules": [
+        {
+            "ID": "Purge all versions of an expired object",
+            "Status": "Enabled",
+            "Filter": {
+                "Prefix": "users-uploads/"
+            },
+            "Expiration": {
+                "Days": 7,
+                "ExpiredObjectAllVersions": true
+            }
+        }
+    ]
+}
+```
+
 ### 3.3 Automatic removal of delete markers with no other versions
 
 When an object has only one version as a delete marker, the latter can be automatically removed after a certain number of days using the following configuration:

docs/debugging/reorder-disks/main.go

@@ -162,7 +162,6 @@ func getDiskLocation(f format) (string, error) {
 }
 
 func main() {
-
 	var node, args string
 
 	flag.StringVar(&node, "local-node-name", "", "the name of the local node")

internal/bucket/lifecycle/action_string.go

@@ -15,12 +15,13 @@ func _() {
 	_ = x[TransitionVersionAction-4]
 	_ = x[DeleteRestoredAction-5]
 	_ = x[DeleteRestoredVersionAction-6]
-	_ = x[ActionCount-7]
+	_ = x[DeleteAllVersionsAction-7]
+	_ = x[ActionCount-8]
 }
 
-const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionActionActionCount"
+const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionActionDeleteAllVersionsActionActionCount"
 
-var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127, 138}
+var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127, 150, 161}
 
 func (i Action) String() string {
 	if i < 0 || i >= Action(len(_Action_index)-1) {

internal/bucket/lifecycle/expiration.go

@@ -100,6 +100,11 @@ func (eDate ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartEle
 
 // ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element.
 type ExpireDeleteMarker struct {
+	Boolean
+}
+
+// Boolean signifies a boolean XML struct with custom marshaling
+type Boolean struct {
 	val bool
 	set bool
 }
@@ -110,12 +115,16 @@ type Expiration struct {
 	Days         ExpirationDays     `xml:"Days,omitempty"`
 	Date         ExpirationDate     `xml:"Date,omitempty"`
 	DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker"`
+	// Indicates whether MinIO will remove all versions. If set to true, all versions will be deleted;
+	// if set to false the policy takes no action. This action uses the Days/Date to expire objects.
+	// This check is verified for latest version of the object.
+	DeleteAll Boolean `xml:"ExpiredObjectAllVersions"`
 
 	set bool
 }
 
 // MarshalXML encodes delete marker boolean into an XML form.
-func (b ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+func (b Boolean) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
 	if !b.set {
 		return nil
 	}
@@ -123,7 +132,7 @@ func (b ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartEle
 }
 
 // UnmarshalXML decodes delete marker boolean from the XML form.
-func (b *ExpireDeleteMarker) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
+func (b *Boolean) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
 	var exp bool
 	err := d.DecodeElement(&exp, &startElement)
 	if err != nil {

internal/bucket/lifecycle/lifecycle.go

@@ -65,6 +65,8 @@ const (
 	DeleteRestoredAction
 	// DeleteRestoredVersionAction deletes a particular version that was temporarily restored
 	DeleteRestoredVersionAction
+	// DeleteAllVersionsAction deletes all versions when an object expires
+	DeleteAllVersionsAction
 
 	// ActionCount must be the last action and shouldn't be used as a regular action.
 	ActionCount
@@ -80,6 +82,11 @@ func (a Action) DeleteVersioned() bool {
 	return a == DeleteVersionAction || a == DeleteRestoredVersionAction
 }
 
+// DeleteAll - Returns true if the action demands deleting all versions of an object
+func (a Action) DeleteAll() bool {
+	return a == DeleteAllVersionsAction
+}
+
 // Delete - Returns true if action demands delete on all objects (including restored)
 func (a Action) Delete() bool {
 	if a.DeleteRestored() {
@@ -324,6 +331,23 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 	}
 
 	for _, rule := range lc.FilterRules(obj) {
+		if obj.IsLatest && rule.Expiration.DeleteAll.val {
+			if !rule.Expiration.IsDaysNull() {
+				// Specifying the Days tag will automatically perform all versions cleanup
+				// once the latest object is old enough to satisfy the age criteria.
+				// This is a MinIO only extension.
+				if expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)); now.IsZero() || now.After(expectedExpiry) {
+					events = append(events, Event{
+						Action: DeleteAllVersionsAction,
+						RuleID: rule.ID,
+						Due:    expectedExpiry,
+					})
+					// No other conflicting actions apply to an all version expired object.
+					break
+				}
+			}
+		}
+
 		if obj.ExpiredObjectDeleteMarker() {
 			if rule.Expiration.DeleteMarker.val {
 				// Indicates whether MinIO will remove a delete marker with no noncurrent versions.

internal/bucket/lifecycle/lifecycle_test.go

@@ -388,6 +388,15 @@ func TestEval(t *testing.T) {
 			isExpiredDelMarker: true,
 			expectedAction:     DeleteVersionAction,
 		},
+		// Should delete expired object right away with 1 day expiration
+		{
+			inputConfig:        `<BucketLifecycleConfiguration><Rule><Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration><Filter></Filter><Status>Enabled</Status></Rule></BucketLifecycleConfiguration>`,
+			objectName:         "foodir/fooobject",
+			objectModTime:      time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			isExpiredDelMarker: true,
+			expectedAction:     DeleteAllVersionsAction,
+		},
+
 		// Should not delete expired marker if its time has not come yet
 		{
 			inputConfig:        `<BucketLifecycleConfiguration><Rule><Filter></Filter><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule></BucketLifecycleConfiguration>`,