ilm: Handle DeleteAllVersions action differently for DEL markers (#19481)

i.e., this rule element doesn't apply to DEL markers.

This is a breaking change to how ExpiredObejctDeleteAllVersions
functions today. This is necessary to avoid the following highly probable
footgun scenario in the future.

Scenario:
The user uses tags-based filtering to select an object's time to live(TTL). 
The application sometimes deletes objects, too, making its latest
version a DEL marker. The previous implementation skipped tag-based filters
if the newest version was DEL marker, voiding the tag-based TTL. The user is
surprised to find objects that have expired sooner than expected.

* Add DelMarkerExpiration action

This ILM action removes all versions of an object if its
the latest version is a DEL marker.

```xml
<DelMarkerObjectExpiration>
    <Days> 10 </Days>
</DelMarkerObjectExpiration>
```

1. Applies only to objects whose,
  • The latest version is a DEL marker.
  • satisfies the number of days criteria
2. Deletes all versions of this object
3. Associated rule can't have tag-based filtering

Includes,
- New bucket event type for deletion due to DelMarkerExpiration

internal/bucket/lifecycle/lifecycle.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -22,7 +22,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"sort"
+	"slices"
 	"strings"
 	"time"
 
@@ -67,7 +67,8 @@ const (
 	DeleteRestoredVersionAction
 	// DeleteAllVersionsAction deletes all versions when an object expires
 	DeleteAllVersionsAction
-
+	// DelMarkerDeleteAllVersionsAction deletes all versions when an object with delete marker as latest version expires
+	DelMarkerDeleteAllVersionsAction
 	// ActionCount must be the last action and shouldn't be used as a regular action.
 	ActionCount
 )
@@ -84,7 +85,7 @@ func (a Action) DeleteVersioned() bool {
 
 // DeleteAll - Returns true if the action demands deleting all versions of an object
 func (a Action) DeleteAll() bool {
-	return a == DeleteAllVersionsAction
+	return a == DeleteAllVersionsAction || a == DelMarkerDeleteAllVersionsAction
 }
 
 // Delete - Returns true if action demands delete on all objects (including restored)
@@ -92,7 +93,7 @@ func (a Action) Delete() bool {
 	if a.DeleteRestored() {
 		return true
 	}
-	return a == DeleteVersionAction || a == DeleteAction || a == DeleteAllVersionsAction
+	return a == DeleteVersionAction || a == DeleteAction || a == DeleteAllVersionsAction || a == DelMarkerDeleteAllVersionsAction
 }
 
 // Lifecycle - Configuration for bucket lifecycle.
@@ -279,7 +280,7 @@ func (lc Lifecycle) FilterRules(obj ObjectOpts) []Rule {
 		if !strings.HasPrefix(obj.Name, rule.GetPrefix()) {
 			continue
 		}
-		if !obj.DeleteMarker && !rule.Filter.TestTags(obj.UserTags) {
+		if !rule.Filter.TestTags(obj.UserTags) {
 			continue
 		}
 		if !obj.DeleteMarker && !rule.Filter.BySize(obj.Size) {
@@ -353,23 +354,6 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 	}
 
 	for _, rule := range lc.FilterRules(obj) {
-		if obj.IsLatest && rule.Expiration.DeleteAll.val {
-			if !rule.Expiration.IsDaysNull() {
-				// Specifying the Days tag will automatically perform all versions cleanup
-				// once the latest object is old enough to satisfy the age criteria.
-				// This is a MinIO only extension.
-				if expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)); now.IsZero() || now.After(expectedExpiry) {
-					events = append(events, Event{
-						Action: DeleteAllVersionsAction,
-						RuleID: rule.ID,
-						Due:    expectedExpiry,
-					})
-					// No other conflicting actions apply to an all version expired object.
-					break
-				}
-			}
-		}
-
 		if obj.ExpiredObjectDeleteMarker() {
 			if rule.Expiration.DeleteMarker.val {
 				// Indicates whether MinIO will remove a delete marker with no noncurrent versions.
@@ -401,6 +385,21 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 			}
 		}
 
+		// DelMarkerExpiration
+		if obj.IsLatest && obj.DeleteMarker && !rule.DelMarkerExpiration.Empty() {
+			if due, ok := rule.DelMarkerExpiration.NextDue(obj); ok && (now.IsZero() || now.After(due)) {
+				events = append(events, Event{
+					Action: DelMarkerDeleteAllVersionsAction,
+					RuleID: rule.ID,
+					Due:    due,
+				})
+			}
+			// No other conflicting actions in this rule can apply to an object with current version as DEL marker
+			// Note: There could be other rules with earlier expiration which need to be considered.
+			// See TestDelMarkerExpiration
+			continue
+		}
+
 		// Skip rules with newer noncurrent versions specified. These rules are
 		// not handled at an individual version level. eval applies only to a
 		// specific version.
@@ -448,11 +447,17 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 				}
 			case !rule.Expiration.IsDaysNull():
 				if expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)); now.IsZero() || now.After(expectedExpiry) {
-					events = append(events, Event{
+					event := Event{
 						Action: DeleteAction,
 						RuleID: rule.ID,
 						Due:    expectedExpiry,
-					})
+					}
+					if rule.Expiration.DeleteAll.val {
+						// Expires all versions of this object once the latest object is old enough.
+						// This is a MinIO only extension.
+						event.Action = DeleteAllVersionsAction
+					}
+					events = append(events, event)
 				}
 			}
 
@@ -470,25 +475,30 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 	}
 
 	if len(events) > 0 {
-		sort.Slice(events, func(i, j int) bool {
+		slices.SortFunc(events, func(a, b Event) int {
 			// Prefer Expiration over Transition for both current
 			// and noncurrent versions when,
 			// - now is past the expected time to action
 			// - expected time to action is the same for both actions
-			if now.After(events[i].Due) && now.After(events[j].Due) || events[i].Due.Equal(events[j].Due) {
-				switch events[i].Action {
-				case DeleteAction, DeleteVersionAction:
-					return true
+			if now.After(a.Due) && now.After(b.Due) || a.Due.Equal(b.Due) {
+				switch a.Action {
+				case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction,
+					DeleteAction, DeleteVersionAction:
+					return -1
 				}
-				switch events[j].Action {
-				case DeleteAction, DeleteVersionAction:
-					return false
+				switch b.Action {
+				case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction,
+					DeleteAction, DeleteVersionAction:
+					return 1
 				}
-				return true
+				return -1
 			}
 
 			// Prefer earlier occurring event
-			return events[i].Due.Before(events[j].Due)
+			if a.Due.Before(b.Due) {
+				return -1
+			}
+			return 1
 		})
 		return events[0]
 	}
@@ -517,7 +527,7 @@ func ExpectedExpiryTime(modTime time.Time, days int) time.Time {
 func (lc Lifecycle) SetPredictionHeaders(w http.ResponseWriter, obj ObjectOpts) {
 	event := lc.eval(obj, time.Time{})
 	switch event.Action {
-	case DeleteAction, DeleteVersionAction, DeleteAllVersionsAction:
+	case DeleteAction, DeleteVersionAction, DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction:
 		w.Header()[xhttp.AmzExpiration] = []string{
 			fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, event.Due.Format(http.TimeFormat), event.RuleID),
 		}