ilm: Handle DeleteAllVersions action differently for DEL markers (#19481)

i.e., this rule element doesn't apply to DEL markers.

This is a breaking change to how ExpiredObejctDeleteAllVersions
functions today. This is necessary to avoid the following highly probable
footgun scenario in the future.

Scenario:
The user uses tags-based filtering to select an object's time to live(TTL). 
The application sometimes deletes objects, too, making its latest
version a DEL marker. The previous implementation skipped tag-based filters
if the newest version was DEL marker, voiding the tag-based TTL. The user is
surprised to find objects that have expired sooner than expected.

* Add DelMarkerExpiration action

This ILM action removes all versions of an object if its
the latest version is a DEL marker.

```xml
<DelMarkerObjectExpiration>
    <Days> 10 </Days>
</DelMarkerObjectExpiration>
```

1. Applies only to objects whose,
  • The latest version is a DEL marker.
  • satisfies the number of days criteria
2. Deletes all versions of this object
3. Associated rule can't have tag-based filtering

Includes,
- New bucket event type for deletion due to DelMarkerExpiration

cmd/data-scanner.go

@@ -993,7 +993,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 	// This can happen when,
 	// - ExpireObjectAllVersions flag is enabled
 	// - NoncurrentVersionExpiration is applicable
-	case lifecycle.DeleteVersionAction, lifecycle.DeleteAllVersionsAction:
+	case lifecycle.DeleteVersionAction, lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
 		size = 0
 	case lifecycle.DeleteAction:
 		// On a non-versioned bucket, DeleteObject removes the only version permanently.
@@ -1162,7 +1162,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 
 	// Note: objDeleted is true if and only if action ==
 	// lifecycle.DeleteAllVersionsAction
-	if action == lifecycle.DeleteAllVersionsAction {
+	if action.DeleteAll() {
 		return true, 0
 	}
 
@@ -1292,7 +1292,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 
 		if lcEvent.Action != lifecycle.NoneAction {
 			numVersions := uint64(1)
-			if lcEvent.Action == lifecycle.DeleteAllVersionsAction {
+			if lcEvent.Action.DeleteAll() {
 				numVersions = uint64(obj.NumVersions)
 			}
 			globalScannerMetrics.timeILM(lcEvent.Action)(numVersions)
@@ -1320,8 +1320,11 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	if obj.DeleteMarker {
 		eventName = event.ObjectRemovedDeleteMarkerCreated
 	}
-	if lcEvent.Action.DeleteAll() {
+	switch lcEvent.Action {
+	case lifecycle.DeleteAllVersionsAction:
 		eventName = event.ObjectRemovedDeleteAllVersions
+	case lifecycle.DelMarkerDeleteAllVersionsAction:
+		eventName = event.ILMDelMarkerExpirationDelete
 	}
 	// Notify object deleted event.
 	sendEvent(eventArgs{
@@ -1346,7 +1349,7 @@ func applyLifecycleAction(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
 	switch action := event.Action; action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction,
 		lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction,
-		lifecycle.DeleteAllVersionsAction:
+		lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
 		success = applyExpiryRule(event, src, obj)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
 		success = applyTransitionRule(event, src, obj)

cmd/erasure-object.go

@@ -1886,12 +1886,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			// based on the latest objectInfo and see if the object still
 			// qualifies for deletion.
 			if gerr == nil {
-				evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 				var isErr bool
+				evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 				switch evt.Action {
-				case lifecycle.NoneAction:
+				case lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
-					isErr = true
+					// opts.DeletePrefix is used only in the above lifecycle Expiration actions.
-				case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+				default:
+					// object has been modified since lifecycle action was previously evaluated
 					isErr = true
 				}
 				if isErr {

internal/bucket/lifecycle/action_string.go

@@ -16,12 +16,13 @@ func _() {
 	_ = x[DeleteRestoredAction-5]
 	_ = x[DeleteRestoredVersionAction-6]
 	_ = x[DeleteAllVersionsAction-7]
-	_ = x[ActionCount-8]
+	_ = x[DelMarkerDeleteAllVersionsAction-8]
+	_ = x[ActionCount-9]
 }
 
-const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionActionDeleteAllVersionsActionActionCount"
+const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionActionDeleteAllVersionsActionDelMarkerDeleteAllVersionsActionActionCount"
 
-var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127, 150, 161}
+var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127, 150, 182, 193}
 
 func (i Action) String() string {
 	if i < 0 || i >= Action(len(_Action_index)-1) {

internal/bucket/lifecycle/delmarker-expiration.go

@@ -0,0 +1,74 @@
+// Copyright (c) 2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lifecycle
+
+import (
+	"encoding/xml"
+	"time"
+)
+
+var errInvalidDaysDelMarkerExpiration = Errorf("Days must be a positive integer with DelMarkerExpiration")
+
+// DelMarkerExpiration used to xml encode/decode ILM action by the same name
+type DelMarkerExpiration struct {
+	XMLName xml.Name `xml:"DelMarkerExpiration"`
+	Days    int      `xml:"Days,omitempty"`
+}
+
+// Empty returns if a DelMarkerExpiration XML element is empty.
+// Used to detect if lifecycle.Rule contained a DelMarkerExpiration element.
+func (de DelMarkerExpiration) Empty() bool {
+	return de.Days == 0
+}
+
+// UnmarshalXML decodes a single XML element into a DelMarkerExpiration value
+func (de *DelMarkerExpiration) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error {
+	type delMarkerExpiration DelMarkerExpiration
+	var dexp delMarkerExpiration
+	err := dec.DecodeElement(&dexp, &start)
+	if err != nil {
+		return err
+	}
+
+	if dexp.Days <= 0 {
+		return errInvalidDaysDelMarkerExpiration
+	}
+
+	*de = DelMarkerExpiration(dexp)
+	return nil
+}
+
+// MarshalXML encodes a DelMarkerExpiration value into an XML element
+func (de DelMarkerExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
+	if de.Empty() {
+		return nil
+	}
+
+	type delMarkerExpiration DelMarkerExpiration
+	return enc.EncodeElement(delMarkerExpiration(de), start)
+}
+
+// NextDue returns upcoming DelMarkerExpiration date for obj if
+// applicable, returns false otherwise.
+func (de DelMarkerExpiration) NextDue(obj ObjectOpts) (time.Time, bool) {
+	if !obj.IsLatest || !obj.DeleteMarker {
+		return time.Time{}, false
+	}
+
+	return ExpectedExpiryTime(obj.ModTime, de.Days), true
+}

internal/bucket/lifecycle/delmarker-expiration_test.go

@@ -0,0 +1,63 @@
+// Copyright (c) 2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lifecycle
+
+import (
+	"encoding/xml"
+	"fmt"
+	"testing"
+)
+
+func TestDelMarkerExpParseAndValidate(t *testing.T) {
+	tests := []struct {
+		xml string
+		err error
+	}{
+		{
+			xml: `<DelMarkerExpiration> <Days> 1 </Days> </DelMarkerExpiration>`,
+			err: nil,
+		},
+		{
+			xml: `<DelMarkerExpiration> <Days> -1 </Days> </DelMarkerExpiration>`,
+			err: errInvalidDaysDelMarkerExpiration,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("TestDelMarker-%d", i), func(t *testing.T) {
+			var dexp DelMarkerExpiration
+			var fail bool
+			err := xml.Unmarshal([]byte(test.xml), &dexp)
+			if test.err == nil {
+				if err != nil {
+					fail = true
+				}
+			} else {
+				if err == nil {
+					fail = true
+				}
+				if test.err.Error() != err.Error() {
+					fail = true
+				}
+			}
+			if fail {
+				t.Fatalf("Expected %v but got %v", test.err, err)
+			}
+		})
+	}
+}

internal/bucket/lifecycle/lifecycle.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -22,7 +22,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"sort"
+	"slices"
 	"strings"
 	"time"
 
@@ -67,7 +67,8 @@ const (
 	DeleteRestoredVersionAction
 	// DeleteAllVersionsAction deletes all versions when an object expires
 	DeleteAllVersionsAction
-
+	// DelMarkerDeleteAllVersionsAction deletes all versions when an object with delete marker as latest version expires
+	DelMarkerDeleteAllVersionsAction
 	// ActionCount must be the last action and shouldn't be used as a regular action.
 	ActionCount
 )
@@ -84,7 +85,7 @@ func (a Action) DeleteVersioned() bool {
 
 // DeleteAll - Returns true if the action demands deleting all versions of an object
 func (a Action) DeleteAll() bool {
-	return a == DeleteAllVersionsAction
+	return a == DeleteAllVersionsAction || a == DelMarkerDeleteAllVersionsAction
 }
 
 // Delete - Returns true if action demands delete on all objects (including restored)
@@ -92,7 +93,7 @@ func (a Action) Delete() bool {
 	if a.DeleteRestored() {
 		return true
 	}
-	return a == DeleteVersionAction || a == DeleteAction || a == DeleteAllVersionsAction
+	return a == DeleteVersionAction || a == DeleteAction || a == DeleteAllVersionsAction || a == DelMarkerDeleteAllVersionsAction
 }
 
 // Lifecycle - Configuration for bucket lifecycle.
@@ -279,7 +280,7 @@ func (lc Lifecycle) FilterRules(obj ObjectOpts) []Rule {
 		if !strings.HasPrefix(obj.Name, rule.GetPrefix()) {
 			continue
 		}
-		if !obj.DeleteMarker && !rule.Filter.TestTags(obj.UserTags) {
+		if !rule.Filter.TestTags(obj.UserTags) {
 			continue
 		}
 		if !obj.DeleteMarker && !rule.Filter.BySize(obj.Size) {
@@ -353,23 +354,6 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 	}
 
 	for _, rule := range lc.FilterRules(obj) {
-		if obj.IsLatest && rule.Expiration.DeleteAll.val {
-			if !rule.Expiration.IsDaysNull() {
-				// Specifying the Days tag will automatically perform all versions cleanup
-				// once the latest object is old enough to satisfy the age criteria.
-				// This is a MinIO only extension.
-				if expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)); now.IsZero() || now.After(expectedExpiry) {
-					events = append(events, Event{
-						Action: DeleteAllVersionsAction,
-						RuleID: rule.ID,
-						Due:    expectedExpiry,
-					})
-					// No other conflicting actions apply to an all version expired object.
-					break
-				}
-			}
-		}
-
 		if obj.ExpiredObjectDeleteMarker() {
 			if rule.Expiration.DeleteMarker.val {
 				// Indicates whether MinIO will remove a delete marker with no noncurrent versions.
@@ -401,6 +385,21 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 			}
 		}
 
+		// DelMarkerExpiration
+		if obj.IsLatest && obj.DeleteMarker && !rule.DelMarkerExpiration.Empty() {
+			if due, ok := rule.DelMarkerExpiration.NextDue(obj); ok && (now.IsZero() || now.After(due)) {
+				events = append(events, Event{
+					Action: DelMarkerDeleteAllVersionsAction,
+					RuleID: rule.ID,
+					Due:    due,
+				})
+			}
+			// No other conflicting actions in this rule can apply to an object with current version as DEL marker
+			// Note: There could be other rules with earlier expiration which need to be considered.
+			// See TestDelMarkerExpiration
+			continue
+		}
+
 		// Skip rules with newer noncurrent versions specified. These rules are
 		// not handled at an individual version level. eval applies only to a
 		// specific version.
@@ -448,11 +447,17 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 				}
 			case !rule.Expiration.IsDaysNull():
 				if expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)); now.IsZero() || now.After(expectedExpiry) {
-					events = append(events, Event{
+					event := Event{
 						Action: DeleteAction,
 						RuleID: rule.ID,
 						Due:    expectedExpiry,
-					})
+					}
+					if rule.Expiration.DeleteAll.val {
+						// Expires all versions of this object once the latest object is old enough.
+						// This is a MinIO only extension.
+						event.Action = DeleteAllVersionsAction
+					}
+					events = append(events, event)
 				}
 			}
 
@@ -470,25 +475,30 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 	}
 
 	if len(events) > 0 {
-		sort.Slice(events, func(i, j int) bool {
+		slices.SortFunc(events, func(a, b Event) int {
 			// Prefer Expiration over Transition for both current
 			// and noncurrent versions when,
 			// - now is past the expected time to action
 			// - expected time to action is the same for both actions
-			if now.After(events[i].Due) && now.After(events[j].Due) || events[i].Due.Equal(events[j].Due) {
+			if now.After(a.Due) && now.After(b.Due) || a.Due.Equal(b.Due) {
-				switch events[i].Action {
+				switch a.Action {
-				case DeleteAction, DeleteVersionAction:
+				case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction,
-					return true
+					DeleteAction, DeleteVersionAction:
+					return -1
 				}
-				switch events[j].Action {
+				switch b.Action {
-				case DeleteAction, DeleteVersionAction:
+				case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction,
-					return false
+					DeleteAction, DeleteVersionAction:
+					return 1
 				}
-				return true
+				return -1
 			}
 
 			// Prefer earlier occurring event
-			return events[i].Due.Before(events[j].Due)
+			if a.Due.Before(b.Due) {
+				return -1
+			}
+			return 1
 		})
 		return events[0]
 	}
@@ -517,7 +527,7 @@ func ExpectedExpiryTime(modTime time.Time, days int) time.Time {
 func (lc Lifecycle) SetPredictionHeaders(w http.ResponseWriter, obj ObjectOpts) {
 	event := lc.eval(obj, time.Time{})
 	switch event.Action {
-	case DeleteAction, DeleteVersionAction, DeleteAllVersionsAction:
+	case DeleteAction, DeleteVersionAction, DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction:
 		w.Header()[xhttp.AmzExpiration] = []string{
 			fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, event.Due.Format(http.TimeFormat), event.RuleID),
 		}

internal/bucket/lifecycle/lifecycle_test.go

@@ -115,10 +115,22 @@ func TestParseAndValidateLifecycleConfig(t *testing.T) {
 		},
 		// Lifecycle with max noncurrent versions
 		{
-			inputConfig:           `<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ID>rule</ID>><Status>Enabled</Status><Filter></Filter><NoncurrentVersionExpiration><NewerNoncurrentVersions>5</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>`,
+			inputConfig:           `<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ID>rule</ID><Status>Enabled</Status><Filter></Filter><NoncurrentVersionExpiration><NewerNoncurrentVersions>5</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>`,
 			expectedParsingErr:    nil,
 			expectedValidationErr: nil,
 		},
+		// Lifecycle with delmarker expiration
+		{
+			inputConfig:           `<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ID>rule</ID><Status>Enabled</Status><Filter></Filter><DelMarkerExpiration><Days>5</Days></DelMarkerExpiration></Rule></LifecycleConfiguration>`,
+			expectedParsingErr:    nil,
+			expectedValidationErr: nil,
+		},
+		// Lifecycle with empty delmarker expiration
+		{
+			inputConfig:           `<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ID>rule</ID><Status>Enabled</Status><Filter></Filter><DelMarkerExpiration><Days></Days></DelMarkerExpiration></Rule></LifecycleConfiguration>`,
+			expectedParsingErr:    errInvalidDaysDelMarkerExpiration,
+			expectedValidationErr: nil,
+		},
 	}
 
 	for i, tc := range testCases {
@@ -228,7 +240,8 @@ func TestEval(t *testing.T) {
 		objectName             string
 		objectTags             string
 		objectModTime          time.Time
-		isExpiredDelMarker     bool
+		isDelMarker            bool
+		hasManyVersions        bool
 		expectedAction         Action
 		isNoncurrent           bool
 		objectSuccessorModTime time.Time
@@ -383,36 +396,52 @@ func TestEval(t *testing.T) {
 		},
 		// Should delete expired delete marker right away
 		{
-			inputConfig:        `<BucketLifecycleConfiguration><Rule><Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration><Filter></Filter><Status>Enabled</Status></Rule></BucketLifecycleConfiguration>`,
+			inputConfig:    `<BucketLifecycleConfiguration><Rule><Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration><Filter></Filter><Status>Enabled</Status></Rule></BucketLifecycleConfiguration>`,
-			objectName:         "foodir/fooobject",
+			objectName:     "foodir/fooobject",
-			objectModTime:      time.Now().UTC().Add(-1 * time.Hour), // Created one hour ago
+			objectModTime:  time.Now().UTC().Add(-1 * time.Hour), // Created one hour ago
-			isExpiredDelMarker: true,
+			isDelMarker:    true,
-			expectedAction:     DeleteVersionAction,
+			expectedAction: DeleteVersionAction,
 		},
-		// Should delete expired object right away with 1 day expiration
+		// Should not expire a delete marker; ExpiredObjectDeleteAllVersions applies only when current version is not a DEL marker.
 		{
-			inputConfig:        `<BucketLifecycleConfiguration><Rule><Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration><Filter></Filter><Status>Enabled</Status></Rule></BucketLifecycleConfiguration>`,
+			inputConfig:     `<BucketLifecycleConfiguration><Rule><Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration><Filter></Filter><Status>Enabled</Status></Rule></BucketLifecycleConfiguration>`,
-			objectName:         "foodir/fooobject",
+			objectName:      "foodir/fooobject",
-			objectModTime:      time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			objectModTime:   time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
-			isExpiredDelMarker: true,
+			isDelMarker:     true,
-			expectedAction:     DeleteAllVersionsAction,
+			hasManyVersions: true,
+			expectedAction:  NoneAction,
+		},
+		// Should delete all versions of this object since the latest version has past the expiry days criteria
+		{
+			inputConfig:     `<BucketLifecycleConfiguration><Rule><Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration><Filter></Filter><Status>Enabled</Status></Rule></BucketLifecycleConfiguration>`,
+			objectName:      "foodir/fooobject",
+			objectModTime:   time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			hasManyVersions: true,
+			expectedAction:  DeleteAllVersionsAction,
+		},
+		// TransitionAction applies since object doesn't meet the age criteria for DeleteAllVersions
+		{
+			inputConfig:     `<BucketLifecycleConfiguration><Rule><Expiration><Days>30</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration><Transition><Days>10</Days><StorageClass>WARM-1</StorageClass></Transition><Filter></Filter><Status>Enabled</Status></Rule></BucketLifecycleConfiguration>`,
+			objectName:      "foodir/fooobject",
+			objectModTime:   time.Now().UTC().Add(-11 * 24 * time.Hour), // Created 11 days ago
+			hasManyVersions: true,
+			expectedAction:  TransitionAction,
 		},
-
 		// Should not delete expired marker if its time has not come yet
 		{
-			inputConfig:        `<BucketLifecycleConfiguration><Rule><Filter></Filter><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule></BucketLifecycleConfiguration>`,
+			inputConfig:    `<BucketLifecycleConfiguration><Rule><Filter></Filter><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule></BucketLifecycleConfiguration>`,
-			objectName:         "foodir/fooobject",
+			objectName:     "foodir/fooobject",
-			objectModTime:      time.Now().UTC().Add(-12 * time.Hour), // Created 12 hours ago
+			objectModTime:  time.Now().UTC().Add(-12 * time.Hour), // Created 12 hours ago
-			isExpiredDelMarker: true,
+			isDelMarker:    true,
-			expectedAction:     NoneAction,
+			expectedAction: NoneAction,
 		},
 		// Should delete expired marker since its time has come
 		{
-			inputConfig:        `<BucketLifecycleConfiguration><Rule><Filter></Filter><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule></BucketLifecycleConfiguration>`,
+			inputConfig:    `<BucketLifecycleConfiguration><Rule><Filter></Filter><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule></BucketLifecycleConfiguration>`,
-			objectName:         "foodir/fooobject",
+			objectName:     "foodir/fooobject",
-			objectModTime:      time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			objectModTime:  time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
-			isExpiredDelMarker: true,
+			isDelMarker:    true,
-			expectedAction:     DeleteVersionAction,
+			expectedAction: DeleteVersionAction,
 		},
 		// Should transition immediately when Transition days is zero
 		{
@@ -579,6 +608,82 @@ func TestEval(t *testing.T) {
 			objectSuccessorModTime: time.Now().UTC().Add(-90 * 24 * time.Hour),
 			expectedAction:         DeleteVersionAction,
 		},
+		{
+			// DelMarkerExpiration is preferred since object age is past both transition and expiration days.
+			inputConfig: `<LifecycleConfiguration>
+                            <Rule>
+                              <ID>DelMarkerExpiration with Transition</ID>
+                              <Filter></Filter>
+                              <Status>Enabled</Status>
+                              <DelMarkerExpiration>
+                                <Days>60</Days>
+                              </DelMarkerExpiration>
+	                      <Transition>
+                                <StorageClass>WARM-1</StorageClass>
+                                <Days>30</Days>
+                              </Transition>
+                             </Rule>
+                       </LifecycleConfiguration>`,
+			objectName:     "obj-1",
+			objectModTime:  time.Now().UTC().Add(-90 * 24 * time.Hour),
+			isDelMarker:    true,
+			expectedAction: DelMarkerDeleteAllVersionsAction,
+		},
+		{
+			// NoneAction since object doesn't qualify for DelMarkerExpiration yet.
+			// Note: TransitionAction doesn't apply to DEL marker
+			inputConfig: `<LifecycleConfiguration>
+                            <Rule>
+                              <ID>DelMarkerExpiration with Transition</ID>
+                              <Filter></Filter>
+                              <Status>Enabled</Status>
+                              <DelMarkerExpiration>
+                                <Days>60</Days>
+                              </DelMarkerExpiration>
+	                      <Transition>
+                                <StorageClass>WARM-1</StorageClass>
+                                <Days>30</Days>
+                              </Transition>
+                             </Rule>
+                       </LifecycleConfiguration>`,
+			objectName:     "obj-1",
+			objectModTime:  time.Now().UTC().Add(-50 * 24 * time.Hour),
+			isDelMarker:    true,
+			expectedAction: NoneAction,
+		},
+		{
+			inputConfig: `<LifecycleConfiguration>
+                            <Rule>
+                              <ID>DelMarkerExpiration with non DEL-marker object</ID>
+                              <Filter></Filter>
+                              <Status>Enabled</Status>
+                              <DelMarkerExpiration>
+                                <Days>60</Days>
+                              </DelMarkerExpiration>
+                             </Rule>
+                       </LifecycleConfiguration>`,
+			objectName:     "obj-1",
+			objectModTime:  time.Now().UTC().Add(-90 * 24 * time.Hour),
+			expectedAction: NoneAction,
+		},
+		{
+			inputConfig: `<LifecycleConfiguration>
+                            <Rule>
+                              <ID>DelMarkerExpiration with noncurrent DEL-marker</ID>
+                              <Filter></Filter>
+                              <Status>Enabled</Status>
+                              <DelMarkerExpiration>
+                                <Days>60</Days>
+                              </DelMarkerExpiration>
+                             </Rule>
+                       </LifecycleConfiguration>`,
+			objectName:             "obj-1",
+			objectModTime:          time.Now().UTC().Add(-90 * 24 * time.Hour),
+			objectSuccessorModTime: time.Now().UTC().Add(-60 * 24 * time.Hour),
+			isDelMarker:            true,
+			isNoncurrent:           true,
+			expectedAction:         NoneAction,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -588,16 +693,20 @@ func TestEval(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Got unexpected error: %v", err)
 			}
-			if res := lc.Eval(ObjectOpts{
+			opts := ObjectOpts{
 				Name:             tc.objectName,
 				UserTags:         tc.objectTags,
 				ModTime:          tc.objectModTime,
-				DeleteMarker:     tc.isExpiredDelMarker,
+				DeleteMarker:     tc.isDelMarker,
-				NumVersions:      1,
 				IsLatest:         !tc.isNoncurrent,
 				SuccessorModTime: tc.objectSuccessorModTime,
 				VersionID:        tc.versionID,
-			}); res.Action != tc.expectedAction {
+			}
+			opts.NumVersions = 1
+			if tc.hasManyVersions {
+				opts.NumVersions = 2 // at least one noncurrent version
+			}
+			if res := lc.Eval(opts); res.Action != tc.expectedAction {
 				t.Fatalf("Expected action: `%v`, got: `%v`", tc.expectedAction, res.Action)
 			}
 		})
@@ -1160,7 +1269,7 @@ func TestFilterRules(t *testing.T) {
 		opts     ObjectOpts
 		hasRules bool
 	}{
-		{ // Delete marker should match filter without tags
+		{ // Delete marker shouldn't match filter without tags
 			lc: Lifecycle{
 				Rules: []Rule{
 					rules[0],
@@ -1171,7 +1280,7 @@ func TestFilterRules(t *testing.T) {
 				IsLatest:     true,
 				Name:         "obj-1",
 			},
-			hasRules: true,
+			hasRules: false,
 		},
 		{ // PUT version with no matching tags
 			lc: Lifecycle{
@@ -1269,3 +1378,86 @@ func TestFilterRules(t *testing.T) {
 		})
 	}
 }
+
+// TestDeleteAllVersions tests ordering among events, especially ones which
+// expire all versions like ExpiredObjectDeleteAllVersions and
+// DelMarkerExpiration
+func TestDeleteAllVersions(t *testing.T) {
+	// ExpiredObjectDeleteAllVersions
+	lc := Lifecycle{
+		Rules: []Rule{
+			{
+				ID:     "ExpiredObjectDeleteAllVersions-20",
+				Status: "Enabled",
+				Expiration: Expiration{
+					set:       true,
+					DeleteAll: Boolean{val: true, set: true},
+					Days:      20,
+				},
+			},
+			{
+				ID:     "Transition-10",
+				Status: "Enabled",
+				Transition: Transition{
+					set:          true,
+					StorageClass: "WARM-1",
+					Days:         10,
+				},
+			},
+		},
+	}
+	opts := ObjectOpts{
+		Name:        "foo.txt",
+		ModTime:     time.Now().UTC().Add(-10 * 24 * time.Hour), // created 10 days ago
+		Size:        0,
+		VersionID:   uuid.New().String(),
+		IsLatest:    true,
+		NumVersions: 4,
+	}
+
+	event := lc.eval(opts, time.Time{})
+	if event.Action != TransitionAction {
+		t.Fatalf("Expected %v action but got %v", TransitionAction, event.Action)
+	}
+	// The earlier upcoming lifecycle event must be picked, i.e rule with id "Transition-10"
+	if exp := ExpectedExpiryTime(opts.ModTime, 10); exp != event.Due {
+		t.Fatalf("Expected due %v but got %v, ruleID=%v", exp, event.Due, event.RuleID)
+	}
+
+	// DelMarkerExpiration
+	lc = Lifecycle{
+		Rules: []Rule{
+			{
+				ID:     "delmarker-exp-20",
+				Status: "Enabled",
+				DelMarkerExpiration: DelMarkerExpiration{
+					Days: 20,
+				},
+			},
+			{
+				ID:     "delmarker-exp-10",
+				Status: "Enabled",
+				DelMarkerExpiration: DelMarkerExpiration{
+					Days: 10,
+				},
+			},
+		},
+	}
+	opts = ObjectOpts{
+		Name:         "foo.txt",
+		ModTime:      time.Now().UTC().Add(-10 * 24 * time.Hour), // created 10 days ago
+		Size:         0,
+		VersionID:    uuid.New().String(),
+		IsLatest:     true,
+		DeleteMarker: true,
+		NumVersions:  4,
+	}
+	event = lc.eval(opts, time.Time{})
+	if event.Action != DelMarkerDeleteAllVersionsAction {
+		t.Fatalf("Expected %v action but got %v", DelMarkerDeleteAllVersionsAction, event.Action)
+	}
+	// The earlier upcoming lifecycle event must be picked, i.e rule with id "delmarker-exp-10"
+	if exp := ExpectedExpiryTime(opts.ModTime, 10); exp != event.Due {
+		t.Fatalf("Expected due %v but got %v, ruleID=%v", exp, event.Due, event.RuleID)
+	}
+}

internal/bucket/lifecycle/rule.go

@@ -33,22 +33,24 @@ const (
 
 // Rule - a rule for lifecycle configuration.
 type Rule struct {
-	XMLName    xml.Name   `xml:"Rule"`
+	XMLName             xml.Name            `xml:"Rule"`
-	ID         string     `xml:"ID,omitempty"`
+	ID                  string              `xml:"ID,omitempty"`
-	Status     Status     `xml:"Status"`
+	Status              Status              `xml:"Status"`
-	Filter     Filter     `xml:"Filter,omitempty"`
+	Filter              Filter              `xml:"Filter,omitempty"`
-	Prefix     Prefix     `xml:"Prefix,omitempty"`
+	Prefix              Prefix              `xml:"Prefix,omitempty"`
-	Expiration Expiration `xml:"Expiration,omitempty"`
+	Expiration          Expiration          `xml:"Expiration,omitempty"`
-	Transition Transition `xml:"Transition,omitempty"`
+	Transition          Transition          `xml:"Transition,omitempty"`
+	DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty"`
 	// FIXME: add a type to catch unsupported AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
 	NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
 	NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
 }
 
 var (
-	errInvalidRuleID     = Errorf("ID length is limited to 255 characters")
+	errInvalidRuleID                  = Errorf("ID length is limited to 255 characters")
-	errEmptyRuleStatus   = Errorf("Status should not be empty")
+	errEmptyRuleStatus                = Errorf("Status should not be empty")
-	errInvalidRuleStatus = Errorf("Status must be set to either Enabled or Disabled")
+	errInvalidRuleStatus              = Errorf("Status must be set to either Enabled or Disabled")
+	errInvalidRuleDelMarkerExpiration = Errorf("Rule with DelMarkerExpiration cannot have tags based filtering")
 )
 
 // validateID - checks if ID is valid or not.
@@ -158,7 +160,10 @@ func (r Rule) Validate() error {
 	if err := r.validateNoncurrentTransition(); err != nil {
 		return err
 	}
-	if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set {
+	if (!r.Filter.Tag.IsEmpty() || len(r.Filter.And.Tags) != 0) && !r.DelMarkerExpiration.Empty() {
+		return errInvalidRuleDelMarkerExpiration
+	}
+	if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set && r.DelMarkerExpiration.Empty() {
 		return errXMLNotWellFormed
 	}
 	return nil

internal/bucket/lifecycle/rule_test.go

@@ -105,6 +105,31 @@ func TestInvalidRules(t *testing.T) {
 	                    </Rule>`,
 			expectedErr: errXMLNotWellFormed,
 		},
+		{
+			inputXML: `<Rule>
+				<ID>Rule with a tag and DelMarkerExpiration</ID>
+				<Filter><Tag><Key>k1</Key><Value>v1</Value></Tag></Filter>
+				<DelMarkerExpiration>
+					<Days>365</Days>
+				</DelMarkerExpiration>
+                            <Status>Enabled</Status>
+	                    </Rule>`,
+			expectedErr: errInvalidRuleDelMarkerExpiration,
+		},
+		{
+			inputXML: `<Rule>
+				<ID>Rule with multiple tags and DelMarkerExpiration</ID>
+				<Filter><And>
+				<Tag><Key>k1</Key><Value>v1</Value></Tag>
+				<Tag><Key>k2</Key><Value>v2</Value></Tag>
+				</And></Filter>
+				<DelMarkerExpiration>
+					<Days>365</Days>
+				</DelMarkerExpiration>
+                            <Status>Enabled</Status>
+	                    </Rule>`,
+			expectedErr: errInvalidRuleDelMarkerExpiration,
+		},
 	}
 
 	for i, tc := range invalidTestCases {

internal/event/name.go

@@ -63,6 +63,7 @@ const (
 	ObjectManyVersions
 	ObjectLargeVersions
 	PrefixManyFolders
+	ILMDelMarkerExpirationDelete
 
 	objectSingleTypesEnd
 	// Start Compound types that require expansion:
@@ -199,6 +200,8 @@ func (name Name) String() string {
 		return "s3:ObjectRemoved:NoOP"
 	case ObjectRemovedDeleteAllVersions:
 		return "s3:ObjectRemoved:DeleteAllVersions"
+	case ILMDelMarkerExpirationDelete:
+		return "s3:LifecycleDelMarkerExpiration:Delete"
 	case ObjectReplicationAll:
 		return "s3:Replication:*"
 	case ObjectReplicationFailed:
@@ -324,6 +327,8 @@ func ParseName(s string) (Name, error) {
 		return ObjectRemovedNoOP, nil
 	case "s3:ObjectRemoved:DeleteAllVersions":
 		return ObjectRemovedDeleteAllVersions, nil
+	case "s3:LifecycleDelMarkerExpiration:Delete":
+		return ILMDelMarkerExpirationDelete, nil
 	case "s3:Replication:*":
 		return ObjectReplicationAll, nil
 	case "s3:Replication:OperationFailedReplication":

internal/event/name_test.go

@@ -68,6 +68,8 @@ func TestNameString(t *testing.T) {
 		{ObjectCreatedPut, "s3:ObjectCreated:Put"},
 		{ObjectRemovedAll, "s3:ObjectRemoved:*"},
 		{ObjectRemovedDelete, "s3:ObjectRemoved:Delete"},
+		{ObjectRemovedDeleteAllVersions, "s3:ObjectRemoved:DeleteAllVersions"},
+		{ILMDelMarkerExpirationDelete, "s3:LifecycleDelMarkerExpiration:Delete"},
 		{ObjectRemovedNoOP, "s3:ObjectRemoved:NoOP"},
 		{ObjectCreatedPutRetention, "s3:ObjectCreated:PutRetention"},
 		{ObjectCreatedPutLegalHold, "s3:ObjectCreated:PutLegalHold"},
@@ -219,6 +221,7 @@ func TestParseName(t *testing.T) {
 		{"s3:ObjectAccessed:*", ObjectAccessedAll, false},
 		{"s3:ObjectRemoved:Delete", ObjectRemovedDelete, false},
 		{"s3:ObjectRemoved:NoOP", ObjectRemovedNoOP, false},
+		{"s3:LifecycleDelMarkerExpiration:Delete", ILMDelMarkerExpirationDelete, false},
 		{"", blankName, true},
 	}