lifecycle: refactor rules filtering and tagging support (#15914)

cmd/bucket-lifecycle.go

@@ -155,9 +155,14 @@ type newerNoncurrentTask struct {
 	versions []ObjectToDelete
 }
 
+type transitionTask struct {
+	tier    string
+	objInfo ObjectInfo
+}
+
 type transitionState struct {
 	once         sync.Once
-	transitionCh chan ObjectInfo
+	transitionCh chan transitionTask
 
 	ctx        context.Context
 	objAPI     ObjectLayer
@@ -171,13 +176,13 @@ type transitionState struct {
 	lastDayStats map[string]*lastDayTierStats
 }
 
-func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
+func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
 	select {
 	case <-GlobalContext.Done():
 		t.once.Do(func() {
 			close(t.transitionCh)
 		})
-	case t.transitionCh <- oi:
+	case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
 	default:
 	}
 }
@@ -186,7 +191,7 @@ var globalTransitionState *transitionState
 
 func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionState {
 	return &transitionState{
-		transitionCh: make(chan ObjectInfo, 10000),
+		transitionCh: make(chan transitionTask, 10000),
 		ctx:          ctx,
 		objAPI:       objAPI,
 		killCh:       make(chan struct{}),
@@ -213,27 +218,25 @@ func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
 			return
 		case <-ctx.Done():
 			return
-		case oi, ok := <-t.transitionCh:
+		case task, ok := <-t.transitionCh:
 			if !ok {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			var tier string
-			var err error
-			if tier, err = transitionObject(ctx, objectAPI, oi); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
+			if err := transitionObject(ctx, objectAPI, task.objInfo, task.tier); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
+					task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 			} else {
 				ts := tierStats{
-					TotalSize:   uint64(oi.Size),
+					TotalSize:   uint64(task.objInfo.Size),
 					NumVersions: 1,
 				}
-				if oi.IsLatest {
+				if task.objInfo.IsLatest {
 					ts.NumObjects = 1
 				}
-				t.addLastDayStats(tier, ts)
+				t.addLastDayStats(task.tier, ts)
 			}
 			atomic.AddInt32(&t.activeTasks, -1)
-
 		}
 	}
 }
@@ -304,9 +307,10 @@ func validateTransitionTier(lc *lifecycle.Lifecycle) error {
 // This is to be called after a successful upload of an object (version).
 func enqueueTransitionImmediate(obj ObjectInfo) {
 	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
-		switch lc.ComputeAction(obj.ToLifecycleOpts()) {
+		event := lc.Eval(obj.ToLifecycleOpts(), time.Now())
+		switch event.Action {
 		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			globalTransitionState.queueTransitionTask(obj)
+			globalTransitionState.queueTransitionTask(obj, event.StorageClass)
 		}
 	}
 }
@@ -403,12 +407,7 @@ func genTransitionObjName(bucket string) (string, error) {
 // storage specified by the transition ARN, the metadata is left behind on source cluster and original content
 // is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
 // to the transition tier without decrypting or re-encrypting.
-func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo) (string, error) {
-	lc, err := globalLifecycleSys.Get(oi.Bucket)
-	if err != nil {
-		return "", err
-	}
-	tier := lc.TransitionTier(oi.ToLifecycleOpts())
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, tier string) error {
 	opts := ObjectOptions{
 		Transition: TransitionOptions{
 			Status: lifecycle.TransitionPending,
@@ -420,7 +419,7 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
 		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
 		MTime:            oi.ModTime,
 	}
-	return tier, objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
+	return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
 }
 
 type auditTierOp struct {

cmd/data-scanner.go

@@ -963,21 +963,21 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 
 	versionID := oi.VersionID
 	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
-	action := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
+	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
 	if i.debug {
 		if versionID != "" {
-			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
+			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
 		} else {
-			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
+			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
 		}
 	}
-	defer globalScannerMetrics.timeILM(action)
+	defer globalScannerMetrics.timeILM(lcEvt.Action)
 
-	switch action {
+	switch lcEvt.Action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-		return applyLifecycleAction(action, oi), 0
+		return applyLifecycleAction(lcEvt.Action, oi, ""), 0
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		return applyLifecycleAction(action, oi), size
+		return applyLifecycleAction(lcEvt.Action, oi, lcEvt.StorageClass), size
 	default:
 		// No action.
 		return false, size
@@ -1109,21 +1109,21 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 	return size
 }
 
-func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) (action lifecycle.Action) {
-	action = lc.ComputeAction(obj.ToLifecycleOpts())
+func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
+	event := lc.Eval(obj.ToLifecycleOpts(), time.Now().UTC())
 	if serverDebugLog {
-		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
+		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
 	}
 
-	if action == lifecycle.NoneAction {
-		return action
+	if event.Action == lifecycle.NoneAction {
+		return event
 	}
 
-	switch action {
+	switch event.Action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
 		// Defensive code, should never happen
 		if obj.VersionID == "" {
-			return lifecycle.NoneAction
+			return lifecycle.Event{Action: lifecycle.NoneAction}
 		}
 		if lr.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 			if serverDebugLog {
@@ -1133,18 +1133,18 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 					console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
 				}
 			}
-			return lifecycle.NoneAction
+			return lifecycle.Event{Action: lifecycle.NoneAction}
 		}
 	}
 
-	return action
+	return event
 }
 
-func applyTransitionRule(obj ObjectInfo) bool {
+func applyTransitionRule(obj ObjectInfo, storageClass string) bool {
 	if obj.DeleteMarker {
 		return false
 	}
-	globalTransitionState.queueTransitionTask(obj)
+	globalTransitionState.queueTransitionTask(obj, storageClass)
 	return true
 }
 
@@ -1213,14 +1213,14 @@ func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
 }
 
 // Perform actions (removal or transitioning of objects), return true the action is successfully performed
-func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo) (success bool) {
+func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo, storageClass string) (success bool) {
 	switch action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
 		success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
 	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 		success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		success = applyTransitionRule(obj)
+		success = applyTransitionRule(obj, storageClass)
 	}
 	return
 }

cmd/erasure-object.go

@@ -1530,9 +1530,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	if opts.Expiration.Expire {
 		goi, _, err := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
 		if err == nil {
-			action := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
+			evt := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
 			var isErr bool
-			switch action {
+			switch evt.Action {
 			case lifecycle.NoneAction:
 				isErr = true
 			case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:

cmd/erasure-server-pool-decom.go

@@ -701,14 +701,14 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			}
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
-			action := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
-			switch action {
+			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			switch evt.Action {
 			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
+				globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
 				// Skip this entry.
 				return true
 			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-				globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
+				globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
 				// Skip this entry.
 				return true
 			}

cmd/erasure-server-pool.go

@@ -1302,8 +1302,8 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
 			if opts.Lifecycle != nil {
-				action := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
-				switch action {
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
+				switch evt.Action {
 				case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
 					fallthrough
 				case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:

cmd/erasure-single-drive.go

@@ -1477,9 +1477,9 @@ func (es *erasureSingle) DeleteObject(ctx context.Context, bucket, object string
 	}
 
 	if opts.Expiration.Expire {
-		action := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
+		evt := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
 		var isErr bool
-		switch action {
+		switch evt.Action {
 		case lifecycle.NoneAction:
 			isErr = true
 		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
@@ -2962,8 +2962,8 @@ func (es *erasureSingle) ListObjects(ctx context.Context, bucket, prefix, marker
 		objInfo, err := es.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
 			if opts.Lifecycle != nil {
-				action := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
-				switch action {
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
+				switch evt.Action {
 				case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
 					fallthrough
 				case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:

cmd/metacache-server-pool.go

@@ -627,14 +627,14 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 
 		objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
 		if o.Lifecycle != nil {
-			action := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
-			switch action {
+			evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
+			switch evt.Action {
 			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
+				globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
 				// Skip this entry.
 				continue
 			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-				globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
+				globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
 				// Skip this entry.
 				continue
 			}

cmd/object-handlers.go

@@ -495,15 +495,15 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		// Automatically remove the object/version is an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			action := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
 			var success bool
-			switch action {
+			switch evt.Action {
 			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				success = applyExpiryRule(objInfo, false, action == lifecycle.DeleteVersionAction)
+				success = applyExpiryRule(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
 			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 				// Restored object delete would be still allowed to proceed as success
 				// since transition behavior is slightly different.
-				applyExpiryRule(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
+				applyExpiryRule(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
 			}
 			if success {
 				writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
@@ -757,15 +757,15 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		// Automatically remove the object/version is an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			action := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
 			var success bool
-			switch action {
+			switch evt.Action {
 			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				success = applyExpiryRule(objInfo, false, action == lifecycle.DeleteVersionAction)
+				success = applyExpiryRule(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
 			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 				// Restored object delete would be still allowed to proceed as success
 				// since transition behavior is slightly different.
-				applyExpiryRule(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
+				applyExpiryRule(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
 			}
 			if success {
 				writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))