Support for remote tier management (#12090)

With this change, MinIO's ILM supports transitioning objects to a remote tier.
This change includes support for Azure Blob Storage, AWS S3 compatible object
storage incl. MinIO and Google Cloud Storage as remote tier storage backends.

Some new additions include:

 - Admin APIs remote tier configuration management

 - Simple journal to track remote objects to be 'collected'
   This is used by object API handlers which 'mutate' object versions by
   overwriting/replacing content (Put/CopyObject) or removing the version
   itself (e.g DeleteObjectVersion).

 - Rework of previous ILM transition to fit the new model
   In the new model, a storage class (a.k.a remote tier) is defined by the
   'remote' object storage type (one of s3, azure, GCS), bucket name and a
   prefix.

* Fixed bugs, review comments, and more unit-tests

- Leverage inline small object feature
- Migrate legacy objects to the latest object format before transitioning
- Fix restore to particular version if specified
- Extend SharedDataDirCount to handle transitioned and restored objects
- Restore-object should accept version-id for version-suspended bucket (#12091)
- Check if remote tier creds have sufficient permissions
- Bonus minor fixes to existing error messages

Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io>
Co-authored-by: Krishna Srinivas <krishna@minio.io>
Signed-off-by: Harshavardhana <harsha@minio.io>

pkg/bucket/lifecycle/action_string.go

@@ -1,20 +1,3 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
 // Code generated by "stringer -type Action lifecycle.go"; DO NOT EDIT.
 
 package lifecycle

pkg/bucket/lifecycle/lifecycle.go

@@ -244,17 +244,29 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
 // ObjectOpts provides information to deduce the lifecycle actions
 // which can be triggered on the resultant object.
 type ObjectOpts struct {
-	Name             string
-	UserTags         string
-	ModTime          time.Time
-	VersionID        string
-	IsLatest         bool
-	DeleteMarker     bool
-	NumVersions      int
-	SuccessorModTime time.Time
-	TransitionStatus string
-	RestoreOngoing   bool
-	RestoreExpires   time.Time
+	Name                   string
+	UserTags               string
+	ModTime                time.Time
+	VersionID              string
+	IsLatest               bool
+	DeleteMarker           bool
+	NumVersions            int
+	SuccessorModTime       time.Time
+	TransitionStatus       string
+	RestoreOngoing         bool
+	RestoreExpires         time.Time
+	RemoteTiersImmediately []string // strictly for debug only
+}
+
+// doesMatchDebugTiers returns true if tier matches one of the debugTiers, false
+// otherwise.
+func doesMatchDebugTiers(tier string, debugTiers []string) bool {
+	for _, t := range debugTiers {
+		if strings.ToUpper(tier) == strings.ToUpper(t) {
+			return true
+		}
+	}
+	return false
 }
 
 // ExpiredObjectDeleteMarker returns true if an object version referred to by o
@@ -271,7 +283,6 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 	if obj.ModTime.IsZero() {
 		return action
 	}
-
 	for _, rule := range lc.FilterActionableRules(obj) {
 		if obj.ExpiredObjectDeleteMarker() && rule.Expiration.DeleteMarker.val {
 			// Indicates whether MinIO will remove a delete marker with no noncurrent versions.
@@ -304,11 +315,17 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 
 		if !rule.NoncurrentVersionTransition.IsDaysNull() {
 			if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() && !obj.DeleteMarker && obj.TransitionStatus != TransitionComplete {
-				// Non current versions should be deleted if their age exceeds non current days configuration
+				// Non current versions should be transitioned if their age exceeds non current days configuration
 				// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
 				if time.Now().After(ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionTransition.NoncurrentDays))) {
 					return TransitionVersionAction
 				}
+
+				// this if condition is strictly for debug purposes to force immediate
+				// transition to remote tier if _MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY is set
+				if doesMatchDebugTiers(rule.NoncurrentVersionTransition.StorageClass, obj.RemoteTiersImmediately) {
+					return TransitionVersionAction
+				}
 			}
 		}
 
@@ -335,6 +352,20 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 					if time.Now().UTC().After(ExpectedExpiryTime(obj.ModTime, int(rule.Transition.Days))) {
 						action = TransitionAction
 					}
+
+				}
+				// this if condition is strictly for debug purposes to force immediate
+				// transition to remote tier if _MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY is set
+				if !rule.Transition.IsNull() && doesMatchDebugTiers(rule.Transition.StorageClass, obj.RemoteTiersImmediately) {
+					action = TransitionAction
+				}
+
+				if !obj.RestoreExpires.IsZero() && time.Now().After(obj.RestoreExpires) {
+					if obj.VersionID != "" {
+						action = DeleteRestoredVersionAction
+					} else {
+						action = DeleteRestoredAction
+					}
 				}
 			}
 			if !obj.RestoreExpires.IsZero() && time.Now().After(obj.RestoreExpires) {
@@ -361,7 +392,7 @@ func ExpectedExpiryTime(modTime time.Time, days int) time.Time {
 }
 
 // PredictExpiryTime returns the expiry date/time of a given object
-// after evaluting the current lifecycle document.
+// after evaluating the current lifecycle document.
 func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
 	if obj.DeleteMarker {
 		// We don't need to send any x-amz-expiration for delete marker.
@@ -394,3 +425,38 @@ func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
 	}
 	return finalExpiryRuleID, finalExpiryDate
 }
+
+// PredictTransitionTime returns the transition date/time of a given object
+// after evaluating the current lifecycle document.
+func (lc Lifecycle) PredictTransitionTime(obj ObjectOpts) (string, time.Time) {
+	if obj.DeleteMarker {
+		// We don't need to send any x-minio-transition for delete marker.
+		return "", time.Time{}
+	}
+
+	if obj.TransitionStatus == TransitionComplete {
+		return "", time.Time{}
+	}
+
+	var finalTransitionDate time.Time
+	var finalTransitionRuleID string
+
+	// Iterate over all actionable rules and find the earliest
+	// transition date and its associated rule ID.
+	for _, rule := range lc.FilterActionableRules(obj) {
+		switch {
+		case !rule.Transition.IsDateNull():
+			if finalTransitionDate.IsZero() || finalTransitionDate.After(rule.Transition.Date.Time) {
+				finalTransitionRuleID = rule.ID
+				finalTransitionDate = rule.Transition.Date.Time
+			}
+		case !rule.Transition.IsDaysNull():
+			expectedTransition := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days))
+			if finalTransitionDate.IsZero() || finalTransitionDate.After(expectedTransition) {
+				finalTransitionRuleID = rule.ID
+				finalTransitionDate = expectedTransition
+			}
+		}
+	}
+	return finalTransitionRuleID, finalTransitionDate
+}