Add support for ILM transition (#10565)

This PR adds transition support for ILM
to transition data to another MinIO target
represented by a storage class ARN. Subsequent
GET or HEAD for that object will be streamed from
the transition tier. If PostRestoreObject API is
invoked, the transitioned object can be restored for
duration specified to the source cluster.

pkg/bucket/lifecycle/lifecycle.go

@@ -30,6 +30,13 @@ var (
 	errXMLNotWellFormed      = Errorf("The XML you provided was not well-formed or did not validate against our published schema")
 )
 
+const (
+	// TransitionComplete marks completed transition
+	TransitionComplete = "complete"
+	// TransitionPending - transition is yet to be attempted
+	TransitionPending = "pending"
+)
+
 // Action represents a delete action or other transition
 // actions that will be implemented later.
 type Action int
@@ -39,10 +46,18 @@ type Action int
 const (
 	// NoneAction means no action required after evaluting lifecycle rules
 	NoneAction Action = iota
-	// DeleteAction means the object needs to be removed after evaluting lifecycle rules
+	// DeleteAction means the object needs to be removed after evaluating lifecycle rules
 	DeleteAction
 	// DeleteVersionAction deletes a particular version
 	DeleteVersionAction
+	// TransitionAction transitions a particular object after evaluating lifecycle transition rules
+	TransitionAction
+	//TransitionVersionAction transitions a particular object version after evaluating lifecycle transition rules
+	TransitionVersionAction
+	// DeleteRestoredAction means the temporarily restored object needs to be removed after evaluating lifecycle rules
+	DeleteRestoredAction
+	// DeleteRestoredVersionAction deletes a particular version that was temporarily restored
+	DeleteRestoredVersionAction
 )
 
 // Lifecycle - Configuration for bucket lifecycle.
@@ -85,13 +100,18 @@ func (lc Lifecycle) HasActiveRules(prefix string, recursive bool) bool {
 		if rule.NoncurrentVersionTransition.NoncurrentDays > 0 {
 			return true
 		}
-		if rule.Expiration.IsNull() {
+		if rule.Expiration.IsNull() && rule.Transition.IsNull() {
 			continue
 		}
-		if !rule.Expiration.IsDateNull() && rule.Expiration.Date.After(time.Now()) {
-			continue
+		if !rule.Expiration.IsDateNull() && rule.Expiration.Date.Before(time.Now()) {
+			return true
+		}
+		if !rule.Transition.IsDateNull() && rule.Transition.Date.Before(time.Now()) {
+			return true
+		}
+		if !rule.Expiration.IsDaysNull() || !rule.Transition.IsDaysNull() {
+			return true
 		}
-		return true
 	}
 	return false
 }
@@ -160,15 +180,26 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
 			continue
 		}
 		// The NoncurrentVersionExpiration action requests MinIO to expire
-		// noncurrent versions of objects 100 days after the objects become
+		// noncurrent versions of objects x days after the objects become
 		// noncurrent.
 		if !rule.NoncurrentVersionExpiration.IsDaysNull() {
 			rules = append(rules, rule)
 			continue
 		}
+		// The NoncurrentVersionTransition action requests MinIO to transition
+		// noncurrent versions of objects x days after the objects become
+		// noncurrent.
+		if !rule.NoncurrentVersionTransition.IsDaysNull() {
+			rules = append(rules, rule)
+			continue
+		}
+
 		if rule.Filter.TestTags(strings.Split(obj.UserTags, "&")) {
 			rules = append(rules, rule)
 		}
+		if !rule.Transition.IsNull() {
+			rules = append(rules, rule)
+		}
 	}
 	return rules
 }
@@ -184,6 +215,9 @@ type ObjectOpts struct {
 	DeleteMarker     bool
 	NumVersions      int
 	SuccessorModTime time.Time
+	TransitionStatus string
+	RestoreOngoing   bool
+	RestoreExpires   time.Time
 }
 
 // ComputeAction returns the action to perform by evaluating all lifecycle rules
@@ -207,11 +241,20 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 			if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() {
 				// Non current versions should be deleted if their age exceeds non current days configuration
 				// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
-				if time.Now().After(expectedExpiryTime(obj.SuccessorModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) {
+				if time.Now().After(ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionExpiration.NoncurrentDays))) {
 					return DeleteVersionAction
 				}
 			}
 		}
+		if !rule.NoncurrentVersionTransition.IsDaysNull() {
+			if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() && obj.TransitionStatus != TransitionComplete {
+				// Non current versions should be deleted if their age exceeds non current days configuration
+				// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
+				if time.Now().After(ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionTransition.NoncurrentDays))) {
+					return TransitionVersionAction
+				}
+			}
+		}
 
 		// Remove the object or simply add a delete marker (once) in a versioned bucket
 		if obj.VersionID == "" || obj.IsLatest && !obj.DeleteMarker {
@@ -221,22 +264,42 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 					action = DeleteAction
 				}
 			case !rule.Expiration.IsDaysNull():
-				if time.Now().UTC().After(expectedExpiryTime(obj.ModTime, rule.Expiration.Days)) {
+				if time.Now().UTC().After(ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days))) {
 					action = DeleteAction
 				}
 			}
+			if action == NoneAction {
+				if obj.TransitionStatus != TransitionComplete {
+					switch {
+					case !rule.Transition.IsDateNull():
+						if time.Now().UTC().After(rule.Transition.Date.Time) {
+							action = TransitionAction
+						}
+					case !rule.Transition.IsDaysNull():
+						if time.Now().UTC().After(ExpectedExpiryTime(obj.ModTime, int(rule.Transition.Days))) {
+							action = TransitionAction
+						}
+					}
+				}
+				if !obj.RestoreExpires.IsZero() && time.Now().After(obj.RestoreExpires) {
+					if obj.VersionID != "" {
+						action = DeleteRestoredVersionAction
+					} else {
+						action = DeleteRestoredAction
+					}
+				}
+			}
 		}
 	}
-
 	return action
 }
 
-// expectedExpiryTime calculates the expiry date/time based on a object modtime.
-// The expected expiry time is always a midnight time following the the object
-// modification time plus the number of expiration days.
+// ExpectedExpiryTime calculates the expiry, transition or restore date/time based on a object modtime.
+// The expected transition or restore time is always a midnight time following the the object
+// modification time plus the number of transition/restore days.
 //   e.g. If the object modtime is `Thu May 21 13:42:50 GMT 2020` and the object should
-//        expire in 1 day, then the expected expiry time is `Fri, 23 May 2020 00:00:00 GMT`
-func expectedExpiryTime(modTime time.Time, days ExpirationDays) time.Time {
+//       transition in 1 day, then the expected transition time is `Fri, 23 May 2020 00:00:00 GMT`
+func ExpectedExpiryTime(modTime time.Time, days int) time.Time {
 	t := modTime.UTC().Add(time.Duration(days+1) * 24 * time.Hour)
 	return t.Truncate(24 * time.Hour)
 }
@@ -256,7 +319,7 @@ func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
 	// expiration date and its associated rule ID.
 	for _, rule := range lc.FilterActionableRules(obj) {
 		if !rule.NoncurrentVersionExpiration.IsDaysNull() && !obj.IsLatest && obj.VersionID != "" {
-			return rule.ID, expectedExpiryTime(time.Now(), ExpirationDays(rule.NoncurrentVersionExpiration.NoncurrentDays))
+			return rule.ID, ExpectedExpiryTime(time.Now(), int(rule.NoncurrentVersionExpiration.NoncurrentDays))
 		}
 
 		if !rule.Expiration.IsDateNull() {
@@ -266,7 +329,7 @@ func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
 			}
 		}
 		if !rule.Expiration.IsDaysNull() {
-			expectedExpiry := expectedExpiryTime(obj.ModTime, rule.Expiration.Days)
+			expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days))
 			if finalExpiryDate.IsZero() || finalExpiryDate.After(expectedExpiry) {
 				finalExpiryRuleID = rule.ID
 				finalExpiryDate = expectedExpiry

pkg/bucket/lifecycle/lifecycle_test.go

@@ -240,7 +240,7 @@ func TestExpectedExpiryTime(t *testing.T) {
 
 	for i, tc := range testCases {
 		t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
-			got := expectedExpiryTime(tc.modTime, tc.days)
+			got := ExpectedExpiryTime(tc.modTime, int(tc.days))
 			if !got.Equal(tc.expected) {
 				t.Fatalf("Expected %v to be equal to %v", got, tc.expected)
 			}

pkg/bucket/lifecycle/noncurrentversion.go

@@ -32,10 +32,6 @@ type NoncurrentVersionTransition struct {
 	StorageClass   string         `xml:"StorageClass"`
 }
 
-var (
-	errNoncurrentVersionTransitionUnsupported = Errorf("Specifying <NoncurrentVersionTransition></NoncurrentVersionTransition> is not supported")
-)
-
 // MarshalXML if non-current days not set to non zero value
 func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	if n.IsDaysNull() {
@@ -50,13 +46,6 @@ func (n NoncurrentVersionExpiration) IsDaysNull() bool {
 	return n.NoncurrentDays == ExpirationDays(0)
 }
 
-// UnmarshalXML is extended to indicate lack of support for
-// NoncurrentVersionTransition xml tag in object lifecycle
-// configuration
-func (n NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
-	return errNoncurrentVersionTransitionUnsupported
-}
-
 // MarshalXML is extended to leave out
 // <NoncurrentVersionTransition></NoncurrentVersionTransition> tags
 func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
@@ -65,3 +54,8 @@ func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartE
 	}
 	return e.EncodeElement(&n, start)
 }
+
+// IsDaysNull returns true if days field is null
+func (n NoncurrentVersionTransition) IsDaysNull() bool {
+	return n.NoncurrentDays == ExpirationDays(0)
+}

pkg/bucket/lifecycle/rule.go

@@ -100,6 +100,10 @@ func (r Rule) validateFilter() error {
 	return r.Filter.Validate()
 }
 
+func (r Rule) validateTransition() error {
+	return r.Transition.Validate()
+}
+
 // Prefix - a rule can either have prefix under <filter></filter> or under
 // <filter><and></and></filter>. This method returns the prefix from the
 // location where it is available
@@ -147,5 +151,8 @@ func (r Rule) Validate() error {
 	if err := r.validateFilter(); err != nil {
 		return err
 	}
+	if err := r.validateTransition(); err != nil {
+		return err
+	}
 	return nil
 }

pkg/bucket/lifecycle/rule_test.go

@@ -22,39 +22,6 @@ import (
 	"testing"
 )
 
-// TestUnsupportedRules checks if Rule xml with unsuported tags return
-// appropriate errors on parsing
-func TestUnsupportedRules(t *testing.T) {
-	// NoncurrentVersionTransition, and Transition tags aren't supported
-	unsupportedTestCases := []struct {
-		inputXML    string
-		expectedErr error
-	}{
-		{ // Rule with unsupported NoncurrentVersionTransition
-			inputXML: ` <Rule>
-	                     <NoncurrentVersionTransition></NoncurrentVersionTransition>
-	                    </Rule>`,
-			expectedErr: errNoncurrentVersionTransitionUnsupported,
-		},
-		{ // Rule with unsupported Transition action
-			inputXML: ` <Rule>
-	                     <Transition></Transition>
-	                    </Rule>`,
-			expectedErr: errTransitionUnsupported,
-		},
-	}
-
-	for i, tc := range unsupportedTestCases {
-		t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
-			var rule Rule
-			err := xml.Unmarshal([]byte(tc.inputXML), &rule)
-			if err != tc.expectedErr {
-				t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
-			}
-		})
-	}
-}
-
 // TestInvalidRules checks if Rule xml with invalid elements returns
 // appropriate errors on validation
 func TestInvalidRules(t *testing.T) {

pkg/bucket/lifecycle/transition.go

@@ -18,25 +18,147 @@ package lifecycle
 
 import (
 	"encoding/xml"
+	"time"
 )
 
+var (
+	errTransitionInvalidDays     = Errorf("Days must be 0 or greater when used with Transition")
+	errTransitionInvalidDate     = Errorf("Date must be provided in ISO 8601 format")
+	errTransitionInvalid         = Errorf("Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present inside Expiration.")
+	errTransitionDateNotMidnight = Errorf("'Date' must be at midnight GMT")
+)
+
+// TransitionDate is a embedded type containing time.Time to unmarshal
+// Date in Transition
+type TransitionDate struct {
+	time.Time
+}
+
+// UnmarshalXML parses date from Transition and validates date format
+func (tDate *TransitionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
+	var dateStr string
+	err := d.DecodeElement(&dateStr, &startElement)
+	if err != nil {
+		return err
+	}
+	// While AWS documentation mentions that the date specified
+	// must be present in ISO 8601 format, in reality they allow
+	// users to provide RFC 3339 compliant dates.
+	trnDate, err := time.Parse(time.RFC3339, dateStr)
+	if err != nil {
+		return errTransitionInvalidDate
+	}
+	// Allow only date timestamp specifying midnight GMT
+	hr, min, sec := trnDate.Clock()
+	nsec := trnDate.Nanosecond()
+	loc := trnDate.Location()
+	if !(hr == 0 && min == 0 && sec == 0 && nsec == 0 && loc.String() == time.UTC.String()) {
+		return errTransitionDateNotMidnight
+	}
+
+	*tDate = TransitionDate{trnDate}
+	return nil
+}
+
+// MarshalXML encodes expiration date if it is non-zero and encodes
+// empty string otherwise
+func (tDate TransitionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if tDate.Time.IsZero() {
+		return nil
+	}
+	return e.EncodeElement(tDate.Format(time.RFC3339), startElement)
+}
+
+// TransitionDays is a type alias to unmarshal Days in Transition
+type TransitionDays int
+
+// UnmarshalXML parses number of days from Transition and validates if
+// >= 0
+func (tDays *TransitionDays) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
+	var numDays int
+	err := d.DecodeElement(&numDays, &startElement)
+	if err != nil {
+		return err
+	}
+	if numDays < 0 {
+		return errTransitionInvalidDays
+	}
+	*tDays = TransitionDays(numDays)
+	return nil
+}
+
+// MarshalXML encodes number of days to expire if it is non-zero and
+// encodes empty string otherwise
+func (tDays TransitionDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if tDays == 0 {
+		return nil
+	}
+	return e.EncodeElement(int(tDays), startElement)
+}
+
 // Transition - transition actions for a rule in lifecycle configuration.
 type Transition struct {
-	XMLName      xml.Name `xml:"Transition"`
-	Days         int      `xml:"Days,omitempty"`
-	Date         string   `xml:"Date,omitempty"`
-	StorageClass string   `xml:"StorageClass"`
+	XMLName      xml.Name       `xml:"Transition"`
+	Days         TransitionDays `xml:"Days,omitempty"`
+	Date         TransitionDate `xml:"Date,omitempty"`
+	StorageClass string         `xml:"StorageClass,omitempty"`
+
+	set bool
 }
 
-var errTransitionUnsupported = Errorf("Specifying <Transition></Transition> tag is not supported")
-
-// UnmarshalXML is extended to indicate lack of support for Transition
-// xml tag in object lifecycle configuration
-func (t Transition) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
-	return errTransitionUnsupported
+// MarshalXML encodes transition field into an XML form.
+func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
+	if !t.set {
+		return nil
+	}
+	type transitionWrapper Transition
+	return enc.EncodeElement(transitionWrapper(t), start)
 }
 
-// MarshalXML is extended to leave out <Transition></Transition> tags
-func (t Transition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+// UnmarshalXML decodes transition field from the XML form.
+func (t *Transition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
+	type transitionWrapper Transition
+	var trw transitionWrapper
+	err := d.DecodeElement(&trw, &startElement)
+	if err != nil {
+		return err
+	}
+	*t = Transition(trw)
+	t.set = true
 	return nil
 }
+
+// Validate - validates the "Expiration" element
+func (t Transition) Validate() error {
+	if !t.set {
+		return nil
+	}
+
+	if t.IsDaysNull() && t.IsDateNull() {
+		return errXMLNotWellFormed
+	}
+
+	// Both transition days and date are specified
+	if !t.IsDaysNull() && !t.IsDateNull() {
+		return errTransitionInvalid
+	}
+	if t.StorageClass == "" {
+		return errXMLNotWellFormed
+	}
+	return nil
+}
+
+// IsDaysNull returns true if days field is null
+func (t Transition) IsDaysNull() bool {
+	return t.Days == TransitionDays(0)
+}
+
+// IsDateNull returns true if date field is null
+func (t Transition) IsDateNull() bool {
+	return t.Date.Time.IsZero()
+}
+
+// IsNull returns true if both date and days fields are null
+func (t Transition) IsNull() bool {
+	return t.IsDaysNull() && t.IsDateNull()
+}