From d0963974a5072abc9c650aa4e8407d1a2c88f1d9 Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Tue, 20 Jul 2021 17:36:55 -0700 Subject: [PATCH] pkg/lifecycle: Add SetPredictionHeaders method (#12755) This method is used to add expected expiration and transition time for an object in GET/HEAD Object response headers. Also fixed bugs in lifecycle.PredictTransitionTime and getLifecycleTransitionTier in handling current and non-current versions. --- cmd/api-headers.go | 33 +------ cmd/bucket-lifecycle.go | 23 ++++- cmd/bucket-lifecycle_test.go | 37 ++++++++ cmd/object-handlers-common.go | 16 +--- internal/bucket/lifecycle/lifecycle.go | 38 +++++--- internal/bucket/lifecycle/lifecycle_test.go | 87 +++++++++++++++++++ .../bucket/lifecycle/noncurrentversion.go | 12 +++ internal/bucket/lifecycle/transition.go | 17 ++++ 8 files changed, 204 insertions(+), 59 deletions(-) diff --git a/cmd/api-headers.go b/cmd/api-headers.go index 7ab6984da..567bb53e2 100644 --- a/cmd/api-headers.go +++ b/cmd/api-headers.go @@ -28,7 +28,6 @@ import ( "strings" "time" - "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/crypto" xhttp "github.com/minio/minio/internal/http" ) @@ -186,42 +185,12 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp } if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil { - if opts.VersionID == "" { - if ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{ - Name: objInfo.Name, - UserTags: objInfo.UserTags, - VersionID: objInfo.VersionID, - ModTime: objInfo.ModTime, - IsLatest: objInfo.IsLatest, - DeleteMarker: objInfo.DeleteMarker, - SuccessorModTime: objInfo.SuccessorModTime, - }); !expiryTime.IsZero() { - w.Header()[xhttp.AmzExpiration] = []string{ - fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID), - } - } - } if objInfo.IsRemote() { // Check if object is being restored. For more information on x-amz-restore header see // https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax w.Header()[xhttp.AmzStorageClass] = []string{objInfo.TransitionTier} } - ruleID, transitionTime := lc.PredictTransitionTime(lifecycle.ObjectOpts{ - Name: objInfo.Name, - UserTags: objInfo.UserTags, - VersionID: objInfo.VersionID, - ModTime: objInfo.ModTime, - IsLatest: objInfo.IsLatest, - DeleteMarker: objInfo.DeleteMarker, - TransitionStatus: objInfo.TransitionStatus, - }) - if !transitionTime.IsZero() { - // This header is a MinIO centric extension to show expected transition date in a similar spirit as x-amz-expiration - w.Header()[xhttp.MinIOTransition] = []string{ - fmt.Sprintf(`transition-date="%s", rule-id="%s"`, transitionTime.Format(http.TimeFormat), ruleID), - } - } - + lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts()) } return nil diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index ad15377f1..8bafd7b7a 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -309,9 +309,12 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo) // getLifeCycleTransitionTier returns storage class for transition target func getLifeCycleTransitionTier(ctx context.Context, lc *lifecycle.Lifecycle, bucket string, obj lifecycle.ObjectOpts) string { for _, rule := range lc.FilterActionableRules(obj) { - if rule.Transition.StorageClass != "" { + if obj.IsLatest && rule.Transition.StorageClass != "" { return rule.Transition.StorageClass } + if !obj.IsLatest && rule.NoncurrentVersionTransition.StorageClass != "" { + return rule.NoncurrentVersionTransition.StorageClass + } } return "" } @@ -673,3 +676,21 @@ func isRestoredObjectOnDisk(meta map[string]string) (onDisk bool) { } return onDisk } + +// ToLifecycleOpts returns lifecycle.ObjectOpts value for oi. +func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts { + return lifecycle.ObjectOpts{ + Name: oi.Name, + UserTags: oi.UserTags, + VersionID: oi.VersionID, + ModTime: oi.ModTime, + IsLatest: oi.IsLatest, + NumVersions: oi.NumVersions, + DeleteMarker: oi.DeleteMarker, + SuccessorModTime: oi.SuccessorModTime, + RestoreOngoing: oi.RestoreOngoing, + RestoreExpires: oi.RestoreExpires, + TransitionStatus: oi.TransitionStatus, + RemoteTiersImmediately: globalDebugRemoteTiersImmediately, + } +} diff --git a/cmd/bucket-lifecycle_test.go b/cmd/bucket-lifecycle_test.go index f278e9684..feda1efee 100644 --- a/cmd/bucket-lifecycle_test.go +++ b/cmd/bucket-lifecycle_test.go @@ -246,3 +246,40 @@ func TestValidateTransitionTier(t *testing.T) { } } } + +func TestGetLifecycleTransitionTier(t *testing.T) { + lc := lifecycle.Lifecycle{ + Rules: []lifecycle.Rule{ + { + ID: "rule-1", + Status: "Enabled", + Transition: lifecycle.Transition{ + Days: lifecycle.TransitionDays(3), + StorageClass: "TIER-1", + }, + }, + { + ID: "rule-2", + Status: "Enabled", + NoncurrentVersionTransition: lifecycle.NoncurrentVersionTransition{ + NoncurrentDays: lifecycle.ExpirationDays(3), + StorageClass: "TIER-2", + }, + }, + }, + } + + obj1 := lifecycle.ObjectOpts{ + Name: "obj1", + IsLatest: true, + } + obj2 := lifecycle.ObjectOpts{ + Name: "obj2", + } + if got := getLifeCycleTransitionTier(context.TODO(), &lc, "bucket", obj1); got != "TIER-1" { + t.Fatalf("Expected TIER-1 but got %s", got) + } + if got := getLifeCycleTransitionTier(context.TODO(), &lc, "bucket", obj2); got != "TIER-2" { + t.Fatalf("Expected TIER-2 but got %s", got) + } +} diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go index 41608f5fe..ac3fb3091 100644 --- a/cmd/object-handlers-common.go +++ b/cmd/object-handlers-common.go @@ -19,13 +19,11 @@ package cmd import ( "context" - "fmt" "net/http" "regexp" "strconv" "time" - "github.com/minio/minio/internal/bucket/lifecycle" xhttp "github.com/minio/minio/internal/http" ) @@ -262,19 +260,7 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) { if objInfo.Bucket != "" && objInfo.Name != "" { if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil && !delete { - ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{ - Name: objInfo.Name, - UserTags: objInfo.UserTags, - VersionID: objInfo.VersionID, - ModTime: objInfo.ModTime, - IsLatest: objInfo.IsLatest, - DeleteMarker: objInfo.DeleteMarker, - }) - if !expiryTime.IsZero() { - w.Header()[xhttp.AmzExpiration] = []string{ - fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID), - } - } + lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts()) } } } diff --git a/internal/bucket/lifecycle/lifecycle.go b/internal/bucket/lifecycle/lifecycle.go index 9c8234392..78f050eed 100644 --- a/internal/bucket/lifecycle/lifecycle.go +++ b/internal/bucket/lifecycle/lifecycle.go @@ -21,8 +21,11 @@ import ( "encoding/xml" "fmt" "io" + "net/http" "strings" "time" + + xhttp "github.com/minio/minio/internal/http" ) var ( @@ -438,25 +441,38 @@ func (lc Lifecycle) PredictTransitionTime(obj ObjectOpts) (string, time.Time) { return "", time.Time{} } - var finalTransitionDate time.Time - var finalTransitionRuleID string - // Iterate over all actionable rules and find the earliest // transition date and its associated rule ID. + var finalTransitionDate time.Time + var finalTransitionRuleID string for _, rule := range lc.FilterActionableRules(obj) { - switch { - case !rule.Transition.IsDateNull(): - if finalTransitionDate.IsZero() || finalTransitionDate.After(rule.Transition.Date.Time) { + if due, ok := rule.Transition.NextDue(obj); ok { + if finalTransitionDate.IsZero() || finalTransitionDate.After(due) { finalTransitionRuleID = rule.ID - finalTransitionDate = rule.Transition.Date.Time + finalTransitionDate = due } - case !rule.Transition.IsDaysNull(): - expectedTransition := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)) - if finalTransitionDate.IsZero() || finalTransitionDate.After(expectedTransition) { + } + if due, ok := rule.NoncurrentVersionTransition.NextDue(obj); ok { + if finalTransitionDate.IsZero() || finalTransitionDate.After(due) { finalTransitionRuleID = rule.ID - finalTransitionDate = expectedTransition + finalTransitionDate = due } } } return finalTransitionRuleID, finalTransitionDate } + +// SetPredictionHeaders sets time to expiry and transition headers on w for a +// given obj. +func (lc Lifecycle) SetPredictionHeaders(w http.ResponseWriter, obj ObjectOpts) { + if ruleID, expiry := lc.PredictExpiryTime(obj); !expiry.IsZero() { + w.Header()[xhttp.AmzExpiration] = []string{ + fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiry.Format(http.TimeFormat), ruleID), + } + } + if ruleID, transition := lc.PredictTransitionTime(obj); !transition.IsZero() { + w.Header()[xhttp.MinIOTransition] = []string{ + fmt.Sprintf(`transition-date="%s", rule-id="%s"`, transition.Format(http.TimeFormat), ruleID), + } + } +} diff --git a/internal/bucket/lifecycle/lifecycle_test.go b/internal/bucket/lifecycle/lifecycle_test.go index 33a4e65c7..92cbe9e8c 100644 --- a/internal/bucket/lifecycle/lifecycle_test.go +++ b/internal/bucket/lifecycle/lifecycle_test.go @@ -21,8 +21,13 @@ import ( "bytes" "encoding/xml" "fmt" + "net/http" + "net/http/httptest" + "strings" "testing" "time" + + xhttp "github.com/minio/minio/internal/http" ) func TestParseAndValidateLifecycleConfig(t *testing.T) { @@ -429,3 +434,85 @@ func TestHasActiveRules(t *testing.T) { } } + +func TestSetPredictionHeaders(t *testing.T) { + lc := Lifecycle{ + Rules: []Rule{ + { + ID: "rule-1", + Status: "Enabled", + Expiration: Expiration{ + Days: ExpirationDays(3), + set: true, + }, + }, + { + ID: "rule-2", + Status: "Enabled", + Transition: Transition{ + Days: TransitionDays(3), + StorageClass: "TIER-1", + set: true, + }, + }, + { + ID: "rule-3", + Status: "Enabled", + NoncurrentVersionTransition: NoncurrentVersionTransition{ + NoncurrentDays: ExpirationDays(5), + StorageClass: "TIER-2", + set: true, + }, + }, + }, + } + + // current version + obj1 := ObjectOpts{ + Name: "obj1", + IsLatest: true, + } + // non-current version + obj2 := ObjectOpts{ + Name: "obj2", + } + + tests := []struct { + obj ObjectOpts + expRuleID int + transRuleID int + }{ + { + obj: obj1, + expRuleID: 0, + transRuleID: 1, + }, + { + obj: obj2, + expRuleID: 0, + transRuleID: 2, + }, + } + for i, tc := range tests { + w := httptest.NewRecorder() + lc.SetPredictionHeaders(w, tc.obj) + if expHdrs, ok := w.Header()[xhttp.AmzExpiration]; ok && !strings.Contains(expHdrs[0], lc.Rules[tc.expRuleID].ID) { + t.Fatalf("Test %d: Expected %s header", i+1, xhttp.AmzExpiration) + } + if transHdrs, ok := w.Header()[xhttp.MinIOTransition]; ok { + if !strings.Contains(transHdrs[0], lc.Rules[tc.transRuleID].ID) { + t.Fatalf("Test %d: Expected %s header", i+1, xhttp.MinIOTransition) + } + + if tc.obj.IsLatest { + if expectedDue, _ := lc.Rules[tc.transRuleID].Transition.NextDue(tc.obj); !strings.Contains(transHdrs[0], expectedDue.Format(http.TimeFormat)) { + t.Fatalf("Test %d: Expected transition time %s", i+1, expectedDue) + } + } else { + if expectedDue, _ := lc.Rules[tc.transRuleID].NoncurrentVersionTransition.NextDue(tc.obj); !strings.Contains(transHdrs[0], expectedDue.Format(http.TimeFormat)) { + t.Fatalf("Test %d: Expected transition time %s", i+1, expectedDue) + } + } + } + } +} diff --git a/internal/bucket/lifecycle/noncurrentversion.go b/internal/bucket/lifecycle/noncurrentversion.go index b61d829b9..32786bcd7 100644 --- a/internal/bucket/lifecycle/noncurrentversion.go +++ b/internal/bucket/lifecycle/noncurrentversion.go @@ -19,6 +19,7 @@ package lifecycle import ( "encoding/xml" + "time" ) // NoncurrentVersionExpiration - an action for lifecycle configuration rule. @@ -112,3 +113,14 @@ func (n NoncurrentVersionTransition) Validate() error { } return nil } + +// NextDue returns upcoming NoncurrentVersionTransition date for obj if +// applicable, returns false otherwise. +func (n NoncurrentVersionTransition) NextDue(obj ObjectOpts) (time.Time, bool) { + switch { + case obj.IsLatest, n.IsDaysNull(): + return time.Time{}, false + } + + return ExpectedExpiryTime(obj.SuccessorModTime, int(n.NoncurrentDays)), true +} diff --git a/internal/bucket/lifecycle/transition.go b/internal/bucket/lifecycle/transition.go index db61e13fb..17facd4e2 100644 --- a/internal/bucket/lifecycle/transition.go +++ b/internal/bucket/lifecycle/transition.go @@ -163,3 +163,20 @@ func (t Transition) IsDateNull() bool { func (t Transition) IsNull() bool { return t.IsDaysNull() && t.IsDateNull() } + +// NextDue returns upcoming transition date for obj and true if applicable, +// returns false otherwise. +func (t Transition) NextDue(obj ObjectOpts) (time.Time, bool) { + if !obj.IsLatest { + return time.Time{}, false + } + + switch { + case !t.IsDateNull(): + return t.Date.Time, true + case !t.IsDaysNull(): + return ExpectedExpiryTime(obj.ModTime, int(t.Days)), true + } + + return time.Time{}, false +}