MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-04-08 21:55:44 -04:00
Code Issues Packages Projects Releases Wiki Activity

lifecycle: simplify Eval and HasActiveRules (#16036)

Browse Source
This commit is contained in:
Krishnan Parthasarathi 2022-11-10 07:17:45 -08:00 committed by GitHub
parent 5f1999cc71
commit 6eef9b4a23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 58 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/bucket-lifecycle.go
View File

@ -298,7 +298,7 @@ func validateTransitionTier(lc *lifecycle.Lifecycle) error {
// This is to be called after a successful upload of an object (version). // This is to be called after a successful upload of an object (version).
func enqueueTransitionImmediate(obj ObjectInfo) { func enqueueTransitionImmediate(obj ObjectInfo) {
if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil { if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
event := lc.Eval(obj.ToLifecycleOpts(), time.Now()) event := lc.Eval(obj.ToLifecycleOpts())
switch event.Action { switch event.Action {
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
globalTransitionState.queueTransitionTask(obj, event.StorageClass) globalTransitionState.queueTransitionTask(obj, event.StorageClass)

4
cmd/data-scanner.go
View File

@ -420,7 +420,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
filter := f.withFilter filter := f.withFilter
_, prefix := path2BucketObjectWithBasePath(f.root, folder.name) _, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
var activeLifeCycle *lifecycle.Lifecycle var activeLifeCycle *lifecycle.Lifecycle
if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) { if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) {
if f.dataUsageScannerDebug { if f.dataUsageScannerDebug {
console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix) console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
} }
@ -1101,7 +1101,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
} }
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event { func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
event := lc.Eval(obj.ToLifecycleOpts(), time.Now().UTC()) event := lc.Eval(obj.ToLifecycleOpts())
if serverDebugLog { if serverDebugLog {
console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action) console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
} }

2
cmd/xl-storage.go
View File

@ -445,7 +445,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
// Check if the current bucket has a configured lifecycle policy // Check if the current bucket has a configured lifecycle policy
if globalLifecycleSys != nil { if globalLifecycleSys != nil {
lc, err = globalLifecycleSys.Get(cache.Info.Name) lc, err = globalLifecycleSys.Get(cache.Info.Name)
if err == nil && lc.HasActiveRules("", true) { if err == nil && lc.HasActiveRules("") {
cache.Info.lifeCycle = lc cache.Info.lifeCycle = lc
if intDataUpdateTracker.debug { if intDataUpdateTracker.debug {
console.Debugln(color.Green("scannerDisk:") + " lifecycle: Active rules found") console.Debugln(color.Green("scannerDisk:") + " lifecycle: Active rules found")

43
internal/bucket/lifecycle/lifecycle.go
View File

@ -121,11 +121,8 @@ func (lc *Lifecycle) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err e
return nil return nil
} }
// HasActiveRules - returns whether policy has active rules for. // HasActiveRules - returns whether lc has active rules at any level below or at prefix.
// Optionally a prefix can be supplied. func (lc Lifecycle) HasActiveRules(prefix string) bool {
// If recursive is specified the function will also return true if any level below the
// prefix has active rules. If no prefix is specified recursive is effectively true.
func (lc Lifecycle) HasActiveRules(prefix string, recursive bool) bool {
if len(lc.Rules) == 0 { if len(lc.Rules) == 0 {
return false return false
} }
@ -135,17 +132,9 @@ func (lc Lifecycle) HasActiveRules(prefix string, recursive bool) bool {
} }
if len(prefix) > 0 && len(rule.GetPrefix()) > 0 { if len(prefix) > 0 && len(rule.GetPrefix()) > 0 {
if !recursive { // If recursive, we can skip this rule if it doesn't match the tested prefix.
// If not recursive, incoming prefix must be in rule prefix if !strings.HasPrefix(prefix, rule.GetPrefix()) && !strings.HasPrefix(rule.GetPrefix(), prefix) {
if !strings.HasPrefix(prefix, rule.GetPrefix()) { continue
continue
}
}
if recursive {
// If recursive, we can skip this rule if it doesn't match the tested prefix.
if !strings.HasPrefix(prefix, rule.GetPrefix()) && !strings.HasPrefix(rule.GetPrefix(), prefix) {
continue
}
} }
} }
@ -316,8 +305,14 @@ func (es lifecycleEvents) Less(i, j int) bool {
return es[i].Due.Before(es[j].Due) return es[i].Due.Before(es[j].Due)
} }
// Eval returns the lifecycle event applicable at now. If now is the zero value of time.Time, it returns the upcoming lifecycle event. // Eval returns the lifecycle event applicable now.
func (lc Lifecycle) Eval(obj ObjectOpts, now time.Time) Event { func (lc Lifecycle) Eval(obj ObjectOpts) Event {
return lc.eval(obj, time.Now().UTC())
}
// eval returns the lifecycle event applicable at the given now. If now is the
// zero value of time.Time, it returns the upcoming lifecycle event.
func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
var events []Event var events []Event
if obj.ModTime.IsZero() { if obj.ModTime.IsZero() {
return Event{} return Event{}
@ -371,8 +366,8 @@ func (lc Lifecycle) Eval(obj ObjectOpts, now time.Time) Event {
} }
// Skip rules with newer noncurrent versions specified. These rules are // Skip rules with newer noncurrent versions specified. These rules are
// not handled at an individual version level. ComputeAction applies // not handled at an individual version level. eval applies only to a
// only to a specific version. // specific version.
if !obj.IsLatest && rule.NoncurrentVersionExpiration.NewerNoncurrentVersions > 0 { if !obj.IsLatest && rule.NoncurrentVersionExpiration.NewerNoncurrentVersions > 0 {
continue continue
} }
@ -448,12 +443,6 @@ func (lc Lifecycle) Eval(obj ObjectOpts, now time.Time) Event {
} }
} }
// ComputeAction returns the action to perform by evaluating all lifecycle rules
// against the object name and its modification time.
func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
return lc.Eval(obj, time.Now().UTC()).Action
}
// ExpectedExpiryTime calculates the expiry, transition or restore date/time based on a object modtime. // ExpectedExpiryTime calculates the expiry, transition or restore date/time based on a object modtime.
// The expected transition or restore time is always a midnight time following the object // The expected transition or restore time is always a midnight time following the object
// modification time plus the number of transition/restore days. // modification time plus the number of transition/restore days.
@ -471,7 +460,7 @@ func ExpectedExpiryTime(modTime time.Time, days int) time.Time {
// SetPredictionHeaders sets time to expiry and transition headers on w for a // SetPredictionHeaders sets time to expiry and transition headers on w for a
// given obj. // given obj.
func (lc Lifecycle) SetPredictionHeaders(w http.ResponseWriter, obj ObjectOpts) { func (lc Lifecycle) SetPredictionHeaders(w http.ResponseWriter, obj ObjectOpts) {
event := lc.Eval(obj, time.Time{}) event := lc.eval(obj, time.Time{})
switch event.Action { switch event.Action {
case DeleteAction, DeleteVersionAction: case DeleteAction, DeleteVersionAction:
w.Header()[xhttp.AmzExpiration] = []string{ w.Header()[xhttp.AmzExpiration] = []string{

81
internal/bucket/lifecycle/lifecycle_test.go
View File

@ -221,7 +221,7 @@ func TestExpectedExpiryTime(t *testing.T) {
} }
} }
func TestComputeActions(t *testing.T) { func TestEval(t *testing.T) {
testCases := []struct { testCases := []struct {
inputConfig string inputConfig string
objectName string objectName string
@ -538,7 +538,7 @@ func TestComputeActions(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Got unexpected error: %v", err) t.Fatalf("Got unexpected error: %v", err)
} }
if resultAction := lc.ComputeAction(ObjectOpts{ if res := lc.Eval(ObjectOpts{
Name: tc.objectName, Name: tc.objectName,
UserTags: tc.objectTags, UserTags: tc.objectTags,
ModTime: tc.objectModTime, ModTime: tc.objectModTime,
@ -547,8 +547,8 @@ func TestComputeActions(t *testing.T) {
IsLatest: !tc.isNoncurrent, IsLatest: !tc.isNoncurrent,
SuccessorModTime: tc.objectSuccessorModTime, SuccessorModTime: tc.objectSuccessorModTime,
VersionID: tc.versionID, VersionID: tc.versionID,
}); resultAction != tc.expectedAction { }); res.Action != tc.expectedAction {
t.Fatalf("Expected action: `%v`, got: `%v`", tc.expectedAction, resultAction) t.Fatalf("Expected action: `%v`, got: `%v`", tc.expectedAction, res.Action)
} }
}) })
} }
@ -556,50 +556,49 @@ func TestComputeActions(t *testing.T) {
func TestHasActiveRules(t *testing.T) { func TestHasActiveRules(t *testing.T) {
testCases := []struct { testCases := []struct {
inputConfig string inputConfig string
prefix string prefix string
expectedNonRec bool want bool
expectedRec bool
}{ }{
{ {
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`, inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject", prefix: "foodir/foobject",
expectedNonRec: true, expectedRec: true, want: true,
}, },
{ // empty prefix { // empty prefix
inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`, inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject/foo.txt", prefix: "foodir/foobject/foo.txt",
expectedNonRec: true, expectedRec: true, want: true,
}, },
{ {
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`, inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "zdir/foobject", prefix: "zdir/foobject",
expectedNonRec: false, expectedRec: false, want: false,
}, },
{ {
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/zdir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`, inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/zdir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/", prefix: "foodir/",
expectedNonRec: false, expectedRec: true, want: true,
}, },
{ {
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix></Prefix></Filter><Status>Disabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`, inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix></Prefix></Filter><Status>Disabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/", prefix: "foodir/",
expectedNonRec: false, expectedRec: false, want: false,
}, },
{ {
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>2999-01-01T00:00:00.000Z</Date></Expiration></Rule></LifecycleConfiguration>`, inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>2999-01-01T00:00:00.000Z</Date></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject", prefix: "foodir/foobject",
expectedNonRec: false, expectedRec: false, want: false,
}, },
{ {
inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><Transition><StorageClass>S3TIER-1</StorageClass></Transition></Rule></LifecycleConfiguration>`, inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><Transition><StorageClass>S3TIER-1</StorageClass></Transition></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject/foo.txt", prefix: "foodir/foobject/foo.txt",
expectedNonRec: true, expectedRec: true, want: true,
}, },
{ {
inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><NoncurrentVersionTransition><StorageClass>S3TIER-1</StorageClass></NoncurrentVersionTransition></Rule></LifecycleConfiguration>`, inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><NoncurrentVersionTransition><StorageClass>S3TIER-1</StorageClass></NoncurrentVersionTransition></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject/foo.txt", prefix: "foodir/foobject/foo.txt",
expectedNonRec: true, expectedRec: true, want: true,
}, },
} }
@ -610,14 +609,10 @@ func TestHasActiveRules(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Got unexpected error: %v", err) t.Fatalf("Got unexpected error: %v", err)
} }
if got := lc.HasActiveRules(tc.prefix, false); got != tc.expectedNonRec { if got := lc.HasActiveRules(tc.prefix); got != tc.want {
t.Fatalf("Expected result with recursive set to false: `%v`, got: `%v`", tc.expectedNonRec, got) t.Fatalf("Expected result with recursive set to false: `%v`, got: `%v`", tc.want, got)
}
if got := lc.HasActiveRules(tc.prefix, true); got != tc.expectedRec {
t.Fatalf("Expected result with recursive set to true: `%v`, got: `%v`", tc.expectedRec, got)
} }
}) })
} }
} }
@ -741,7 +736,7 @@ func TestTransitionTier(t *testing.T) {
// Go back seven days in the past // Go back seven days in the past
now = now.Add(7 * 24 * time.Hour) now = now.Add(7 * 24 * time.Hour)
evt := lc.Eval(obj1, now) evt := lc.eval(obj1, now)
if evt.Action != TransitionAction { if evt.Action != TransitionAction {
t.Fatalf("Expected action: %s but got %s", TransitionAction, evt.Action) t.Fatalf("Expected action: %s but got %s", TransitionAction, evt.Action)
} }
@ -749,7 +744,7 @@ func TestTransitionTier(t *testing.T) {
t.Fatalf("Expected TIER-1 but got %s", evt.StorageClass) t.Fatalf("Expected TIER-1 but got %s", evt.StorageClass)
} }
evt = lc.Eval(obj2, now) evt = lc.eval(obj2, now)
if evt.Action != TransitionVersionAction { if evt.Action != TransitionVersionAction {
t.Fatalf("Expected action: %s but got %s", TransitionVersionAction, evt.Action) t.Fatalf("Expected action: %s but got %s", TransitionVersionAction, evt.Action)
} }
@ -818,13 +813,13 @@ func TestTransitionTierWithPrefixAndTags(t *testing.T) {
now = now.Add(7 * 24 * time.Hour) now = now.Add(7 * 24 * time.Hour)
// Eval object 1 // Eval object 1
evt := lc.Eval(obj1, now) evt := lc.eval(obj1, now)
if evt.Action != NoneAction { if evt.Action != NoneAction {
t.Fatalf("Expected action: %s but got %s", NoneAction, evt.Action) t.Fatalf("Expected action: %s but got %s", NoneAction, evt.Action)
} }
// Eval object 2 // Eval object 2
evt = lc.Eval(obj2, now) evt = lc.eval(obj2, now)
if evt.Action != TransitionAction { if evt.Action != TransitionAction {
t.Fatalf("Expected action: %s but got %s", TransitionAction, evt.Action) t.Fatalf("Expected action: %s but got %s", TransitionAction, evt.Action)
} }
@ -833,7 +828,7 @@ func TestTransitionTierWithPrefixAndTags(t *testing.T) {
} }
// Eval object 3 // Eval object 3
evt = lc.Eval(obj3, now) evt = lc.eval(obj3, now)
if evt.Action != TransitionAction { if evt.Action != TransitionAction {
t.Fatalf("Expected action: %s but got %s", TransitionAction, evt.Action) t.Fatalf("Expected action: %s but got %s", TransitionAction, evt.Action)
} }