// Copyright (c) 2015-2021 MinIO, Inc. // // This file is part of MinIO Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package lifecycle import ( "bytes" "encoding/xml" "fmt" "net/http" "net/http/httptest" "strconv" "strings" "testing" "time" "github.com/google/uuid" xhttp "github.com/minio/minio/internal/http" ) func TestParseAndValidateLifecycleConfig(t *testing.T) { testCases := []struct { inputConfig string expectedParsingErr error expectedValidationErr error }{ { // Valid lifecycle config inputConfig: ` testRule1 prefix Enabled 3 testRule2 another-prefix Enabled 3 `, expectedParsingErr: nil, expectedValidationErr: nil, }, { // Valid lifecycle config inputConfig: ` key1val1key2val2 3 `, expectedParsingErr: errDuplicatedXMLTag, expectedValidationErr: nil, }, { // lifecycle config with no rules inputConfig: ` `, expectedParsingErr: nil, expectedValidationErr: errLifecycleNoRule, }, { // lifecycle config with rules having overlapping prefix inputConfig: `rule1Enabled/a/b3rule2Enabled/a/b/ckey1val13 `, expectedParsingErr: nil, expectedValidationErr: nil, }, { // lifecycle config with rules having duplicate ID inputConfig: `duplicateIDEnabled/a/b3duplicateIDEnabled/x/zkey1val14`, expectedParsingErr: nil, expectedValidationErr: errLifecycleDuplicateID, }, // Missing in { inputConfig: `sample-rule-2/a/b/cEnabled1`, expectedParsingErr: nil, expectedValidationErr: errXMLNotWellFormed, }, // Lifecycle with the deprecated Prefix tag { inputConfig: `ruleEnabled1`, expectedParsingErr: nil, expectedValidationErr: nil, }, // Lifecycle with empty Filter tag { inputConfig: `ruleEnabled1`, expectedParsingErr: nil, expectedValidationErr: nil, }, // Lifecycle with zero Transition Days { inputConfig: `ruleEnabled0S3TIER-1`, expectedParsingErr: nil, expectedValidationErr: nil, }, // Lifecycle with max noncurrent versions { inputConfig: `rule>Enabled5`, expectedParsingErr: nil, expectedValidationErr: nil, }, } for i, tc := range testCases { t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig))) if err != tc.expectedParsingErr { t.Fatalf("%d: Expected %v during parsing but got %v", i+1, tc.expectedParsingErr, err) } if tc.expectedParsingErr != nil { // We already expect a parsing error, // no need to continue this test. return } err = lc.Validate() if err != tc.expectedValidationErr { t.Fatalf("%d: Expected %v during validation but got %v", i+1, tc.expectedValidationErr, err) } }) } } // TestMarshalLifecycleConfig checks if lifecycleconfig xml // marshaling/unmarshaling can handle output from each other func TestMarshalLifecycleConfig(t *testing.T) { // Time at midnight UTC midnightTS := ExpirationDate{time.Date(2019, time.April, 20, 0, 0, 0, 0, time.UTC)} lc := Lifecycle{ Rules: []Rule{ { Status: "Enabled", Filter: Filter{Prefix: Prefix{string: "prefix-1", set: true}}, Expiration: Expiration{Days: ExpirationDays(3)}, }, { Status: "Enabled", Filter: Filter{Prefix: Prefix{string: "prefix-1", set: true}}, Expiration: Expiration{Date: midnightTS}, }, { Status: "Enabled", Filter: Filter{Prefix: Prefix{string: "prefix-1", set: true}}, Expiration: Expiration{Date: midnightTS}, NoncurrentVersionTransition: NoncurrentVersionTransition{NoncurrentDays: TransitionDays(2), StorageClass: "TEST"}, }, }, } b, err := xml.MarshalIndent(&lc, "", "\t") if err != nil { t.Fatal(err) } var lc1 Lifecycle err = xml.Unmarshal(b, &lc1) if err != nil { t.Fatal(err) } ruleSet := make(map[string]struct{}) for _, rule := range lc.Rules { ruleBytes, err := xml.Marshal(rule) if err != nil { t.Fatal(err) } ruleSet[string(ruleBytes)] = struct{}{} } for _, rule := range lc1.Rules { ruleBytes, err := xml.Marshal(rule) if err != nil { t.Fatal(err) } if _, ok := ruleSet[string(ruleBytes)]; !ok { t.Fatalf("Expected %v to be equal to %v, %v missing", lc, lc1, rule) } } } func TestExpectedExpiryTime(t *testing.T) { testCases := []struct { modTime time.Time days ExpirationDays expected time.Time }{ { time.Date(2020, time.March, 15, 10, 10, 10, 0, time.UTC), 4, time.Date(2020, time.March, 20, 0, 0, 0, 0, time.UTC), }, { time.Date(2020, time.March, 15, 0, 0, 0, 0, time.UTC), 1, time.Date(2020, time.March, 17, 0, 0, 0, 0, time.UTC), }, } for i, tc := range testCases { t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { got := ExpectedExpiryTime(tc.modTime, int(tc.days)) if !got.Equal(tc.expected) { t.Fatalf("Expected %v to be equal to %v", got, tc.expected) } }) } } func TestComputeActions(t *testing.T) { testCases := []struct { inputConfig string objectName string objectTags string objectModTime time.Time isExpiredDelMarker bool expectedAction Action isNoncurrent bool objectSuccessorModTime time.Time versionID string }{ // Empty object name (unexpected case) should always return NoneAction { inputConfig: `prefixEnabled5`, expectedAction: NoneAction, }, // Disabled should always return NoneAction { inputConfig: `foodir/Disabled5`, objectName: "foodir/fooobject", objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago expectedAction: NoneAction, }, // No modTime, should be none-action { inputConfig: `foodir/Enabled5`, objectName: "foodir/fooobject", expectedAction: NoneAction, }, // Prefix not matched { inputConfig: `foodir/Enabled5`, objectName: "foxdir/fooobject", objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago expectedAction: NoneAction, }, // Test rule with empty prefix e.g. for whole bucket { inputConfig: `Enabled5`, objectName: "foxdir/fooobject/foo.txt", objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago expectedAction: DeleteAction, }, // Too early to remove (test Days) { inputConfig: `foodir/Enabled5`, objectName: "foxdir/fooobject", objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago expectedAction: NoneAction, }, // Should remove (test Days) { inputConfig: `foodir/Enabled5`, objectName: "foodir/fooobject", objectModTime: time.Now().UTC().Add(-6 * 24 * time.Hour), // Created 6 days ago expectedAction: DeleteAction, }, // Too early to remove (test Date) { inputConfig: `foodir/Enabled` + time.Now().UTC().Truncate(24*time.Hour).Add(24*time.Hour).Format(time.RFC3339) + ``, objectName: "foodir/fooobject", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: NoneAction, }, // Should remove (test Days) { inputConfig: `foodir/Enabled` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "foodir/fooobject", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: DeleteAction, }, // Should remove (Tags match) { inputConfig: `foodir/tag1value1Enabled` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "foodir/fooobject", objectTags: "tag1=value1&tag2=value2", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: DeleteAction, }, // Should remove (Multiple Rules, Tags match) { inputConfig: `foodir/tag1value1tag2value2Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `abc/tag2valueEnabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "foodir/fooobject", objectTags: "tag1=value1&tag2=value2", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: DeleteAction, }, // Should remove (Tags match) { inputConfig: `foodir/tag1value1tag2value2Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "foodir/fooobject", objectTags: "tag1=value1&tag2=value2", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: DeleteAction, }, // Should remove (Tags match with inverted order) { inputConfig: `factorytruestoreforeverfalseEnabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "fooobject", objectTags: "storeforever=false&factory=true", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: DeleteAction, }, // Should remove (Tags with encoded chars) { inputConfig: `factorytruestore foreverfalseEnabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "fooobject", objectTags: "store+forever=false&factory=true", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: DeleteAction, }, // Should not remove (Tags don't match) { inputConfig: `foodir/tagvalue1Enabled` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "foodir/fooobject", objectTags: "tag1=value1", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: NoneAction, }, // Should not remove (Tags match, but prefix doesn't match) { inputConfig: `foodir/tag1value1Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "foxdir/fooobject", objectTags: "tag1=value1", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: NoneAction, }, // Should remove - empty prefix, tags match, date expiration kicked in { inputConfig: `tag1value1Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "foxdir/fooobject", objectTags: "tag1=value1", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: DeleteAction, }, // Should remove - empty prefix, tags match, object is expired based on specified Days { inputConfig: `tag1value1Enabled1`, objectName: "foxdir/fooobject", objectTags: "tag1=value1", objectModTime: time.Now().UTC().Add(-48 * time.Hour), // Created 2 day ago expectedAction: DeleteAction, }, // Should remove, the second rule has expiration kicked in { inputConfig: `Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(24*time.Hour).Format(time.RFC3339) + `foxdir/Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "foxdir/fooobject", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: DeleteAction, }, // Should accept BucketLifecycleConfiguration root tag { inputConfig: `foodir/Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, objectName: "foodir/fooobject", objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago expectedAction: DeleteAction, }, // Should delete expired delete marker right away { inputConfig: `trueEnabled`, objectName: "foodir/fooobject", objectModTime: time.Now().UTC().Add(-1 * time.Hour), // Created one hour ago isExpiredDelMarker: true, expectedAction: DeleteVersionAction, }, // Should not delete expired marker if its time has not come yet { inputConfig: `Enabled1`, objectName: "foodir/fooobject", objectModTime: time.Now().UTC().Add(-12 * time.Hour), // Created 12 hours ago isExpiredDelMarker: true, expectedAction: NoneAction, }, // Should delete expired marker since its time has come { inputConfig: `Enabled1`, objectName: "foodir/fooobject", objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago isExpiredDelMarker: true, expectedAction: DeleteVersionAction, }, // Should transition immediately when Transition days is zero { inputConfig: `Enabled0S3TIER-1`, objectName: "foodir/fooobject", objectModTime: time.Now().Add(-1 * time.Nanosecond).UTC(), // Created now expectedAction: TransitionAction, }, // Should transition immediately when NoncurrentVersion Transition days is zero { inputConfig: `Enabled0S3TIER-1`, objectName: "foodir/fooobject", objectModTime: time.Now().Add(-1 * time.Nanosecond).UTC(), // Created now expectedAction: TransitionVersionAction, isNoncurrent: true, objectSuccessorModTime: time.Now().Add(-1 * time.Nanosecond).UTC(), versionID: uuid.New().String(), }, // Lifecycle rules with NewerNoncurrentVersions specified must return NoneAction. { inputConfig: `foodir/Enabled5`, objectName: "foodir/fooobject", versionID: uuid.NewString(), objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago expectedAction: NoneAction, }, // Disabled rules with NewerNoncurrentVersions shouldn't affect outcome. { inputConfig: `foodir/Enabled5foodir/Disabled5`, objectName: "foodir/fooobject", versionID: uuid.NewString(), objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago objectSuccessorModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago isNoncurrent: true, expectedAction: DeleteVersionAction, }, } for _, tc := range testCases { tc := tc t.Run("", func(t *testing.T) { lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig))) if err != nil { t.Fatalf("Got unexpected error: %v", err) } if resultAction := lc.ComputeAction(ObjectOpts{ Name: tc.objectName, UserTags: tc.objectTags, ModTime: tc.objectModTime, DeleteMarker: tc.isExpiredDelMarker, NumVersions: 1, IsLatest: !tc.isNoncurrent, SuccessorModTime: tc.objectSuccessorModTime, VersionID: tc.versionID, }); resultAction != tc.expectedAction { t.Fatalf("Expected action: `%v`, got: `%v`", tc.expectedAction, resultAction) } }) } } func TestHasActiveRules(t *testing.T) { testCases := []struct { inputConfig string prefix string expectedNonRec bool expectedRec bool }{ { inputConfig: `foodir/Enabled5`, prefix: "foodir/foobject", expectedNonRec: true, expectedRec: true, }, { // empty prefix inputConfig: `Enabled5`, prefix: "foodir/foobject/foo.txt", expectedNonRec: true, expectedRec: true, }, { inputConfig: `foodir/Enabled5`, prefix: "zdir/foobject", expectedNonRec: false, expectedRec: false, }, { inputConfig: `foodir/zdir/Enabled5`, prefix: "foodir/", expectedNonRec: false, expectedRec: true, }, { inputConfig: `Disabled5`, prefix: "foodir/", expectedNonRec: false, expectedRec: false, }, { inputConfig: `foodir/Enabled2999-01-01T00:00:00.000Z`, prefix: "foodir/foobject", expectedNonRec: false, expectedRec: false, }, { inputConfig: `EnabledS3TIER-1`, prefix: "foodir/foobject/foo.txt", expectedNonRec: true, expectedRec: true, }, { inputConfig: `EnabledS3TIER-1`, prefix: "foodir/foobject/foo.txt", expectedNonRec: true, expectedRec: true, }, } for i, tc := range testCases { tc := tc t.Run(fmt.Sprintf("Test_%d", i+1), func(t *testing.T) { lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig))) if err != nil { t.Fatalf("Got unexpected error: %v", err) } if got := lc.HasActiveRules(tc.prefix, false); got != tc.expectedNonRec { t.Fatalf("Expected result with recursive set to false: `%v`, got: `%v`", tc.expectedNonRec, got) } if got := lc.HasActiveRules(tc.prefix, true); got != tc.expectedRec { t.Fatalf("Expected result with recursive set to true: `%v`, got: `%v`", tc.expectedRec, got) } }) } } func TestSetPredictionHeaders(t *testing.T) { lc := Lifecycle{ Rules: []Rule{ { ID: "rule-1", Status: "Enabled", Expiration: Expiration{ Days: ExpirationDays(3), set: true, }, }, { ID: "rule-2", Status: "Enabled", Transition: Transition{ Days: TransitionDays(3), StorageClass: "TIER-1", set: true, }, }, { ID: "rule-3", Status: "Enabled", NoncurrentVersionTransition: NoncurrentVersionTransition{ NoncurrentDays: TransitionDays(5), StorageClass: "TIER-2", set: true, }, }, }, } // current version obj1 := ObjectOpts{ Name: "obj1", IsLatest: true, } // non-current version obj2 := ObjectOpts{ Name: "obj2", } tests := []struct { obj ObjectOpts expRuleID int transRuleID int }{ { obj: obj1, expRuleID: 0, transRuleID: 1, }, { obj: obj2, expRuleID: 0, transRuleID: 2, }, } for i, tc := range tests { w := httptest.NewRecorder() lc.SetPredictionHeaders(w, tc.obj) if expHdrs, ok := w.Header()[xhttp.AmzExpiration]; ok && !strings.Contains(expHdrs[0], lc.Rules[tc.expRuleID].ID) { t.Fatalf("Test %d: Expected %s header", i+1, xhttp.AmzExpiration) } if transHdrs, ok := w.Header()[xhttp.MinIOTransition]; ok { if !strings.Contains(transHdrs[0], lc.Rules[tc.transRuleID].ID) { t.Fatalf("Test %d: Expected %s header", i+1, xhttp.MinIOTransition) } if tc.obj.IsLatest { if expectedDue, _ := lc.Rules[tc.transRuleID].Transition.NextDue(tc.obj); !strings.Contains(transHdrs[0], expectedDue.Format(http.TimeFormat)) { t.Fatalf("Test %d: Expected transition time %s", i+1, expectedDue) } } else { if expectedDue, _ := lc.Rules[tc.transRuleID].NoncurrentVersionTransition.NextDue(tc.obj); !strings.Contains(transHdrs[0], expectedDue.Format(http.TimeFormat)) { t.Fatalf("Test %d: Expected transition time %s", i+1, expectedDue) } } } } } func TestTransitionTier(t *testing.T) { lc := Lifecycle{ Rules: []Rule{ { ID: "rule-1", Status: "Enabled", Transition: Transition{ Days: TransitionDays(3), StorageClass: "TIER-1", }, }, { ID: "rule-2", Status: "Enabled", NoncurrentVersionTransition: NoncurrentVersionTransition{ NoncurrentDays: TransitionDays(3), StorageClass: "TIER-2", }, }, }, } obj1 := ObjectOpts{ Name: "obj1", IsLatest: true, } obj2 := ObjectOpts{ Name: "obj2", } if got := lc.TransitionTier(obj1); got != "TIER-1" { t.Fatalf("Expected TIER-1 but got %s", got) } if got := lc.TransitionTier(obj2); got != "TIER-2" { t.Fatalf("Expected TIER-2 but got %s", got) } } func TestNoncurrentVersionsLimit(t *testing.T) { // test that the lowest max noncurrent versions limit is returned among // matching rules var rules []Rule for i := 1; i <= 10; i++ { rules = append(rules, Rule{ ID: strconv.Itoa(i), Status: "Enabled", NoncurrentVersionExpiration: NoncurrentVersionExpiration{ NewerNoncurrentVersions: i, NoncurrentDays: ExpirationDays(i), }, }) } lc := Lifecycle{ Rules: rules, } if ruleID, days, lim := lc.NoncurrentVersionsExpirationLimit(ObjectOpts{Name: "obj"}); ruleID != "1" || days != 1 || lim != 10 { t.Fatalf("Expected (ruleID, days, lim) to be (\"1\", 1, 10) but got (%s, %d, %d)", ruleID, days, lim) } } func TestMaxNoncurrentBackwardCompat(t *testing.T) { testCases := []struct { xml string expected NoncurrentVersionExpiration }{ { xml: `13`, expected: NoncurrentVersionExpiration{ XMLName: xml.Name{ Local: "NoncurrentVersionExpiration", }, NoncurrentDays: 1, NewerNoncurrentVersions: 3, set: true, }, }, { xml: `24`, expected: NoncurrentVersionExpiration{ XMLName: xml.Name{ Local: "NoncurrentVersionExpiration", }, NoncurrentDays: 2, NewerNoncurrentVersions: 4, set: true, }, }, } for i, tc := range testCases { var got NoncurrentVersionExpiration dec := xml.NewDecoder(strings.NewReader(tc.xml)) if err := dec.Decode(&got); err != nil || got != tc.expected { if err != nil { t.Fatalf("%d: Failed to unmarshal xml %v", i+1, err) } t.Fatalf("%d: Expected %v but got %v", i+1, tc.expected, got) } } } func TestParseLifecycleConfigWithID(t *testing.T) { r := bytes.NewReader([]byte(` rule-1 prefix Enabled 3 another-prefix Enabled 3 `)) lc, err := ParseLifecycleConfigWithID(r) if err != nil { t.Fatalf("Expected parsing to succeed but failed with %v", err) } for _, rule := range lc.Rules { if rule.ID == "" { t.Fatalf("Expected all rules to have a unique id assigned %#v", rule) } } }