minio/pkg/bucket/lifecycle/lifecycle_test.go
Poorna Krishnamoorthy 1ebf6f146a Add support for ILM transition (#10565)
This PR adds transition support for ILM
to transition data to another MinIO target
represented by a storage class ARN. Subsequent
GET or HEAD for that object will be streamed from
the transition tier. If PostRestoreObject API is
invoked, the transitioned object can be restored for
duration specified to the source cluster.
2020-11-19 18:47:17 -08:00

471 lines
19 KiB
Go

/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package lifecycle
import (
"bytes"
"encoding/xml"
"fmt"
"testing"
"time"
)
func TestParseAndValidateLifecycleConfig(t *testing.T) {
// Test for lifecycle config with more than 1000 rules
var manyRules []Rule
for i := 0; i < 1001; i++ {
rule := Rule{
ID: fmt.Sprintf("toManyRule%d", i),
Status: "Enabled",
Expiration: Expiration{Days: ExpirationDays(i)},
}
manyRules = append(manyRules, rule)
}
manyRuleLcConfig, err := xml.Marshal(Lifecycle{Rules: manyRules})
if err != nil {
t.Fatal("Failed to marshal lifecycle config with more than 1000 rules")
}
// Test for lifecycle config with rules containing overlapping prefixes
rule1 := Rule{
ID: "rule1",
Status: "Enabled",
Expiration: Expiration{Days: ExpirationDays(3)},
Filter: Filter{
Prefix: "/a/b",
},
}
rule2 := Rule{
ID: "rule2",
Status: "Enabled",
Expiration: Expiration{Days: ExpirationDays(3)},
Filter: Filter{
And: And{
Prefix: "/a/b/c",
},
},
}
overlappingRules := []Rule{rule1, rule2}
overlappingLcConfig, err := xml.Marshal(Lifecycle{Rules: overlappingRules})
if err != nil {
t.Fatal("Failed to marshal lifecycle config with rules having overlapping prefix")
}
// Test for lifecycle rules with duplicate IDs
rule3 := Rule{
ID: "duplicateID",
Status: "Enabled",
Expiration: Expiration{Days: ExpirationDays(3)},
Filter: Filter{
Prefix: "/a/b",
},
}
rule4 := Rule{
ID: "duplicateID",
Status: "Enabled",
Expiration: Expiration{Days: ExpirationDays(4)},
Filter: Filter{
And: And{
Prefix: "/x/z",
},
},
}
duplicateIDRules := []Rule{rule3, rule4}
duplicateIDLcConfig, err := xml.Marshal(Lifecycle{Rules: duplicateIDRules})
if err != nil {
t.Fatal("Failed to marshal lifecycle config of rules with duplicate ID.")
}
testCases := []struct {
inputConfig string
expectedParsingErr error
expectedValidationErr error
}{
{ // Valid lifecycle config
inputConfig: `<LifecycleConfiguration>
<Rule>
<ID>testRule1</ID>
<Filter>
<Prefix>prefix</Prefix>
</Filter>
<Status>Enabled</Status>
<Expiration><Days>3</Days></Expiration>
</Rule>
<Rule>
<ID>testRule2</ID>
<Filter>
<Prefix>another-prefix</Prefix>
</Filter>
<Status>Enabled</Status>
<Expiration><Days>3</Days></Expiration>
</Rule>
</LifecycleConfiguration>`,
expectedParsingErr: nil,
expectedValidationErr: nil,
},
{ // Valid lifecycle config
inputConfig: `<LifecycleConfiguration>
<Rule>
<Filter>
<And><Tag><Key>key1</Key><Value>val1</Value><Key>key2</Key><Value>val2</Value></Tag></And>
</Filter>
<Expiration><Days>3</Days></Expiration>
</Rule>
</LifecycleConfiguration>`,
expectedParsingErr: errDuplicatedXMLTag,
expectedValidationErr: nil,
},
{ // lifecycle config with no rules
inputConfig: `<LifecycleConfiguration>
</LifecycleConfiguration>`,
expectedParsingErr: nil,
expectedValidationErr: errLifecycleNoRule,
},
{ // lifecycle config with more than 1000 rules
inputConfig: string(manyRuleLcConfig),
expectedParsingErr: nil,
expectedValidationErr: errLifecycleTooManyRules,
},
{ // lifecycle config with rules having overlapping prefix
inputConfig: string(overlappingLcConfig),
expectedParsingErr: nil,
expectedValidationErr: nil,
},
{ // lifecycle config with rules having duplicate ID
inputConfig: string(duplicateIDLcConfig),
expectedParsingErr: nil,
expectedValidationErr: errLifecycleDuplicateID,
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig)))
if err != tc.expectedParsingErr {
t.Fatalf("%d: Expected %v during parsing but got %v", i+1, tc.expectedParsingErr, err)
}
if tc.expectedParsingErr != nil {
// We already expect a parsing error,
// no need to continue this test.
return
}
err = lc.Validate()
if err != tc.expectedValidationErr {
t.Fatalf("%d: Expected %v during parsing but got %v", i+1, tc.expectedValidationErr, err)
}
})
}
}
// TestMarshalLifecycleConfig checks if lifecycleconfig xml
// marshaling/unmarshaling can handle output from each other
func TestMarshalLifecycleConfig(t *testing.T) {
// Time at midnight UTC
midnightTS := ExpirationDate{time.Date(2019, time.April, 20, 0, 0, 0, 0, time.UTC)}
lc := Lifecycle{
Rules: []Rule{
{
Status: "Enabled",
Filter: Filter{Prefix: "prefix-1"},
Expiration: Expiration{Days: ExpirationDays(3)},
},
{
Status: "Enabled",
Filter: Filter{Prefix: "prefix-1"},
Expiration: Expiration{Date: ExpirationDate(midnightTS)},
},
},
}
b, err := xml.MarshalIndent(&lc, "", "\t")
if err != nil {
t.Fatal(err)
}
var lc1 Lifecycle
err = xml.Unmarshal(b, &lc1)
if err != nil {
t.Fatal(err)
}
ruleSet := make(map[string]struct{})
for _, rule := range lc.Rules {
ruleBytes, err := xml.Marshal(rule)
if err != nil {
t.Fatal(err)
}
ruleSet[string(ruleBytes)] = struct{}{}
}
for _, rule := range lc1.Rules {
ruleBytes, err := xml.Marshal(rule)
if err != nil {
t.Fatal(err)
}
if _, ok := ruleSet[string(ruleBytes)]; !ok {
t.Fatalf("Expected %v to be equal to %v, %v missing", lc, lc1, rule)
}
}
}
func TestExpectedExpiryTime(t *testing.T) {
testCases := []struct {
modTime time.Time
days ExpirationDays
expected time.Time
}{
{
time.Date(2020, time.March, 15, 10, 10, 10, 0, time.UTC),
4,
time.Date(2020, time.March, 20, 0, 0, 0, 0, time.UTC),
},
{
time.Date(2020, time.March, 15, 0, 0, 0, 0, time.UTC),
1,
time.Date(2020, time.March, 17, 0, 0, 0, 0, time.UTC),
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
got := ExpectedExpiryTime(tc.modTime, int(tc.days))
if !got.Equal(tc.expected) {
t.Fatalf("Expected %v to be equal to %v", got, tc.expected)
}
})
}
}
func TestComputeActions(t *testing.T) {
testCases := []struct {
inputConfig string
objectName string
objectTags string
objectModTime time.Time
expectedAction Action
}{
// Empty object name (unexpected case) should always return NoneAction
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>prefix</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
expectedAction: NoneAction,
},
// Disabled should always return NoneAction
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Disabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foodir/fooobject",
objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
expectedAction: NoneAction,
},
// No modTime, should be none-action
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foodir/fooobject",
expectedAction: NoneAction,
},
// Prefix not matched
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foxdir/fooobject",
objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
expectedAction: NoneAction,
},
// Test rule with empty prefix e.g. for whole bucket
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix></Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foxdir/fooobject/foo.txt",
objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
expectedAction: DeleteAction,
},
// Too early to remove (test Days)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foxdir/fooobject",
objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
expectedAction: NoneAction,
},
// Should remove (test Days)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foodir/fooobject",
objectModTime: time.Now().UTC().Add(-6 * 24 * time.Hour), // Created 6 days ago
expectedAction: DeleteAction,
},
// Too early to remove (test Date)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().UTC().Truncate(24*time.Hour).Add(24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foodir/fooobject",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: NoneAction,
},
// Should remove (test Days)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foodir/fooobject",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: DeleteAction,
},
// Should remove (Tags match)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><And><Prefix>foodir/</Prefix><Tag><Key>tag1</Key><Value>value1</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foodir/fooobject",
objectTags: "tag1=value1&tag2=value2",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: DeleteAction,
},
// Should remove (Multiple Rules, Tags match)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><And><Prefix>foodir/</Prefix><Tag><Key>tag1</Key><Value>value1</Value></Tag><Tag><Key>tag2</Key><Value>value2</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule><Rule><Filter><And><Prefix>abc/</Prefix><Tag><Key>tag2</Key><Value>value</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foodir/fooobject",
objectTags: "tag1=value1&tag2=value2",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: DeleteAction,
},
// Should remove (Tags match)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><And><Prefix>foodir/</Prefix><Tag><Key>tag1</Key><Value>value1</Value></Tag><Tag><Key>tag2</Key><Value>value2</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foodir/fooobject",
objectTags: "tag1=value1&tag2=value2",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: DeleteAction,
},
// Should remove (Tags match with inverted order)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><And><Tag><Key>factory</Key><Value>true</Value></Tag><Tag><Key>storeforever</Key><Value>false</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "fooobject",
objectTags: "storeforever=false&factory=true",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: DeleteAction,
},
// Should not remove (Tags don't match)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><And><Prefix>foodir/</Prefix><Tag><Key>tag</Key><Value>value1</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foodir/fooobject",
objectTags: "tag1=value1",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: NoneAction,
},
// Should not remove (Tags match, but prefix doesn't match)
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><And><Prefix>foodir/</Prefix><Tag><Key>tag1</Key><Value>value1</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foxdir/fooobject",
objectTags: "tag1=value1",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: NoneAction,
},
// Should remove - empty prefix, tags match, date expiration kicked in
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><And><Tag><Key>tag1</Key><Value>value1</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foxdir/fooobject",
objectTags: "tag1=value1",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: DeleteAction,
},
// Should remove - empty prefix, tags match, object is expired based on specified Days
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><And><Prefix></Prefix><Tag><Key>tag1</Key><Value>value1</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foxdir/fooobject",
objectTags: "tag1=value1",
objectModTime: time.Now().UTC().Add(-48 * time.Hour), // Created 2 day ago
expectedAction: DeleteAction,
},
// Should remove, the second rule has expiration kicked in
{
inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule><Rule><Filter><Prefix>foxdir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
objectName: "foxdir/fooobject",
objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
expectedAction: DeleteAction,
},
}
for _, tc := range testCases {
tc := tc
t.Run("", func(t *testing.T) {
lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig)))
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
if resultAction := lc.ComputeAction(ObjectOpts{
Name: tc.objectName,
UserTags: tc.objectTags,
ModTime: tc.objectModTime,
IsLatest: true,
}); resultAction != tc.expectedAction {
t.Fatalf("Expected action: `%v`, got: `%v`", tc.expectedAction, resultAction)
}
})
}
}
func TestHasActiveRules(t *testing.T) {
testCases := []struct {
inputConfig string
prefix string
expectedNonRec bool
expectedRec bool
}{
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject",
expectedNonRec: true, expectedRec: true,
},
{ // empty prefix
inputConfig: `<LifecycleConfiguration><Rule><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject/foo.txt",
expectedNonRec: true, expectedRec: true,
},
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "zdir/foobject",
expectedNonRec: false, expectedRec: false,
},
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/zdir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/",
expectedNonRec: false, expectedRec: true,
},
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix></Prefix></Filter><Status>Disabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/",
expectedNonRec: false, expectedRec: false,
},
{
inputConfig: `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>2999-01-01T00:00:00.000Z</Date></Expiration></Rule></LifecycleConfiguration>`,
prefix: "foodir/foobject",
expectedNonRec: false, expectedRec: false,
},
}
for i, tc := range testCases {
tc := tc
t.Run(fmt.Sprintf("Test_%d", i+1), func(t *testing.T) {
lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig)))
if err != nil {
t.Fatalf("Got unexpected error: %v", err)
}
if got := lc.HasActiveRules(tc.prefix, false); got != tc.expectedNonRec {
t.Fatalf("Expected result with recursive set to false: `%v`, got: `%v`", tc.expectedNonRec, got)
}
if got := lc.HasActiveRules(tc.prefix, true); got != tc.expectedRec {
t.Fatalf("Expected result with recursive set to true: `%v`, got: `%v`", tc.expectedRec, got)
}
})
}
}