Add bucket lifecycle expiry feature (#7834)

pkg/iam/policy/action.go

@@ -76,6 +76,12 @@ const (
 	// ListMultipartUploadPartsAction - ListParts Rest API action.
 	ListMultipartUploadPartsAction = "s3:ListMultipartUploadParts"
 
+	// PutBucketLifecycleAction - PutBucketLifecycle Rest API action.
+	PutBucketLifecycleAction = "s3:PutBucketLifecycle"
+
+	// GetBucketLifecycleAction - GetBucketLifecycle Rest API action.
+	GetBucketLifecycleAction = "s3:GetBucketLifecycle"
+
 	// PutBucketNotificationAction - PutObjectNotification Rest API action.
 	PutBucketNotificationAction = "s3:PutBucketNotification"
 
@@ -110,6 +116,8 @@ var supportedActions = map[Action]struct{}{
 	PutBucketNotificationAction:      {},
 	PutBucketPolicyAction:            {},
 	PutObjectAction:                  {},
+	GetBucketLifecycleAction:         {},
+	PutBucketLifecycleAction:         {},
 }
 
 // isObjectAction - returns whether action is object type or not.

pkg/lifecycle/action.go

@@ -1,33 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package lifecycle
-
-// Action - policy action.
-// Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazons3.html
-// for more information about available actions.
-type Action string
-
-const (
-	// PutBucketLifecycleAction - PutBucketLifecycle Rest API action.
-	PutBucketLifecycleAction = "s3:PutBucketLifecycle"
-
-	// GetBucketLifecycleAction - GetBucketLifecycle Rest API action.
-	GetBucketLifecycleAction = "s3:GetBucketLifecycle"
-
-	// DeleteBucketLifecycleAction - DeleteBucketLifecycleAction Rest API action.
-	DeleteBucketLifecycleAction = "s3:DeleteBucketLifecycle"
-)

pkg/lifecycle/expiration.go

@@ -107,13 +107,29 @@ type Expiration struct {
 // Validate - validates the "Expiration" element
 func (e Expiration) Validate() error {
 	// Neither expiration days or date is specified
-	if e.Days == ExpirationDays(0) && e.Date == (ExpirationDate{time.Time{}}) {
+	if e.IsDaysNull() && e.IsDateNull() {
 		return errLifecycleInvalidExpiration
 	}
 
 	// Both expiration days and date are specified
-	if e.Days != ExpirationDays(0) && e.Date != (ExpirationDate{time.Time{}}) {
+	if !e.IsDaysNull() && !e.IsDateNull() {
 		return errLifecycleInvalidExpiration
 	}
 	return nil
 }
+
+// IsDaysNull returns true if days field is null
+func (e Expiration) IsDaysNull() bool {
+	return e.Days == ExpirationDays(0)
+
+}
+
+// IsDateNull returns true if date field is null
+func (e Expiration) IsDateNull() bool {
+	return e.Date == ExpirationDate{time.Time{}}
+}
+
+// IsNull returns true if both date and days fields are null
+func (e Expiration) IsNull() bool {
+	return e.IsDaysNull() && e.IsDateNull()
+}

pkg/lifecycle/lifecycle.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"io"
 	"strings"
+	"time"
 )
 
 var (
@@ -29,6 +30,17 @@ var (
 	errLifecycleOverlappingPrefix = errors.New("Lifecycle configuration has rules with overlapping prefix")
 )
 
+// Action represents a delete action or other transition
+// actions that will be implemented later.
+type Action int
+
+const (
+	// NoneAction means no action required after evaluting lifecycle rules
+	NoneAction Action = iota
+	// DeleteAction means the object needs to be removed after evaluting lifecycle rules
+	DeleteAction
+)
+
 // Lifecycle - Configuration for bucket lifecycle.
 type Lifecycle struct {
 	XMLName xml.Name `xml:"LifecycleConfiguration"`
@@ -84,3 +96,35 @@ func (lc Lifecycle) Validate() error {
 	}
 	return nil
 }
+
+// FilterRuleActions returns the expiration and transition from the object name
+// after evaluating all rules.
+func (lc Lifecycle) FilterRuleActions(objName string) (Expiration, Transition) {
+	for _, rule := range lc.Rules {
+		if strings.ToLower(rule.Status) != "enabled" {
+			continue
+		}
+		if strings.HasPrefix(objName, rule.Filter.Prefix) {
+			return rule.Expiration, Transition{}
+		}
+	}
+	return Expiration{}, Transition{}
+}
+
+// ComputeAction returns the action to perform by evaluating all lifecycle rules
+// against the object name and its modification time.
+func (lc Lifecycle) ComputeAction(objName string, modTime time.Time) Action {
+	var action = NoneAction
+	exp, _ := lc.FilterRuleActions(objName)
+	if !exp.IsDateNull() {
+		if time.Now().After(exp.Date.Time) {
+			action = DeleteAction
+		}
+	}
+	if !exp.IsDaysNull() {
+		if time.Now().After(modTime.Add(time.Duration(exp.Days) * 24 * time.Hour)) {
+			action = DeleteAction
+		}
+	}
+	return action
+}

pkg/lifecycle/lifecycle_test.go

@@ -158,3 +158,73 @@ func TestMarshalLifecycleConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestComputeActions(t *testing.T) {
+	testCases := []struct {
+		inputConfig    string
+		objectName     string
+		objectModTime  time.Time
+		expectedAction Action
+	}{
+		// Empty object name (unexpected case) should always return NoneAction
+		{
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>prefix</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
+			expectedAction: NoneAction,
+		},
+		// Disabled should always return NoneAction
+		{
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Disabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
+			objectName:     "foodir/fooobject",
+			objectModTime:  time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			expectedAction: NoneAction,
+		},
+		// Prefix not matched
+		{
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
+			objectName:     "foxdir/fooobject",
+			objectModTime:  time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			expectedAction: NoneAction,
+		},
+		// Too early to remove (test Days)
+		{
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
+			objectName:     "foxdir/fooobject",
+			objectModTime:  time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			expectedAction: NoneAction,
+		},
+		// Should remove (test Days)
+		{
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>5</Days></Expiration></Rule></LifecycleConfiguration>`,
+			objectName:     "foodir/fooobject",
+			objectModTime:  time.Now().UTC().Add(-6 * 24 * time.Hour), // Created 6 days ago
+			expectedAction: DeleteAction,
+		},
+		// Too early to remove (test Date)
+		{
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
+			objectName:     "foodir/fooobject",
+			objectModTime:  time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
+			expectedAction: NoneAction,
+		},
+		// Should remove (test Days)
+		{
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
+			objectName:     "foodir/fooobject",
+			objectModTime:  time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
+			expectedAction: DeleteAction,
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
+			lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig)))
+			if err != nil {
+				t.Fatalf("%d: Got unexpected error: %v", i+1, err)
+			}
+			if resultAction := lc.ComputeAction(tc.objectName, tc.objectModTime); resultAction != tc.expectedAction {
+				t.Fatalf("%d: Expected action: `%v`, got: `%v`", i+1, tc.expectedAction, resultAction)
+			}
+		})
+
+	}
+}

pkg/policy/action.go

@@ -83,6 +83,12 @@ const (
 
 	// PutObjectAction - PutObject Rest API action.
 	PutObjectAction = "s3:PutObject"
+
+	// PutBucketLifecycleAction - PutBucketLifecycle Rest API action.
+	PutBucketLifecycleAction = "s3:PutBucketLifecycle"
+
+	// GetBucketLifecycleAction - GetBucketLifecycle Rest API action.
+	GetBucketLifecycleAction = "s3:GetBucketLifecycle"
 )
 
 // isObjectAction - returns whether action is object type or not.
@@ -113,6 +119,8 @@ func (action Action) IsValid() bool {
 	case ListMultipartUploadPartsAction, PutBucketNotificationAction:
 		fallthrough
 	case PutBucketPolicyAction, PutObjectAction:
+		fallthrough
+	case PutBucketLifecycleAction, GetBucketLifecycleAction:
 		return true
 	}