mirror of https://github.com/minio/minio.git
Add initial support for bucket lifecycle (#7563)
This PR is based off @sinhaashish's PR for object lifecycle management, which includes support only for, - Expiration of object - Filter using object prefix (_not_ object tags) N B the code for actual expiration of objects will be included in a subsequent PR.
This commit is contained in:
Krishnan Parthasarathi
kannappanr
parent
59e1763816
commit
559a59220e
|
|
@ -92,6 +92,7 @@ const (
|
|||
ErrMissingRequestBodyError
|
||||
ErrNoSuchBucket
|
||||
ErrNoSuchBucketPolicy
|
||||
ErrNoSuchBucketLifecycle
|
||||
ErrNoSuchKey
|
||||
ErrNoSuchUpload
|
||||
ErrNoSuchVersion
|
||||
|
|
@ -467,6 +468,11 @@ var errorCodes = errorCodeMap{
|
|||
Description: "The bucket policy does not exist",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNoSuchBucketLifecycle: {
|
||||
Code: "NoSuchBucketLifecycle",
|
||||
Description: "The bucket lifecycle configuration does not exist",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNoSuchKey: {
|
||||
Code: "NoSuchKey",
|
||||
Description: "The specified key does not exist.",
|
||||
|
|
@ -1627,6 +1633,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||
apiErr = ErrUnsupportedMetadata
|
||||
case BucketPolicyNotFound:
|
||||
apiErr = ErrNoSuchBucketPolicy
|
||||
case BucketLifecycleNotFound:
|
||||
apiErr = ErrNoSuchBucketLifecycle
|
||||
case *event.ErrInvalidEventName:
|
||||
apiErr = ErrEventNotification
|
||||
case *event.ErrInvalidARN:
|
||||
|
|
|
|||
|
|
@ -91,7 +91,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
|
|||
// GetBucketLocation
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
|
||||
// GetBucketPolicy
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
|
||||
// GetBucketLifecycle
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
|
||||
|
||||
// Dummy Bucket Calls
|
||||
// GetBucketACL -- this is a dummy call.
|
||||
|
|
@ -128,9 +130,12 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
|
|||
// ListObjectsV2
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
|
||||
// PutBucketLifecycle
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketLifecycleHandler)).Queries("lifecycle", "")
|
||||
// PutBucketPolicy
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
|
||||
|
||||
// PutBucketNotification
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
|
||||
// PutBucket
|
||||
|
|
@ -142,7 +147,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
|
|||
// DeleteMultipleObjects
|
||||
bucket.Methods(http.MethodPost).HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
|
||||
// DeleteBucketPolicy
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
|
||||
// DeleteBucketLifecycle
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketLifecycleHandler)).Queries("lifecycle", "")
|
||||
// DeleteBucket
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -800,6 +800,8 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
|||
globalNotificationSys.RemoveNotification(bucket)
|
||||
globalPolicySys.Remove(bucket)
|
||||
globalNotificationSys.DeleteBucket(ctx, bucket)
|
||||
globalLifecycleSys.Remove(bucket)
|
||||
globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
|
||||
|
||||
// Write success response.
|
||||
writeSuccessNoContent(w)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,164 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
)
|
||||
|
||||
const (
|
||||
// Lifecycle configuration file.
|
||||
bucketLifecycleConfig = "lifecycle.xml"
|
||||
)
|
||||
|
||||
// PutBucketLifecycleHandler - This HTTP handler stores given bucket lifecycle configuration as per
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html
|
||||
func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketLifecycle")
|
||||
|
||||
defer logger.AuditLog(w, r, "PutBucketLifecycle", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, lifecycle.PutBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// PutBucketLifecycle always needs a Content-Md5
|
||||
if _, ok := r.Header["Content-Md5"]; !ok {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = objAPI.SetBucketLifecycle(ctx, bucket, bucketLifecycle); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
globalLifecycleSys.Set(bucket, *bucketLifecycle)
|
||||
globalNotificationSys.SetBucketLifecycle(ctx, bucket, bucketLifecycle)
|
||||
|
||||
// Success.
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
// GetBucketLifecycleHandler - This HTTP handler returns bucket policy configuration.
|
||||
func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketLifecycle")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketLifecycle", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, lifecycle.GetBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Read bucket access lifecycle.
|
||||
bucketLifecycle, err := objAPI.GetBucketLifecycle(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
lifecycleData, err := xml.Marshal(bucketLifecycle)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Write lifecycle configuration to client.
|
||||
writeSuccessResponseXML(w, lifecycleData)
|
||||
}
|
||||
|
||||
// DeleteBucketLifecycleHandler - This HTTP handler removes bucket lifecycle configuration.
|
||||
func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucketLifecycle")
|
||||
|
||||
defer logger.AuditLog(w, r, "DeleteBucketLifecycle", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, lifecycle.DeleteBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err := objAPI.DeleteBucketLifecycle(ctx, bucket); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
globalLifecycleSys.Remove(bucket)
|
||||
globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
|
||||
|
||||
// Success.
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
|
@ -72,12 +72,6 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
|
|||
w.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
// GetBucketLifecycleHandler - GET bucket lifecycle, a dummy api
|
||||
func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
// GetBucketReplicationHandler - GET bucket replication, a dummy api
|
||||
func (api objectAPIHandlers) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
|
|
|
|||
16
cmd/fs-v1.go
16
cmd/fs-v1.go
|
|
@ -32,6 +32,7 @@ import (
|
|||
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/mimedb"
|
||||
|
|
@ -1170,6 +1171,21 @@ func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
|
|||
return removePolicyConfig(ctx, fs, bucket)
|
||||
}
|
||||
|
||||
// SetBucketLifecycle sets lifecycle on bucket
|
||||
func (fs *FSObjects) SetBucketLifecycle(ctx context.Context, bucket string, lifecycle *lifecycle.Lifecycle) error {
|
||||
return saveLifecycleConfig(ctx, fs, bucket, lifecycle)
|
||||
}
|
||||
|
||||
// GetBucketLifecycle will get lifecycle on bucket
|
||||
func (fs *FSObjects) GetBucketLifecycle(ctx context.Context, bucket string) (*lifecycle.Lifecycle, error) {
|
||||
return getLifecycleConfig(fs, bucket)
|
||||
}
|
||||
|
||||
// DeleteBucketLifecycle deletes all lifecycle on bucket
|
||||
func (fs *FSObjects) DeleteBucketLifecycle(ctx context.Context, bucket string) error {
|
||||
return removeLifecycleConfig(ctx, fs, bucket)
|
||||
}
|
||||
|
||||
// ListObjectsV2 lists all blobs in bucket filtered by prefix
|
||||
func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
|
||||
marker := continuationToken
|
||||
|
|
|
|||
|
|
@ -317,6 +317,8 @@ func ErrorRespToObjectError(err error, params ...string) error {
|
|||
err = BucketNotEmpty{}
|
||||
case "NoSuchBucketPolicy":
|
||||
err = BucketPolicyNotFound{}
|
||||
case "NoSuchBucketLifecycle":
|
||||
err = BucketLifecycleNotFound{}
|
||||
case "InvalidBucketName":
|
||||
err = BucketNameInvalid{Bucket: bucket}
|
||||
case "InvalidPart":
|
||||
|
|
|
|||
|
|
@ -269,6 +269,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
|||
// Initialize policy system.
|
||||
go globalPolicySys.Init(newObject)
|
||||
|
||||
// Create new lifecycle system
|
||||
globalLifecycleSys = NewLifecycleSys()
|
||||
|
||||
// Create new notification system.
|
||||
globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
|
||||
if globalEtcdClient != nil && newObject.IsNotificationSupported() {
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import (
|
|||
"context"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
|
@ -81,6 +82,22 @@ func (a GatewayUnsupported) DeleteBucketPolicy(ctx context.Context, bucket strin
|
|||
return NotImplemented{}
|
||||
}
|
||||
|
||||
// SetBucketLifecycle sets lifecycle on bucket
|
||||
func (a GatewayUnsupported) SetBucketLifecycle(ctx context.Context, bucket string, lifecycle *lifecycle.Lifecycle) error {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return NotImplemented{}
|
||||
}
|
||||
|
||||
// GetBucketLifecycle will get lifecycle on bucket
|
||||
func (a GatewayUnsupported) GetBucketLifecycle(ctx context.Context, bucket string) (*lifecycle.Lifecycle, error) {
|
||||
return nil, NotImplemented{}
|
||||
}
|
||||
|
||||
// DeleteBucketLifecycle deletes all lifecycle on bucket
|
||||
func (a GatewayUnsupported) DeleteBucketLifecycle(ctx context.Context, bucket string) error {
|
||||
return NotImplemented{}
|
||||
}
|
||||
|
||||
// ReloadFormat - Not implemented stub.
|
||||
func (a GatewayUnsupported) ReloadFormat(ctx context.Context, dryRun bool) error {
|
||||
return NotImplemented{}
|
||||
|
|
|
|||
|
|
@ -473,7 +473,6 @@ var notimplementedBucketResourceNames = map[string]bool{
|
|||
"acl": true,
|
||||
"cors": true,
|
||||
"inventory": true,
|
||||
"lifecycle": true,
|
||||
"logging": true,
|
||||
"metrics": true,
|
||||
"replication": true,
|
||||
|
|
|
|||
|
|
@ -83,6 +83,11 @@ const (
|
|||
// GlobalMultipartCleanupInterval - Cleanup interval when the stale multipart cleanup is initiated.
|
||||
GlobalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
|
||||
|
||||
// GlobalServiceExecutionInterval - Executes the Lifecycle events.
|
||||
GlobalServiceExecutionInterval = time.Hour * 24 // 24 hrs.
|
||||
|
||||
// Refresh interval to update in-memory bucket lifecycle cache.
|
||||
globalRefreshBucketLifecycleInterval = 5 * time.Minute
|
||||
// Refresh interval to update in-memory iam config cache.
|
||||
globalRefreshIAMInterval = 5 * time.Minute
|
||||
|
||||
|
|
@ -148,6 +153,8 @@ var (
|
|||
globalPolicySys *PolicySys
|
||||
globalIAMSys *IAMSys
|
||||
|
||||
globalLifecycleSys *LifecycleSys
|
||||
|
||||
// CA root certificates, a nil value means system certs pool will be used
|
||||
globalRootCAs *x509.CertPool
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,191 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
// Disabled means the lifecycle rule is inactive
|
||||
Disabled = "Disabled"
|
||||
)
|
||||
|
||||
// LifecycleSys - Bucket lifecycle subsystem.
|
||||
type LifecycleSys struct {
|
||||
sync.RWMutex
|
||||
bucketLifecycleMap map[string]lifecycle.Lifecycle
|
||||
}
|
||||
|
||||
// Set - sets lifecycle config to given bucket name.
|
||||
func (sys *LifecycleSys) Set(bucketName string, lifecycle lifecycle.Lifecycle) {
|
||||
sys.Lock()
|
||||
defer sys.Unlock()
|
||||
|
||||
sys.bucketLifecycleMap[bucketName] = lifecycle
|
||||
}
|
||||
|
||||
func saveLifecycleConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, bucketLifecycle *lifecycle.Lifecycle) error {
|
||||
data, err := xml.Marshal(bucketLifecycle)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Construct path to lifecycle.xml for the given bucket.
|
||||
configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
|
||||
return saveConfig(ctx, objAPI, configFile, data)
|
||||
}
|
||||
|
||||
// getLifecycleConfig - get lifecycle config for given bucket name.
|
||||
func getLifecycleConfig(objAPI ObjectLayer, bucketName string) (*lifecycle.Lifecycle, error) {
|
||||
// Construct path to lifecycle.xml for the given bucket.
|
||||
configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
|
||||
configData, err := readConfig(context.Background(), objAPI, configFile)
|
||||
if err != nil {
|
||||
if err == errConfigNotFound {
|
||||
err = BucketLifecycleNotFound{Bucket: bucketName}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return lifecycle.ParseLifecycleConfig(bytes.NewReader(configData))
|
||||
}
|
||||
|
||||
func removeLifecycleConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {
|
||||
// Construct path to lifecycle.xml for the given bucket.
|
||||
configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
|
||||
|
||||
if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
|
||||
if _, ok := err.(ObjectNotFound); ok {
|
||||
return BucketLifecycleNotFound{Bucket: bucketName}
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewLifecycleSys - creates new lifecycle system.
|
||||
func NewLifecycleSys() *LifecycleSys {
|
||||
return &LifecycleSys{
|
||||
bucketLifecycleMap: make(map[string]lifecycle.Lifecycle),
|
||||
}
|
||||
}
|
||||
|
||||
// Init - initializes lifecycle system from lifecycle.xml of all buckets.
|
||||
func (sys *LifecycleSys) Init(objAPI ObjectLayer) error {
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// Refresh LifecycleSys in background.
|
||||
go func() {
|
||||
ticker := time.NewTicker(globalRefreshBucketLifecycleInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-GlobalServiceDoneCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
sys.refresh(objAPI)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}()
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
defer close(doneCh)
|
||||
|
||||
// Initializing lifecycle needs a retry mechanism for
|
||||
// the following reasons:
|
||||
// - Read quorum is lost just after the initialization
|
||||
// of the object layer.
|
||||
for range newRetryTimerSimple(doneCh) {
|
||||
// Load LifecycleSys once during boot.
|
||||
if err := sys.refresh(objAPI); err != nil {
|
||||
if err == errDiskNotFound ||
|
||||
strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
|
||||
strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
|
||||
logger.Info("Waiting for lifecycle subsystem to be initialized..")
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
break
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Refresh LifecycleSys.
|
||||
func (sys *LifecycleSys) refresh(objAPI ObjectLayer) error {
|
||||
buckets, err := objAPI.ListBuckets(context.Background())
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
return err
|
||||
}
|
||||
sys.removeDeletedBuckets(buckets)
|
||||
for _, bucket := range buckets {
|
||||
config, err := objAPI.GetBucketLifecycle(context.Background(), bucket.Name)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketLifecycleNotFound); ok {
|
||||
sys.Remove(bucket.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
sys.Set(bucket.Name, *config)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeDeletedBuckets - to handle a corner case where we have cached the lifecycle for a deleted
|
||||
// bucket. i.e if we miss a delete-bucket notification we should delete the corresponding
|
||||
// bucket policy during sys.refresh()
|
||||
func (sys *LifecycleSys) removeDeletedBuckets(bucketInfos []BucketInfo) {
|
||||
buckets := set.NewStringSet()
|
||||
for _, info := range bucketInfos {
|
||||
buckets.Add(info.Name)
|
||||
}
|
||||
sys.Lock()
|
||||
defer sys.Unlock()
|
||||
|
||||
for bucket := range sys.bucketLifecycleMap {
|
||||
if !buckets.Contains(bucket) {
|
||||
delete(sys.bucketLifecycleMap, bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove - removes policy for given bucket name.
|
||||
func (sys *LifecycleSys) Remove(bucketName string) {
|
||||
sys.Lock()
|
||||
defer sys.Unlock()
|
||||
|
||||
delete(sys.bucketLifecycleMap, bucketName)
|
||||
}
|
||||
|
|
@ -34,6 +34,7 @@ import (
|
|||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
|
|
@ -498,6 +499,48 @@ func (sys *NotificationSys) RemoveBucketPolicy(ctx context.Context, bucketName s
|
|||
}()
|
||||
}
|
||||
|
||||
// SetBucketLifecycle - calls SetBucketLifecycle on all peers.
|
||||
func (sys *NotificationSys) SetBucketLifecycle(ctx context.Context, bucketName string, bucketLifecycle *lifecycle.Lifecycle) {
|
||||
go func() {
|
||||
var wg sync.WaitGroup
|
||||
for _, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(client *peerRESTClient) {
|
||||
defer wg.Done()
|
||||
if err := client.SetBucketLifecycle(bucketName, bucketLifecycle); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name)
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}(client)
|
||||
}
|
||||
wg.Wait()
|
||||
}()
|
||||
}
|
||||
|
||||
// RemoveBucketLifecycle - calls RemoveLifecycle on all peers.
|
||||
func (sys *NotificationSys) RemoveBucketLifecycle(ctx context.Context, bucketName string) {
|
||||
go func() {
|
||||
var wg sync.WaitGroup
|
||||
for _, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(client *peerRESTClient) {
|
||||
defer wg.Done()
|
||||
if err := client.RemoveBucketLifecycle(bucketName); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name)
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}(client)
|
||||
}
|
||||
wg.Wait()
|
||||
}()
|
||||
}
|
||||
|
||||
// PutBucketNotification - calls PutBucketNotification RPC call on all peers.
|
||||
func (sys *NotificationSys) PutBucketNotification(ctx context.Context, bucketName string, rulesMap event.RulesMap) {
|
||||
go func() {
|
||||
|
|
|
|||
|
|
@ -254,6 +254,13 @@ func (e BucketPolicyNotFound) Error() string {
|
|||
return "No bucket policy found for bucket: " + e.Bucket
|
||||
}
|
||||
|
||||
// BucketLifecycleNotFound - no bucket lifecycle found.
|
||||
type BucketLifecycleNotFound GenericError
|
||||
|
||||
func (e BucketLifecycleNotFound) Error() string {
|
||||
return "No bucket life cycle found for bucket : " + e.Bucket
|
||||
}
|
||||
|
||||
/// Bucket related errors.
|
||||
|
||||
// BucketNameInvalid - bucketname provided is invalid.
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"net/http"
|
||||
|
||||
"github.com/minio/minio-go/v6/pkg/encrypt"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
|
@ -107,4 +108,9 @@ type ObjectLayer interface {
|
|||
|
||||
// Compression support check.
|
||||
IsCompressionSupported() bool
|
||||
|
||||
// Lifecycle operations
|
||||
SetBucketLifecycle(context.Context, string, *lifecycle.Lifecycle) error
|
||||
GetBucketLifecycle(context.Context, string) (*lifecycle.Lifecycle, error)
|
||||
DeleteBucketLifecycle(context.Context, string) error
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import (
|
|||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/rest"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
|
|
@ -329,6 +330,37 @@ func (client *peerRESTClient) SetBucketPolicy(bucket string, bucketPolicy *polic
|
|||
return nil
|
||||
}
|
||||
|
||||
// RemoveBucketLifecycle - Remove bucket lifecycle configuration on the peer node
|
||||
func (client *peerRESTClient) RemoveBucketLifecycle(bucket string) error {
|
||||
values := make(url.Values)
|
||||
values.Set(peerRESTBucket, bucket)
|
||||
respBody, err := client.call(peerRESTMethodBucketLifecycleRemove, values, nil, -1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer http.DrainBody(respBody)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetBucketLifecycle - Set bucket lifecycle configuration on the peer node
|
||||
func (client *peerRESTClient) SetBucketLifecycle(bucket string, bucketLifecycle *lifecycle.Lifecycle) error {
|
||||
values := make(url.Values)
|
||||
values.Set(peerRESTBucket, bucket)
|
||||
|
||||
var reader bytes.Buffer
|
||||
err := gob.NewEncoder(&reader).Encode(bucketLifecycle)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
respBody, err := client.call(peerRESTMethodBucketLifecycleSet, values, &reader, -1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer http.DrainBody(respBody)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutBucketNotification - Put bucket notification on the peer node.
|
||||
func (client *peerRESTClient) PutBucketNotification(bucket string, rulesMap event.RulesMap) error {
|
||||
values := make(url.Values)
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ const (
|
|||
peerRESTMethodTargetExists = "targetexists"
|
||||
peerRESTMethodSendEvent = "sendevent"
|
||||
peerRESTMethodTrace = "trace"
|
||||
peerRESTMethodBucketLifecycleSet = "setbucketlifecycle"
|
||||
peerRESTMethodBucketLifecycleRemove = "removebucketlifecycle"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import (
|
|||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
trace "github.com/minio/minio/pkg/trace"
|
||||
|
|
@ -469,6 +470,47 @@ func (s *peerRESTServer) SetBucketPolicyHandler(w http.ResponseWriter, r *http.R
|
|||
w.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
// RemoveBucketLifecycleHandler - Remove bucket lifecycle.
|
||||
func (s *peerRESTServer) RemoveBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.IsValid(w, r) {
|
||||
s.writeErrorResponse(w, errors.New("Invalid request"))
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars[peerRESTBucket]
|
||||
if bucketName == "" {
|
||||
s.writeErrorResponse(w, errors.New("Bucket name is missing"))
|
||||
return
|
||||
}
|
||||
|
||||
globalLifecycleSys.Remove(bucketName)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
// SetBucketLifecycleHandler - Set bucket lifecycle.
|
||||
func (s *peerRESTServer) SetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars[peerRESTBucket]
|
||||
if bucketName == "" {
|
||||
s.writeErrorResponse(w, errors.New("Bucket name is missing"))
|
||||
return
|
||||
}
|
||||
var lifecycleData lifecycle.Lifecycle
|
||||
if r.ContentLength < 0 {
|
||||
s.writeErrorResponse(w, errInvalidArgument)
|
||||
return
|
||||
}
|
||||
|
||||
err := gob.NewDecoder(r.Body).Decode(&lifecycleData)
|
||||
if err != nil {
|
||||
s.writeErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
globalLifecycleSys.Set(bucketName, lifecycleData)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
type remoteTargetExistsResp struct {
|
||||
Exists bool
|
||||
}
|
||||
|
|
@ -768,6 +810,8 @@ func registerPeerRESTHandlers(router *mux.Router) {
|
|||
subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketNotificationListen).HandlerFunc(httpTraceHdrs(server.ListenBucketNotificationHandler)).Queries(restQueries(peerRESTBucket)...)
|
||||
|
||||
subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)
|
||||
subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketLifecycleSet).HandlerFunc(httpTraceHdrs(server.SetBucketLifecycleHandler)).Queries(restQueries(peerRESTBucket)...)
|
||||
subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketLifecycleRemove).HandlerFunc(httpTraceHdrs(server.RemoveBucketLifecycleHandler)).Queries(restQueries(peerRESTBucket)...)
|
||||
|
||||
subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
|
||||
subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)
|
||||
|
|
|
|||
|
|
@ -365,6 +365,14 @@ func serverMain(ctx *cli.Context) {
|
|||
logger.Fatal(err, "Unable to initialize policy system")
|
||||
}
|
||||
|
||||
// Create new lifecycle system.
|
||||
globalLifecycleSys = NewLifecycleSys()
|
||||
|
||||
// Initialize lifecycle system.
|
||||
if err = globalLifecycleSys.Init(newObject); err != nil {
|
||||
logger.Fatal(err, "Unable to initialize lifecycle system")
|
||||
}
|
||||
|
||||
// Create new notification system.
|
||||
globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
|
||||
|
||||
|
|
|
|||
|
|
@ -365,6 +365,9 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
|
|||
globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks)
|
||||
globalNotificationSys.Init(objLayer)
|
||||
|
||||
globalLifecycleSys = NewLifecycleSys()
|
||||
globalLifecycleSys.Init(objLayer)
|
||||
|
||||
return testServer
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import (
|
|||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/bpool"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
"github.com/minio/minio/pkg/sync/errgroup"
|
||||
|
|
@ -518,6 +519,21 @@ func (s *xlSets) DeleteBucketPolicy(ctx context.Context, bucket string) error {
|
|||
return removePolicyConfig(ctx, s, bucket)
|
||||
}
|
||||
|
||||
// SetBucketLifecycle sets lifecycle on bucket
|
||||
func (s *xlSets) SetBucketLifecycle(ctx context.Context, bucket string, lifecycle *lifecycle.Lifecycle) error {
|
||||
return saveLifecycleConfig(ctx, s, bucket, lifecycle)
|
||||
}
|
||||
|
||||
// GetBucketLifecycle will get lifecycle on bucket
|
||||
func (s *xlSets) GetBucketLifecycle(ctx context.Context, bucket string) (*lifecycle.Lifecycle, error) {
|
||||
return getLifecycleConfig(s, bucket)
|
||||
}
|
||||
|
||||
// DeleteBucketLifecycle deletes all lifecycle on bucket
|
||||
func (s *xlSets) DeleteBucketLifecycle(ctx context.Context, bucket string) error {
|
||||
return removeLifecycleConfig(ctx, s, bucket)
|
||||
}
|
||||
|
||||
// IsNotificationSupported returns whether bucket notification is applicable for this layer.
|
||||
func (s *xlSets) IsNotificationSupported() bool {
|
||||
return s.getHashedSet("").IsNotificationSupported()
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import (
|
|||
|
||||
"github.com/minio/minio-go/v6/pkg/s3utils"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/lifecycle"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
||||
|
|
@ -308,6 +309,21 @@ func (xl xlObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error
|
|||
return removePolicyConfig(ctx, xl, bucket)
|
||||
}
|
||||
|
||||
// SetBucketLifecycle sets lifecycle on bucket
|
||||
func (xl xlObjects) SetBucketLifecycle(ctx context.Context, bucket string, lifecycle *lifecycle.Lifecycle) error {
|
||||
return saveLifecycleConfig(ctx, xl, bucket, lifecycle)
|
||||
}
|
||||
|
||||
// GetBucketLifecycle will get lifecycle on bucket
|
||||
func (xl xlObjects) GetBucketLifecycle(ctx context.Context, bucket string) (*lifecycle.Lifecycle, error) {
|
||||
return getLifecycleConfig(xl, bucket)
|
||||
}
|
||||
|
||||
// DeleteBucketLifecycle deletes all lifecycle on bucket
|
||||
func (xl xlObjects) DeleteBucketLifecycle(ctx context.Context, bucket string) error {
|
||||
return removeLifecycleConfig(ctx, xl, bucket)
|
||||
}
|
||||
|
||||
// IsNotificationSupported returns whether bucket notification is applicable for this layer.
|
||||
func (xl xlObjects) IsNotificationSupported() bool {
|
||||
return true
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -60,6 +60,7 @@ require (
|
|||
github.com/minio/highwayhash v1.0.0
|
||||
github.com/minio/lsync v1.0.1
|
||||
github.com/minio/mc v0.0.0-20190529152718-f4bb0b8850cb
|
||||
github.com/minio/minio-go v0.0.0-20190327203652-5325257a208f
|
||||
github.com/minio/minio-go/v6 v6.0.29
|
||||
github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679
|
||||
github.com/minio/sha256-simd v0.1.0
|
||||
|
|
@ -93,9 +94,9 @@ require (
|
|||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
|
||||
go.etcd.io/bbolt v1.3.3 // indirect
|
||||
go.uber.org/atomic v1.3.2
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
|
||||
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7
|
||||
golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect
|
||||
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb
|
||||
google.golang.org/api v0.4.0
|
||||
gopkg.in/Shopify/sarama.v1 v1.20.0
|
||||
gopkg.in/olivere/elastic.v5 v5.0.80
|
||||
|
|
|
|||
12
go.sum
12
go.sum
|
|
@ -693,8 +693,8 @@ golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACk
|
|||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
|
||||
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A=
|
||||
golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
|
@ -725,8 +725,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
|
|||
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
|
||||
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180603041954-1e0a3fa8ba9a/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
|
|
@ -767,8 +767,8 @@ golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI=
|
||||
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
|
||||
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
// Action - policy action.
|
||||
// Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazons3.html
|
||||
// for more information about available actions.
|
||||
type Action string
|
||||
|
||||
const (
|
||||
// PutBucketLifecycleAction - PutBucketLifecycle Rest API action.
|
||||
PutBucketLifecycleAction = "s3:PutBucketLifecycle"
|
||||
|
||||
// GetBucketLifecycleAction - GetBucketLifecycle Rest API action.
|
||||
GetBucketLifecycleAction = "s3:GetBucketLifecycle"
|
||||
|
||||
// DeleteBucketLifecycleAction - DeleteBucketLifecycleAction Rest API action.
|
||||
DeleteBucketLifecycleAction = "s3:DeleteBucketLifecycle"
|
||||
)
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// And - a tag to combine a prefix and multiple tags for lifecycle configuration rule.
|
||||
type And struct {
|
||||
XMLName xml.Name `xml:"And"`
|
||||
Prefix string `xml:"Prefix,omitempty"`
|
||||
Tags []Tag `xml:"Tag,omitempty"`
|
||||
}
|
||||
|
||||
var errAndUnsupported = errors.New("Specifying <And></And> tag is not supported")
|
||||
|
||||
// UnmarshalXML is extended to indicate lack of support for And xml
|
||||
// tag in object lifecycle configuration
|
||||
func (a And) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
return errAndUnsupported
|
||||
}
|
||||
|
||||
// MarshalXML is extended to leave out <And></And> tags
|
||||
func (a And) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
errLifecycleInvalidDate = errors.New("Date must be provided in ISO 8601 format")
|
||||
errLifecycleInvalidDays = errors.New("Days must be positive integer when used with Expiration")
|
||||
errLifecycleInvalidExpiration = errors.New("At least one of Days or Date should be present inside Expiration")
|
||||
errLifecycleDateNotMidnight = errors.New(" 'Date' must be at midnight GMT")
|
||||
)
|
||||
|
||||
// ExpirationDays is a type alias to unmarshal Days in Expiration
|
||||
type ExpirationDays int
|
||||
|
||||
// UnmarshalXML parses number of days from Expiration and validates if
|
||||
// greater than zero
|
||||
func (eDays *ExpirationDays) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
|
||||
var numDays int
|
||||
err := d.DecodeElement(&numDays, &startElement)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if numDays <= 0 {
|
||||
return errLifecycleInvalidDays
|
||||
}
|
||||
*eDays = ExpirationDays(numDays)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalXML encodes number of days to expire if it is non-zero and
|
||||
// encodes empty string otherwise
|
||||
func (eDays *ExpirationDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
|
||||
if *eDays == ExpirationDays(0) {
|
||||
return nil
|
||||
}
|
||||
return e.EncodeElement(int(*eDays), startElement)
|
||||
}
|
||||
|
||||
// ExpirationDate is a embedded type containing time.Time to unmarshal
|
||||
// Date in Expiration
|
||||
type ExpirationDate struct {
|
||||
time.Time
|
||||
}
|
||||
|
||||
// UnmarshalXML parses date from Expiration and validates date format
|
||||
func (eDate *ExpirationDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
|
||||
var dateStr string
|
||||
err := d.DecodeElement(&dateStr, &startElement)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// While AWS documentation mentions that the date specified
|
||||
// must be present in ISO 8601 format, in reality they allow
|
||||
// users to provide RFC 3339 compliant dates.
|
||||
expDate, err := time.Parse(time.RFC3339, dateStr)
|
||||
if err != nil {
|
||||
return errLifecycleInvalidDate
|
||||
}
|
||||
// Allow only date timestamp specifying midnight GMT
|
||||
hr, min, sec := expDate.Clock()
|
||||
nsec := expDate.Nanosecond()
|
||||
loc := expDate.Location()
|
||||
if !(hr == 0 && min == 0 && sec == 0 && nsec == 0 && loc.String() == time.UTC.String()) {
|
||||
return errLifecycleDateNotMidnight
|
||||
}
|
||||
|
||||
*eDate = ExpirationDate{expDate}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalXML encodes expiration date if it is non-zero and encodes
|
||||
// empty string otherwise
|
||||
func (eDate *ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
|
||||
if *eDate == (ExpirationDate{time.Time{}}) {
|
||||
return nil
|
||||
}
|
||||
return e.EncodeElement(eDate.Format(time.RFC3339), startElement)
|
||||
}
|
||||
|
||||
// Expiration - expiration actions for a rule in lifecycle configuration.
|
||||
type Expiration struct {
|
||||
XMLName xml.Name `xml:"Expiration"`
|
||||
Days ExpirationDays `xml:"Days,omitempty"`
|
||||
Date ExpirationDate `xml:"Date,omitempty"`
|
||||
}
|
||||
|
||||
// Validate - validates the "Expiration" element
|
||||
func (e Expiration) Validate() error {
|
||||
// Neither expiration days or date is specified
|
||||
if e.Days == ExpirationDays(0) && e.Date == (ExpirationDate{time.Time{}}) {
|
||||
return errLifecycleInvalidExpiration
|
||||
}
|
||||
|
||||
// Both expiration days and date are specified
|
||||
if e.Days != ExpirationDays(0) && e.Date != (ExpirationDate{time.Time{}}) {
|
||||
return errLifecycleInvalidExpiration
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// appropriate errors on validation
|
||||
func TestInvalidExpiration(t *testing.T) {
|
||||
testCases := []struct {
|
||||
inputXML string
|
||||
expectedErr error
|
||||
}{
|
||||
{ // Expiration with zero days
|
||||
inputXML: ` <Expiration>
|
||||
<Days>0</Days>
|
||||
</Expiration>`,
|
||||
expectedErr: errLifecycleInvalidDays,
|
||||
},
|
||||
{ // Expiration with invalid date
|
||||
inputXML: ` <Expiration>
|
||||
<Date>invalid date</Date>
|
||||
</Expiration>`,
|
||||
expectedErr: errLifecycleInvalidDate,
|
||||
},
|
||||
{ // Expiration with both number of days nor a date
|
||||
inputXML: `<Expiration>
|
||||
<Date>2019-04-20T00:01:00Z</Date>
|
||||
</Expiration>`,
|
||||
expectedErr: errLifecycleDateNotMidnight,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
|
||||
var expiration Expiration
|
||||
err := xml.Unmarshal([]byte(tc.inputXML), &expiration)
|
||||
if err != tc.expectedErr {
|
||||
t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
validationTestCases := []struct {
|
||||
inputXML string
|
||||
expectedErr error
|
||||
}{
|
||||
{ // Expiration with a valid ISO 8601 date
|
||||
inputXML: `<Expiration>
|
||||
<Date>2019-04-20T00:00:00Z</Date>
|
||||
</Expiration>`,
|
||||
expectedErr: nil,
|
||||
},
|
||||
{ // Expiration with a valid number of days
|
||||
inputXML: `<Expiration>
|
||||
<Days>3</Days>
|
||||
</Expiration>`,
|
||||
expectedErr: nil,
|
||||
},
|
||||
{ // Expiration with neither number of days nor a date
|
||||
inputXML: `<Expiration>
|
||||
</Expiration>`,
|
||||
expectedErr: errLifecycleInvalidExpiration,
|
||||
},
|
||||
{ // Expiration with both number of days nor a date
|
||||
inputXML: `<Expiration>
|
||||
<Days>3</Days>
|
||||
<Date>2019-04-20T00:00:00Z</Date>
|
||||
</Expiration>`,
|
||||
expectedErr: errLifecycleInvalidExpiration,
|
||||
},
|
||||
}
|
||||
for i, tc := range validationTestCases {
|
||||
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
|
||||
var expiration Expiration
|
||||
err := xml.Unmarshal([]byte(tc.inputXML), &expiration)
|
||||
if err != nil {
|
||||
t.Fatalf("%d: %v", i+1, err)
|
||||
}
|
||||
|
||||
err = expiration.Validate()
|
||||
if err != tc.expectedErr {
|
||||
t.Fatalf("%d: %v", i+1, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import "encoding/xml"
|
||||
|
||||
// Filter - a filter for a lifecycle configuration Rule.
|
||||
type Filter struct {
|
||||
XMLName xml.Name `xml:"Filter"`
|
||||
And And `xml:"And,omitempty"`
|
||||
Prefix string `xml:"Prefix"`
|
||||
Tag Tag `xml:"Tag,omitempty"`
|
||||
}
|
||||
|
||||
// Validate - validates the filter element
|
||||
func (f Filter) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestUnsupportedFilters checks if parsing Filter xml with
|
||||
// unsupported elements returns appropriate errors
|
||||
func TestUnsupportedFilters(t *testing.T) {
|
||||
testCases := []struct {
|
||||
inputXML string
|
||||
expectedErr error
|
||||
}{
|
||||
{ // Filter with And tags
|
||||
inputXML: ` <Filter>
|
||||
<And>
|
||||
<Prefix></Prefix>
|
||||
</And>
|
||||
</Filter>`,
|
||||
expectedErr: errAndUnsupported,
|
||||
},
|
||||
{ // Filter with Tag tags
|
||||
inputXML: ` <Filter>
|
||||
<Tag></Tag>
|
||||
</Filter>`,
|
||||
expectedErr: errTagUnsupported,
|
||||
},
|
||||
}
|
||||
for i, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
|
||||
var filter Filter
|
||||
err := xml.Unmarshal([]byte(tc.inputXML), &filter)
|
||||
if err != tc.expectedErr {
|
||||
t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
errLifecycleTooManyRules = errors.New("Lifecycle configuration allows a maximum of 1000 rules")
|
||||
errLifecycleNoRule = errors.New("Lifecycle configuration should have at least one rule")
|
||||
errLifecycleOverlappingPrefix = errors.New("Lifecycle configuration has rules with overlapping prefix")
|
||||
)
|
||||
|
||||
// Lifecycle - Configuration for bucket lifecycle.
|
||||
type Lifecycle struct {
|
||||
XMLName xml.Name `xml:"LifecycleConfiguration"`
|
||||
Rules []Rule `xml:"Rule"`
|
||||
}
|
||||
|
||||
// IsEmpty - returns whether policy is empty or not.
|
||||
func (lc Lifecycle) IsEmpty() bool {
|
||||
return len(lc.Rules) == 0
|
||||
}
|
||||
|
||||
// ParseLifecycleConfig - parses data in given reader to Lifecycle.
|
||||
func ParseLifecycleConfig(reader io.Reader) (*Lifecycle, error) {
|
||||
var lc Lifecycle
|
||||
if err := xml.NewDecoder(reader).Decode(&lc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := lc.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &lc, nil
|
||||
}
|
||||
|
||||
// Validate - validates the lifecycle configuration
|
||||
func (lc Lifecycle) Validate() error {
|
||||
// Lifecycle config can't have more than 1000 rules
|
||||
if len(lc.Rules) > 1000 {
|
||||
return errLifecycleTooManyRules
|
||||
}
|
||||
// Lifecycle config should have at least one rule
|
||||
if len(lc.Rules) == 0 {
|
||||
return errLifecycleNoRule
|
||||
}
|
||||
// Validate all the rules in the lifecycle config
|
||||
for _, r := range lc.Rules {
|
||||
if err := r.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Compare every rule's prefix with every other rule's prefix
|
||||
for i := range lc.Rules {
|
||||
if i == len(lc.Rules)-1 {
|
||||
break
|
||||
}
|
||||
// N B Empty prefixes overlap with all prefixes
|
||||
otherRules := lc.Rules[i+1:]
|
||||
for _, otherRule := range otherRules {
|
||||
if strings.HasPrefix(lc.Rules[i].Filter.Prefix, otherRule.Filter.Prefix) ||
|
||||
strings.HasPrefix(otherRule.Filter.Prefix, lc.Rules[i].Filter.Prefix) {
|
||||
return errLifecycleOverlappingPrefix
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseLifecycleConfig(t *testing.T) {
|
||||
// Test for lifecycle config with more than 1000 rules
|
||||
var manyRules []Rule
|
||||
rule := Rule{
|
||||
Status: "Enabled",
|
||||
Expiration: Expiration{Days: ExpirationDays(3)},
|
||||
}
|
||||
for i := 0; i < 1001; i++ {
|
||||
manyRules = append(manyRules, rule)
|
||||
}
|
||||
|
||||
manyRuleLcConfig, err := xml.Marshal(Lifecycle{Rules: manyRules})
|
||||
if err != nil {
|
||||
t.Fatal("Failed to marshal lifecycle config with more than 1000 rules")
|
||||
}
|
||||
|
||||
// Test for lifecycle config with rules containing overlapping prefixes
|
||||
rule1 := Rule{
|
||||
Status: "Enabled",
|
||||
Expiration: Expiration{Days: ExpirationDays(3)},
|
||||
Filter: Filter{
|
||||
Prefix: "/a/b",
|
||||
},
|
||||
}
|
||||
rule2 := Rule{
|
||||
Status: "Enabled",
|
||||
Expiration: Expiration{Days: ExpirationDays(3)},
|
||||
Filter: Filter{
|
||||
Prefix: "/a/b/c",
|
||||
},
|
||||
}
|
||||
overlappingRules := []Rule{rule1, rule2}
|
||||
overlappingLcConfig, err := xml.Marshal(Lifecycle{Rules: overlappingRules})
|
||||
if err != nil {
|
||||
t.Fatal("Failed to marshal lifecycle config with rules having overlapping prefix")
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
inputConfig string
|
||||
expectedErr error
|
||||
}{
|
||||
{ // Valid lifecycle config
|
||||
inputConfig: `<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<Filter>
|
||||
<Prefix>prefix</Prefix>
|
||||
</Filter>
|
||||
<Status>Enabled</Status>
|
||||
<Expiration><Days>3</Days></Expiration>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<Filter>
|
||||
<Prefix>another-prefix</Prefix>
|
||||
</Filter>
|
||||
<Status>Enabled</Status>
|
||||
<Expiration><Days>3</Days></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>`,
|
||||
expectedErr: nil,
|
||||
},
|
||||
{ // lifecycle config with no rules
|
||||
inputConfig: `<LifecycleConfiguration>
|
||||
</LifecycleConfiguration>`,
|
||||
expectedErr: errLifecycleNoRule,
|
||||
},
|
||||
{ // lifecycle config with more than 1000 rules
|
||||
inputConfig: string(manyRuleLcConfig),
|
||||
expectedErr: errLifecycleTooManyRules,
|
||||
},
|
||||
{ // lifecycle config with rules having overlapping prefix
|
||||
inputConfig: string(overlappingLcConfig),
|
||||
expectedErr: errLifecycleOverlappingPrefix,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
|
||||
var err error
|
||||
if _, err = ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig))); err != tc.expectedErr {
|
||||
t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarshalLifecycleConfig checks if lifecycleconfig xml
|
||||
// marshaling/unmarshaling can handle output from each other
|
||||
func TestMarshalLifecycleConfig(t *testing.T) {
|
||||
// Time at midnight UTC
|
||||
midnightTS := ExpirationDate{time.Date(2019, time.April, 20, 0, 0, 0, 0, time.UTC)}
|
||||
lc := Lifecycle{
|
||||
Rules: []Rule{
|
||||
{
|
||||
Status: "Enabled",
|
||||
Filter: Filter{Prefix: "prefix-1"},
|
||||
Expiration: Expiration{Days: ExpirationDays(3)},
|
||||
},
|
||||
{
|
||||
Status: "Enabled",
|
||||
Filter: Filter{Prefix: "prefix-1"},
|
||||
Expiration: Expiration{Date: ExpirationDate(midnightTS)},
|
||||
},
|
||||
},
|
||||
}
|
||||
b, err := xml.MarshalIndent(&lc, "", "\t")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var lc1 Lifecycle
|
||||
err = xml.Unmarshal(b, &lc1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ruleSet := make(map[string]struct{})
|
||||
for _, rule := range lc.Rules {
|
||||
ruleBytes, err := xml.Marshal(rule)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ruleSet[string(ruleBytes)] = struct{}{}
|
||||
}
|
||||
for _, rule := range lc1.Rules {
|
||||
ruleBytes, err := xml.Marshal(rule)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := ruleSet[string(ruleBytes)]; !ok {
|
||||
t.Fatalf("Expected %v to be equal to %v, %v missing", lc, lc1, rule)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// NoncurrentVersionExpiration - an action for lifecycle configuration rule.
|
||||
type NoncurrentVersionExpiration struct {
|
||||
XMLName xml.Name `xml:"NoncurrentVersionExpiration"`
|
||||
NoncurrentDays int `xml:"NoncurrentDays,omitempty"`
|
||||
}
|
||||
|
||||
// NoncurrentVersionTransition - an action for lifecycle configuration rule.
|
||||
type NoncurrentVersionTransition struct {
|
||||
NoncurrentDays int `xml:"NoncurrentDays"`
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
}
|
||||
|
||||
var (
|
||||
errNoncurrentVersionExpirationUnsupported = errors.New("Specifying <NoncurrentVersionExpiration></NoncurrentVersionExpiration> is not supported")
|
||||
errNoncurrentVersionTransitionUnsupported = errors.New("Specifying <NoncurrentVersionTransition></NoncurrentVersionTransition> is not supported")
|
||||
)
|
||||
|
||||
// UnmarshalXML is extended to indicate lack of support for
|
||||
// NoncurrentVersionExpiration xml tag in object lifecycle
|
||||
// configuration
|
||||
func (n NoncurrentVersionExpiration) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
|
||||
return errNoncurrentVersionExpirationUnsupported
|
||||
}
|
||||
|
||||
// UnmarshalXML is extended to indicate lack of support for
|
||||
// NoncurrentVersionTransition xml tag in object lifecycle
|
||||
// configuration
|
||||
func (n NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
|
||||
return errNoncurrentVersionTransitionUnsupported
|
||||
}
|
||||
|
||||
// MarshalXML is extended to leave out
|
||||
// <NoncurrentVersionTransition></NoncurrentVersionTransition> tags
|
||||
func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalXML is extended to leave out
|
||||
// <NoncurrentVersionExpiration></NoncurrentVersionExpiration> tags
|
||||
func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// Rule - a rule for lifecycle configuration.
|
||||
type Rule struct {
|
||||
XMLName xml.Name `xml:"Rule"`
|
||||
ID string `xml:"ID,omitempty"`
|
||||
Status string `xml:"Status"`
|
||||
Filter Filter `xml:"Filter"`
|
||||
Expiration Expiration `xml:"Expiration,omitempty"`
|
||||
Transition Transition `xml:"Transition,omitempty"`
|
||||
// FIXME: add a type to catch unsupported AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
|
||||
NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
|
||||
NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
errInvalidRuleID = errors.New("ID must be less than 255 characters")
|
||||
errEmptyRuleStatus = errors.New("Status should not be empty")
|
||||
errInvalidRuleStatus = errors.New("Status must be set to either Enabled or Disabled")
|
||||
errMissingExpirationAction = errors.New("No expiration action found")
|
||||
)
|
||||
|
||||
// isIDValid - checks if ID is valid or not.
|
||||
func (r Rule) validateID() error {
|
||||
// cannot be longer than 255 characters
|
||||
if len(string(r.ID)) > 255 {
|
||||
return errInvalidRuleID
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isStatusValid - checks if status is valid or not.
|
||||
func (r Rule) validateStatus() error {
|
||||
// Status can't be empty
|
||||
if len(r.Status) == 0 {
|
||||
return errEmptyRuleStatus
|
||||
}
|
||||
|
||||
// Status must be one of Enabled or Disabled
|
||||
if r.Status != "Enabled" && r.Status != "Disabled" {
|
||||
return errInvalidRuleStatus
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r Rule) validateAction() error {
|
||||
if r.Expiration == (Expiration{}) {
|
||||
return errMissingExpirationAction
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate - validates the rule element
|
||||
func (r Rule) Validate() error {
|
||||
if err := r.validateID(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.validateStatus(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.validateAction(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestUnsupportedRules checks if Rule xml with unsuported tags return
|
||||
// appropriate errors on parsing
|
||||
func TestUnsupportedRules(t *testing.T) {
|
||||
// NoncurrentVersionTransition, NoncurrentVersionExpiration
|
||||
// and Transition tags aren't supported
|
||||
unsupportedTestCases := []struct {
|
||||
inputXML string
|
||||
expectedErr error
|
||||
}{
|
||||
{ // Rule with unsupported NoncurrentVersionTransition
|
||||
inputXML: ` <Rule>
|
||||
<NoncurrentVersionTransition></NoncurrentVersionTransition>
|
||||
</Rule>`,
|
||||
expectedErr: errNoncurrentVersionTransitionUnsupported,
|
||||
},
|
||||
{ // Rule with unsupported NoncurrentVersionExpiration
|
||||
|
||||
inputXML: ` <Rule>
|
||||
<NoncurrentVersionExpiration></NoncurrentVersionExpiration>
|
||||
</Rule>`,
|
||||
expectedErr: errNoncurrentVersionExpirationUnsupported,
|
||||
},
|
||||
{ // Rule with unsupported Transition action
|
||||
inputXML: ` <Rule>
|
||||
<Transition></Transition>
|
||||
</Rule>`,
|
||||
expectedErr: errTransitionUnsupported,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range unsupportedTestCases {
|
||||
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
|
||||
var rule Rule
|
||||
err := xml.Unmarshal([]byte(tc.inputXML), &rule)
|
||||
if err != tc.expectedErr {
|
||||
t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestInvalidRules checks if Rule xml with invalid elements returns
|
||||
// appropriate errors on validation
|
||||
func TestInvalidRules(t *testing.T) {
|
||||
invalidTestCases := []struct {
|
||||
inputXML string
|
||||
expectedErr error
|
||||
}{
|
||||
{ // Rule without expiration action
|
||||
inputXML: ` <Rule>
|
||||
<Status>Enabled</Status>
|
||||
</Rule>`,
|
||||
expectedErr: errMissingExpirationAction,
|
||||
},
|
||||
{ // Rule with ID longer than 255 characters
|
||||
inputXML: ` <Rule>
|
||||
<ID> babababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab </ID>
|
||||
</Rule>`,
|
||||
expectedErr: errInvalidRuleID,
|
||||
},
|
||||
{ // Rule with empty status
|
||||
inputXML: ` <Rule>
|
||||
<Status></Status>
|
||||
</Rule>`,
|
||||
expectedErr: errEmptyRuleStatus,
|
||||
},
|
||||
{ // Rule with invalid status
|
||||
inputXML: ` <Rule>
|
||||
<Status>OK</Status>
|
||||
</Rule>`,
|
||||
expectedErr: errInvalidRuleStatus,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range invalidTestCases {
|
||||
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
|
||||
var rule Rule
|
||||
err := xml.Unmarshal([]byte(tc.inputXML), &rule)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := rule.Validate(); err != tc.expectedErr {
|
||||
t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// Tag - a tag for a lifecycle configuration Rule filter.
|
||||
type Tag struct {
|
||||
XMLName xml.Name `xml:"Tag"`
|
||||
Key string `xml:"Key,omitempty"`
|
||||
Value string `xml:"Value,omitempty"`
|
||||
}
|
||||
|
||||
var errTagUnsupported = errors.New("Specifying <Tag></Tag> is not supported")
|
||||
|
||||
// UnmarshalXML is extended to indicate lack of support for Tag
|
||||
// xml tag in object lifecycle configuration
|
||||
func (t Tag) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
return errTagUnsupported
|
||||
}
|
||||
|
||||
// MarshalXML is extended to leave out <Tag></Tag> tags
|
||||
func (t Tag) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// Transition - transition actions for a rule in lifecycle configuration.
|
||||
type Transition struct {
|
||||
XMLName xml.Name `xml:"Transition"`
|
||||
Days int `xml:"Days,omitempty"`
|
||||
Date string `xml:"Date,omitempty"`
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
}
|
||||
|
||||
var errTransitionUnsupported = errors.New("Specifying <Transition></Transition> tag is not supported")
|
||||
|
||||
// UnmarshalXML is extended to indicate lack of support for Transition
|
||||
// xml tag in object lifecycle configuration
|
||||
func (t Transition) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
return errTransitionUnsupported
|
||||
}
|
||||
|
||||
// MarshalXML is extended to leave out <Transition></Transition> tags
|
||||
func (t Transition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue