mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Add support for ILM transition (#10565)
This PR adds transition support for ILM to transition data to another MinIO target represented by a storage class ARN. Subsequent GET or HEAD for that object will be streamed from the transition tier. If PostRestoreObject API is invoked, the transitioned object can be restored for duration specified to the source cluster.
This commit is contained in:
parent
8f7fe0405e
commit
1ebf6f146a
@ -129,16 +129,17 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if !globalIsErasure {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
if !globalIsErasure {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
@ -218,7 +219,10 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
arnType := vars["type"]
|
||||
|
||||
if !globalIsErasure {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
@ -255,16 +259,16 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
|
||||
bucket := vars["bucket"]
|
||||
arn := vars["arn"]
|
||||
|
||||
if !globalIsErasure {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
if !globalIsErasure {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
|
@ -121,6 +121,7 @@ const (
|
||||
ErrReplicationNeedsVersioningError
|
||||
ErrReplicationBucketNeedsVersioningError
|
||||
ErrBucketReplicationDisabledError
|
||||
ErrObjectRestoreAlreadyInProgress
|
||||
ErrNoSuchKey
|
||||
ErrNoSuchUpload
|
||||
ErrNoSuchVersion
|
||||
@ -916,6 +917,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrObjectRestoreAlreadyInProgress: {
|
||||
Code: "RestoreAlreadyInProgress",
|
||||
Description: "Object restore is already in progress",
|
||||
HTTPStatusCode: http.StatusConflict,
|
||||
},
|
||||
/// Bucket notification related errors.
|
||||
ErrEventNotification: {
|
||||
Code: "InvalidArgument",
|
||||
|
@ -307,6 +307,9 @@ func registerAPIRouter(router *mux.Router) {
|
||||
// DeleteBucket
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
maxClients(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler))))
|
||||
// PostRestoreObject
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("restoreobject", httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
|
||||
}
|
||||
|
||||
/// Root operation
|
||||
|
@ -78,6 +78,12 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the transition storage ARNs
|
||||
if err = validateLifecycleTransition(ctx, bucket, bucketLifecycle); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(bucketLifecycle)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
|
@ -17,7 +17,26 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
sse "github.com/minio/minio/pkg/bucket/encryption"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/s3select"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -46,3 +65,547 @@ func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err er
|
||||
func NewLifecycleSys() *LifecycleSys {
|
||||
return &LifecycleSys{}
|
||||
}
|
||||
|
||||
type transitionState struct {
|
||||
// add future metrics here
|
||||
transitionCh chan ObjectInfo
|
||||
}
|
||||
|
||||
func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
|
||||
select {
|
||||
case t.transitionCh <- oi:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
globalTransitionState *transitionState
|
||||
globalTransitionConcurrent = runtime.GOMAXPROCS(0) / 2
|
||||
)
|
||||
|
||||
func newTransitionState() *transitionState {
|
||||
|
||||
// fix minimum concurrent transition to 1 for single CPU setup
|
||||
if globalTransitionConcurrent == 0 {
|
||||
globalTransitionConcurrent = 1
|
||||
}
|
||||
ts := &transitionState{
|
||||
transitionCh: make(chan ObjectInfo, 10000),
|
||||
}
|
||||
go func() {
|
||||
<-GlobalContext.Done()
|
||||
close(ts.transitionCh)
|
||||
}()
|
||||
return ts
|
||||
}
|
||||
|
||||
// addWorker creates a new worker to process tasks
|
||||
func (t *transitionState) addWorker(ctx context.Context, objectAPI ObjectLayer) {
|
||||
// Add a new worker.
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case oi, ok := <-t.transitionCh:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
transitionObject(ctx, objectAPI, oi)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func initBackgroundTransition(ctx context.Context, objectAPI ObjectLayer) {
|
||||
if globalTransitionState == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Start with globalTransitionConcurrent.
|
||||
for i := 0; i < globalTransitionConcurrent; i++ {
|
||||
globalTransitionState.addWorker(ctx, objectAPI)
|
||||
}
|
||||
}
|
||||
|
||||
func validateLifecycleTransition(ctx context.Context, bucket string, lfc *lifecycle.Lifecycle) error {
|
||||
for _, rule := range lfc.Rules {
|
||||
if rule.Transition.StorageClass != "" {
|
||||
sameTarget, destbucket, err := validateTransitionDestination(ctx, bucket, rule.Transition.StorageClass)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sameTarget && destbucket == bucket {
|
||||
return fmt.Errorf("Transition destination cannot be the same as the source bucket")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateTransitionDestination returns error if transition destination bucket missing or not configured
|
||||
// It also returns true if transition destination is same as this server.
|
||||
func validateTransitionDestination(ctx context.Context, bucket string, targetLabel string) (bool, string, error) {
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetWithLabel(ctx, bucket, targetLabel)
|
||||
if tgt == nil {
|
||||
return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
|
||||
}
|
||||
arn, err := madmin.ParseARN(tgt.Arn)
|
||||
if err != nil {
|
||||
return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
|
||||
}
|
||||
clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, tgt.Arn)
|
||||
if clnt == nil {
|
||||
return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
|
||||
}
|
||||
if found, _ := clnt.BucketExists(ctx, arn.Bucket); !found {
|
||||
return false, "", BucketRemoteDestinationNotFound{Bucket: arn.Bucket}
|
||||
}
|
||||
sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
|
||||
return sameTarget, arn.Bucket, nil
|
||||
}
|
||||
|
||||
// return true if ARN representing transition storage class is present in a active rule
|
||||
// for the lifecycle configured on this bucket
|
||||
func transitionSCInUse(ctx context.Context, lfc *lifecycle.Lifecycle, bucket, arnStr string) bool {
|
||||
tgtLabel := globalBucketTargetSys.GetRemoteLabelWithArn(ctx, bucket, arnStr)
|
||||
if tgtLabel == "" {
|
||||
return false
|
||||
}
|
||||
for _, rule := range lfc.Rules {
|
||||
if rule.Status == Disabled {
|
||||
continue
|
||||
}
|
||||
if rule.Transition.StorageClass != "" && rule.Transition.StorageClass == tgtLabel {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// set PutObjectOptions for PUT operation to transition data to target cluster
|
||||
func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
|
||||
meta := make(map[string]string)
|
||||
|
||||
tag, err := tags.ParseObjectTags(objInfo.UserTags)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
putOpts = miniogo.PutObjectOptions{
|
||||
UserMetadata: meta,
|
||||
UserTags: tag.ToMap(),
|
||||
ContentType: objInfo.ContentType,
|
||||
ContentEncoding: objInfo.ContentEncoding,
|
||||
StorageClass: objInfo.StorageClass,
|
||||
Internal: miniogo.AdvancedPutOptions{
|
||||
SourceVersionID: objInfo.VersionID,
|
||||
SourceMTime: objInfo.ModTime,
|
||||
SourceETag: objInfo.ETag,
|
||||
},
|
||||
}
|
||||
if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
|
||||
rmode := miniogo.RetentionMode(mode)
|
||||
putOpts.Mode = rmode
|
||||
}
|
||||
if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
|
||||
rdate, err := time.Parse(time.RFC3339, retainDateStr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
putOpts.RetainUntilDate = rdate
|
||||
}
|
||||
if lhold, ok := objInfo.UserDefined[xhttp.AmzObjectLockLegalHold]; ok {
|
||||
putOpts.LegalHold = miniogo.LegalHoldStatus(lhold)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// handle deletes of transitioned objects or object versions when one of the following is true:
|
||||
// 1. temporarily restored copies of objects (restored with the PostRestoreObject API) expired.
|
||||
// 2. life cycle expiry date is met on the object.
|
||||
func deleteTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket, object string, objInfo ObjectInfo, lcOpts lifecycle.ObjectOpts, action lifecycle.Action) error {
|
||||
if objInfo.TransitionStatus == "" {
|
||||
return nil
|
||||
}
|
||||
lc, err := globalLifecycleSys.Get(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
arn := getLifecycleTransitionTargetArn(ctx, lc, bucket, lcOpts)
|
||||
if arn == nil {
|
||||
return fmt.Errorf("remote target not configured")
|
||||
}
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
|
||||
if tgt == nil {
|
||||
return fmt.Errorf("remote target not configured")
|
||||
}
|
||||
|
||||
var opts ObjectOptions
|
||||
opts.Versioned = globalBucketVersioningSys.Enabled(bucket)
|
||||
opts.VersionID = objInfo.VersionID
|
||||
switch action {
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
// delete locally restored copy of object or object version
|
||||
// from the source, while leaving metadata behind. The data on
|
||||
// transitioned tier lies untouched and still accessible
|
||||
opts.TransitionStatus = objInfo.TransitionStatus
|
||||
_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
|
||||
return err
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
|
||||
// When an object is past expiry, delete the data from transitioned tier and
|
||||
// metadata from source
|
||||
if err := tgt.RemoveObject(ctx, arn.Bucket, object, miniogo.RemoveObjectOptions{VersionID: objInfo.VersionID}); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eventName := event.ObjectRemovedDelete
|
||||
if objInfo.DeleteMarker {
|
||||
eventName = event.ObjectRemovedDeleteMarkerCreated
|
||||
}
|
||||
// Notify object deleted event.
|
||||
sendEvent(eventArgs{
|
||||
EventName: eventName,
|
||||
BucketName: bucket,
|
||||
Object: objInfo,
|
||||
Host: "Internal: [ILM-EXPIRY]",
|
||||
})
|
||||
}
|
||||
|
||||
// should never reach here
|
||||
return nil
|
||||
}
|
||||
|
||||
// transition object to target specified by the transition ARN. When an object is transitioned to another
|
||||
// storage specified by the transition ARN, the metadata is left behind on source cluster and original content
|
||||
// is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
|
||||
// to the transition tier without decrypting or re-encrypting.
|
||||
func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo ObjectInfo) error {
|
||||
lc, err := globalLifecycleSys.Get(objInfo.Bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lcOpts := lifecycle.ObjectOpts{
|
||||
Name: objInfo.Name,
|
||||
UserTags: objInfo.UserTags,
|
||||
}
|
||||
arn := getLifecycleTransitionTargetArn(ctx, lc, objInfo.Bucket, lcOpts)
|
||||
if arn == nil {
|
||||
return fmt.Errorf("remote target not configured")
|
||||
}
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
|
||||
if tgt == nil {
|
||||
return fmt.Errorf("remote target not configured")
|
||||
}
|
||||
|
||||
gr, err := objectAPI.GetObjectNInfo(ctx, objInfo.Bucket, objInfo.Name, nil, http.Header{}, readLock, ObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
TransitionStatus: lifecycle.TransitionPending,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oi := gr.ObjInfo
|
||||
|
||||
if oi.TransitionStatus == lifecycle.TransitionComplete {
|
||||
return nil
|
||||
}
|
||||
|
||||
putOpts := putTransitionOpts(oi)
|
||||
if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, "", "", putOpts); err != nil {
|
||||
return err
|
||||
}
|
||||
gr.Close()
|
||||
|
||||
var opts ObjectOptions
|
||||
opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
|
||||
opts.VersionID = oi.VersionID
|
||||
opts.TransitionStatus = lifecycle.TransitionComplete
|
||||
|
||||
if _, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getLifecycleTransitionTargetArn returns transition ARN for storage class specified in the config.
|
||||
func getLifecycleTransitionTargetArn(ctx context.Context, lc *lifecycle.Lifecycle, bucket string, obj lifecycle.ObjectOpts) *madmin.ARN {
|
||||
for _, rule := range lc.FilterActionableRules(obj) {
|
||||
if rule.Transition.StorageClass != "" {
|
||||
return globalBucketTargetSys.GetRemoteArnWithLabel(ctx, bucket, rule.Transition.StorageClass)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getTransitionedObjectReader returns a reader from the transitioned tier.
|
||||
func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, oi ObjectInfo, opts ObjectOptions) (gr *GetObjectReader, err error) {
|
||||
var lc *lifecycle.Lifecycle
|
||||
lc, err = globalLifecycleSys.Get(bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
arn := getLifecycleTransitionTargetArn(ctx, lc, bucket, lifecycle.ObjectOpts{
|
||||
Name: object,
|
||||
UserTags: oi.UserTags,
|
||||
ModTime: oi.ModTime,
|
||||
VersionID: oi.VersionID,
|
||||
DeleteMarker: oi.DeleteMarker,
|
||||
IsLatest: oi.IsLatest,
|
||||
})
|
||||
if arn == nil {
|
||||
return nil, fmt.Errorf("remote target not configured")
|
||||
}
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
|
||||
if tgt == nil {
|
||||
return nil, fmt.Errorf("remote target not configured")
|
||||
}
|
||||
fn, off, length, err := NewGetObjectReader(rs, oi, opts)
|
||||
if err != nil {
|
||||
return nil, ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
gopts := miniogo.GetObjectOptions{VersionID: opts.VersionID}
|
||||
|
||||
// get correct offsets for encrypted object
|
||||
if off >= 0 && length >= 0 {
|
||||
if err := gopts.SetRange(off, off+length-1); err != nil {
|
||||
return nil, ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
reader, _, _, err := tgt.GetObject(ctx, arn.Bucket, object, gopts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
closeReader := func() { reader.Close() }
|
||||
|
||||
return fn(reader, h, opts.CheckPrecondFn, closeReader)
|
||||
}
|
||||
|
||||
// RestoreRequestType represents type of restore.
|
||||
type RestoreRequestType string
|
||||
|
||||
const (
|
||||
// SelectRestoreRequest specifies select request. This is the only valid value
|
||||
SelectRestoreRequest RestoreRequestType = "SELECT"
|
||||
)
|
||||
|
||||
// Encryption specifies encryption setting on restored bucket
|
||||
type Encryption struct {
|
||||
EncryptionType sse.SSEAlgorithm `xml:"EncryptionType"`
|
||||
KMSContext string `xml:"KMSContext,omitempty"`
|
||||
KMSKeyID string `xml:"KMSKeyId,omitempty"`
|
||||
}
|
||||
|
||||
// MetadataEntry denotes name and value.
|
||||
type MetadataEntry struct {
|
||||
Name string `xml:"Name"`
|
||||
Value string `xml:"Value"`
|
||||
}
|
||||
|
||||
// S3Location specifies s3 location that receives result of a restore object request
|
||||
type S3Location struct {
|
||||
BucketName string `xml:"BucketName,omitempty"`
|
||||
Encryption Encryption `xml:"Encryption,omitempty"`
|
||||
Prefix string `xml:"Prefix,omitempty"`
|
||||
StorageClass string `xml:"StorageClass,omitempty"`
|
||||
Tagging *tags.Tags `xml:"Tagging,omitempty"`
|
||||
UserMetadata []MetadataEntry `xml:"UserMetadata"`
|
||||
}
|
||||
|
||||
// OutputLocation specifies bucket where object needs to be restored
|
||||
type OutputLocation struct {
|
||||
S3 S3Location `xml:"S3,omitempty"`
|
||||
}
|
||||
|
||||
// IsEmpty returns true if output location not specified.
|
||||
func (o *OutputLocation) IsEmpty() bool {
|
||||
return o.S3.BucketName == ""
|
||||
}
|
||||
|
||||
// SelectParameters specifies sql select parameters
|
||||
type SelectParameters struct {
|
||||
s3select.S3Select
|
||||
}
|
||||
|
||||
// IsEmpty returns true if no select parameters set
|
||||
func (sp *SelectParameters) IsEmpty() bool {
|
||||
return sp == nil || sp.S3Select == s3select.S3Select{}
|
||||
}
|
||||
|
||||
var (
|
||||
selectParamsXMLName = "SelectParameters"
|
||||
)
|
||||
|
||||
// UnmarshalXML - decodes XML data.
|
||||
func (sp *SelectParameters) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
// Essentially the same as S3Select barring the xml name.
|
||||
if start.Name.Local == selectParamsXMLName {
|
||||
start.Name = xml.Name{Space: "", Local: "SelectRequest"}
|
||||
}
|
||||
return sp.S3Select.UnmarshalXML(d, start)
|
||||
}
|
||||
|
||||
// RestoreObjectRequest - xml to restore a transitioned object
|
||||
type RestoreObjectRequest struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ RestoreRequest" json:"-"`
|
||||
Days int `xml:"Days,omitempty"`
|
||||
Type RestoreRequestType `xml:"Type,omitempty"`
|
||||
Tier string `xml:"Tier,-"`
|
||||
Description string `xml:"Description,omitempty"`
|
||||
SelectParameters *SelectParameters `xml:"SelectParameters,omitempty"`
|
||||
OutputLocation OutputLocation `xml:"OutputLocation,omitempty"`
|
||||
}
|
||||
|
||||
// Maximum 2MiB size per restore object request.
|
||||
const maxRestoreObjectRequestSize = 2 << 20
|
||||
|
||||
// parseRestoreRequest parses RestoreObjectRequest from xml
|
||||
func parseRestoreRequest(reader io.Reader) (*RestoreObjectRequest, error) {
|
||||
req := RestoreObjectRequest{}
|
||||
if err := xml.NewDecoder(io.LimitReader(reader, maxRestoreObjectRequestSize)).Decode(&req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
// validate a RestoreObjectRequest as per AWS S3 spec https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html
|
||||
func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer) error {
|
||||
if r.Type != SelectRestoreRequest && !r.SelectParameters.IsEmpty() {
|
||||
return fmt.Errorf("Select parameters can only be specified with SELECT request type")
|
||||
}
|
||||
if r.Type == SelectRestoreRequest && r.SelectParameters.IsEmpty() {
|
||||
return fmt.Errorf("SELECT restore request requires select parameters to be specified")
|
||||
}
|
||||
|
||||
if r.Type != SelectRestoreRequest && !r.OutputLocation.IsEmpty() {
|
||||
return fmt.Errorf("OutputLocation required only for SELECT request type")
|
||||
}
|
||||
if r.Type == SelectRestoreRequest && r.OutputLocation.IsEmpty() {
|
||||
return fmt.Errorf("OutputLocation required for SELECT requests")
|
||||
}
|
||||
|
||||
if r.Days != 0 && r.Type == SelectRestoreRequest {
|
||||
return fmt.Errorf("Days cannot be specified with SELECT restore request")
|
||||
}
|
||||
if r.Days == 0 && r.Type != SelectRestoreRequest {
|
||||
return fmt.Errorf("restoration days should be at least 1")
|
||||
}
|
||||
// Check if bucket exists.
|
||||
if !r.OutputLocation.IsEmpty() {
|
||||
if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.OutputLocation.S3.Prefix == "" {
|
||||
return fmt.Errorf("Prefix is a required parameter in OutputLocation")
|
||||
}
|
||||
if r.OutputLocation.S3.Encryption.EncryptionType != crypto.SSEAlgorithmAES256 {
|
||||
return NotImplemented{}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// set ObjectOptions for PUT call to restore temporary copy of transitioned data
|
||||
func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
|
||||
meta := make(map[string]string)
|
||||
sc := rreq.OutputLocation.S3.StorageClass
|
||||
if sc == "" {
|
||||
sc = objInfo.StorageClass
|
||||
}
|
||||
meta[strings.ToLower(xhttp.AmzStorageClass)] = sc
|
||||
|
||||
if rreq.Type == SelectRestoreRequest {
|
||||
for _, v := range rreq.OutputLocation.S3.UserMetadata {
|
||||
if !strings.HasPrefix("x-amz-meta", strings.ToLower(v.Name)) {
|
||||
meta["x-amz-meta-"+v.Name] = v.Value
|
||||
continue
|
||||
}
|
||||
meta[v.Name] = v.Value
|
||||
}
|
||||
meta[xhttp.AmzObjectTagging] = rreq.OutputLocation.S3.Tagging.String()
|
||||
if rreq.OutputLocation.S3.Encryption.EncryptionType != "" {
|
||||
meta[crypto.SSEHeader] = crypto.SSEAlgorithmAES256
|
||||
}
|
||||
return ObjectOptions{
|
||||
Versioned: globalBucketVersioningSys.Enabled(bucket),
|
||||
VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
|
||||
UserDefined: meta,
|
||||
}
|
||||
}
|
||||
for k, v := range objInfo.UserDefined {
|
||||
meta[k] = v
|
||||
}
|
||||
meta[xhttp.AmzObjectTagging] = objInfo.UserTags
|
||||
|
||||
return ObjectOptions{
|
||||
Versioned: globalBucketVersioningSys.Enabled(bucket),
|
||||
VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
|
||||
UserDefined: meta,
|
||||
VersionID: objInfo.VersionID,
|
||||
MTime: objInfo.ModTime,
|
||||
Expires: objInfo.Expires,
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
errRestoreHDRMissing = fmt.Errorf("x-amz-restore header not found")
|
||||
errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
|
||||
)
|
||||
|
||||
// parse x-amz-restore header from user metadata to get the status of ongoing request and expiry of restoration
|
||||
// if any. This header value is of format: ongoing-request=true|false, expires=time
|
||||
func parseRestoreHeaderFromMeta(meta map[string]string) (ongoing bool, expiry time.Time, err error) {
|
||||
restoreHdr, ok := meta[xhttp.AmzRestore]
|
||||
if !ok {
|
||||
return ongoing, expiry, errRestoreHDRMissing
|
||||
}
|
||||
rslc := strings.SplitN(restoreHdr, ",", 2)
|
||||
if len(rslc) != 2 {
|
||||
return ongoing, expiry, errRestoreHDRMalformed
|
||||
}
|
||||
rstatusSlc := strings.SplitN(rslc[0], "=", 2)
|
||||
if len(rstatusSlc) != 2 {
|
||||
return ongoing, expiry, errRestoreHDRMalformed
|
||||
}
|
||||
rExpSlc := strings.SplitN(rslc[1], "=", 2)
|
||||
if len(rExpSlc) != 2 {
|
||||
return ongoing, expiry, errRestoreHDRMalformed
|
||||
}
|
||||
|
||||
expiry, err = time.Parse(http.TimeFormat, rExpSlc[1])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return rstatusSlc[1] == "true", expiry, nil
|
||||
}
|
||||
|
||||
// restoreTransitionedObject is similar to PostObjectRestore from AWS GLACIER
|
||||
// storage class. When PostObjectRestore API is called, a temporary copy of the object
|
||||
// is restored locally to the bucket on source cluster until the restore expiry date.
|
||||
// The copy that was transitioned continues to reside in the transitioned tier.
|
||||
func restoreTransitionedObject(ctx context.Context, bucket, object string, objAPI ObjectLayer, objInfo ObjectInfo, rreq *RestoreObjectRequest, restoreExpiry time.Time) error {
|
||||
var rs *HTTPRangeSpec
|
||||
gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, objInfo, ObjectOptions{
|
||||
VersionID: objInfo.VersionID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gr.Close()
|
||||
hashReader, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size, globalCLIContext.StrictS3Compat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pReader := NewPutObjReader(hashReader, nil, nil)
|
||||
opts := putRestoreOpts(bucket, object, rreq, objInfo)
|
||||
opts.UserDefined[xhttp.AmzRestore] = fmt.Sprintf("ongoing-request=%t, expiry-date=%s", false, restoreExpiry.Format(http.TimeFormat))
|
||||
if _, err := objAPI.PutObject(ctx, bucket, object, pReader, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -168,9 +168,6 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
|
||||
}
|
||||
meta.ReplicationConfigXML = configData
|
||||
case bucketTargetsFile:
|
||||
if !globalIsErasure && !globalIsDistErasure {
|
||||
return NotImplemented{}
|
||||
}
|
||||
meta.BucketTargetsConfigJSON = configData
|
||||
default:
|
||||
return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
|
@ -19,6 +19,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@ -88,6 +89,9 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
if tgt.Type == madmin.ReplicationService {
|
||||
if !globalIsErasure {
|
||||
return NotImplemented{}
|
||||
}
|
||||
if !globalBucketVersioningSys.Enabled(bucket) {
|
||||
return BucketReplicationSourceNotVersioned{Bucket: bucket}
|
||||
}
|
||||
@ -102,6 +106,20 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
}
|
||||
if tgt.Type == madmin.ILMService {
|
||||
if globalBucketVersioningSys.Enabled(bucket) {
|
||||
vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
|
||||
if err != nil {
|
||||
if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
|
||||
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
if vcfg.Status != string(versioning.Enabled) {
|
||||
return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
|
||||
}
|
||||
}
|
||||
}
|
||||
sys.Lock()
|
||||
defer sys.Unlock()
|
||||
|
||||
@ -147,6 +165,9 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
|
||||
return BucketRemoteArnInvalid{Bucket: bucket}
|
||||
}
|
||||
if arn.Type == madmin.ReplicationService {
|
||||
if !globalIsErasure {
|
||||
return NotImplemented{}
|
||||
}
|
||||
// reject removal of remote target if replication configuration is present
|
||||
rcfg, err := getReplicationConfig(ctx, bucket)
|
||||
if err == nil && rcfg.RoleArn == arnStr {
|
||||
@ -155,6 +176,16 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
|
||||
}
|
||||
}
|
||||
}
|
||||
if arn.Type == madmin.ILMService {
|
||||
// reject removal of remote target if lifecycle transition uses this arn
|
||||
config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
|
||||
if err == nil && transitionSCInUse(ctx, config, bucket, arnStr) {
|
||||
if _, ok := sys.arnRemotesMap[arnStr]; ok {
|
||||
return BucketRemoteRemoveDisallowed{Bucket: bucket}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// delete ARN type from list of matching targets
|
||||
sys.Lock()
|
||||
defer sys.Unlock()
|
||||
@ -183,6 +214,44 @@ func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn strin
|
||||
return sys.arnRemotesMap[arn]
|
||||
}
|
||||
|
||||
// GetRemoteTargetWithLabel returns bucket target given a target label
|
||||
func (sys *BucketTargetSys) GetRemoteTargetWithLabel(ctx context.Context, bucket, targetLabel string) *madmin.BucketTarget {
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
for _, t := range sys.targetsMap[bucket] {
|
||||
if strings.ToUpper(t.Label) == strings.ToUpper(targetLabel) {
|
||||
tgt := t.Clone()
|
||||
return &tgt
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetRemoteArnWithLabel returns bucket target's ARN given its target label
|
||||
func (sys *BucketTargetSys) GetRemoteArnWithLabel(ctx context.Context, bucket, tgtLabel string) *madmin.ARN {
|
||||
tgt := sys.GetRemoteTargetWithLabel(ctx, bucket, tgtLabel)
|
||||
if tgt == nil {
|
||||
return nil
|
||||
}
|
||||
arn, err := madmin.ParseARN(tgt.Arn)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return arn
|
||||
}
|
||||
|
||||
// GetRemoteLabelWithArn returns a bucket target's label given its ARN
|
||||
func (sys *BucketTargetSys) GetRemoteLabelWithArn(ctx context.Context, bucket, arnStr string) string {
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
for _, t := range sys.targetsMap[bucket] {
|
||||
if t.Arn == arnStr {
|
||||
return t.Label
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// NewBucketTargetSys - creates new replication system.
|
||||
func NewBucketTargetSys() *BucketTargetSys {
|
||||
return &BucketTargetSys{
|
||||
|
@ -672,12 +672,17 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
IsLatest: meta.oi.IsLatest,
|
||||
NumVersions: meta.numVersions,
|
||||
SuccessorModTime: meta.successorModTime,
|
||||
RestoreOngoing: meta.oi.RestoreOngoing,
|
||||
RestoreExpires: meta.oi.RestoreExpires,
|
||||
TransitionStatus: meta.oi.TransitionStatus,
|
||||
})
|
||||
if i.debug {
|
||||
logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
|
||||
}
|
||||
switch action {
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
default:
|
||||
// No action.
|
||||
return size
|
||||
@ -706,22 +711,28 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
size = obj.Size
|
||||
|
||||
// Recalculate action.
|
||||
action = i.lifeCycle.ComputeAction(
|
||||
lifecycle.ObjectOpts{
|
||||
Name: i.objectPath(),
|
||||
UserTags: obj.UserTags,
|
||||
ModTime: obj.ModTime,
|
||||
VersionID: obj.VersionID,
|
||||
DeleteMarker: obj.DeleteMarker,
|
||||
IsLatest: obj.IsLatest,
|
||||
NumVersions: meta.numVersions,
|
||||
SuccessorModTime: meta.successorModTime,
|
||||
})
|
||||
lcOpts := lifecycle.ObjectOpts{
|
||||
Name: i.objectPath(),
|
||||
UserTags: obj.UserTags,
|
||||
ModTime: obj.ModTime,
|
||||
VersionID: obj.VersionID,
|
||||
DeleteMarker: obj.DeleteMarker,
|
||||
IsLatest: obj.IsLatest,
|
||||
NumVersions: meta.numVersions,
|
||||
SuccessorModTime: meta.successorModTime,
|
||||
RestoreOngoing: obj.RestoreOngoing,
|
||||
RestoreExpires: obj.RestoreExpires,
|
||||
TransitionStatus: obj.TransitionStatus,
|
||||
}
|
||||
action = i.lifeCycle.ComputeAction(lcOpts)
|
||||
|
||||
if i.debug {
|
||||
logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
|
||||
}
|
||||
switch action {
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
|
||||
default:
|
||||
// No action.
|
||||
return size
|
||||
@ -729,7 +740,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
|
||||
opts := ObjectOptions{}
|
||||
switch action {
|
||||
case lifecycle.DeleteVersionAction:
|
||||
case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
|
||||
// Defensive code, should never happen
|
||||
if obj.VersionID == "" {
|
||||
return size
|
||||
@ -744,15 +755,34 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
}
|
||||
}
|
||||
opts.VersionID = obj.VersionID
|
||||
case lifecycle.DeleteAction:
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteRestoredAction:
|
||||
opts.Versioned = globalBucketVersioningSys.Enabled(i.bucket)
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
if obj.TransitionStatus == "" {
|
||||
opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
|
||||
opts.VersionID = obj.VersionID
|
||||
opts.TransitionStatus = lifecycle.TransitionPending
|
||||
if _, err = o.DeleteObject(ctx, obj.Bucket, obj.Name, opts); err != nil {
|
||||
// Assume it is still there.
|
||||
logger.LogIf(ctx, err)
|
||||
return size
|
||||
}
|
||||
}
|
||||
globalTransitionState.queueTransitionTask(obj)
|
||||
return 0
|
||||
}
|
||||
|
||||
obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
|
||||
if err != nil {
|
||||
// Assume it is still there.
|
||||
logger.LogIf(ctx, err)
|
||||
return size
|
||||
if obj.TransitionStatus != "" {
|
||||
if err := deleteTransitionedObject(ctx, o, i.bucket, i.objectPath(), obj, lcOpts, action); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return size
|
||||
}
|
||||
} else {
|
||||
obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
|
||||
if err != nil {
|
||||
// Assume it is still there.
|
||||
logger.LogIf(ctx, err)
|
||||
return size
|
||||
}
|
||||
}
|
||||
|
||||
eventName := event.ObjectRemovedDelete
|
||||
|
@ -271,10 +271,12 @@ func (e Erasure) decode(ctx context.Context, writer io.Writer, readers []io.Read
|
||||
return healRequired, err
|
||||
}
|
||||
}
|
||||
|
||||
if err = e.DecodeDataBlocks(bufs); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return healRequired, err
|
||||
}
|
||||
|
||||
n, err := writeDataBlocks(ctx, writer, bufs, e.dataBlocks, blockOffset, blockLength)
|
||||
if err != nil {
|
||||
return healRequired, err
|
||||
|
@ -142,6 +142,9 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
|
||||
if fi.Deleted {
|
||||
objInfo.ReplicationStatus = replication.StatusType(fi.DeleteMarkerReplicationStatus)
|
||||
}
|
||||
|
||||
objInfo.TransitionStatus = fi.TransitionStatus
|
||||
|
||||
// etag/md5Sum has already been extracted. We need to
|
||||
// remove to avoid it from appearing as part of
|
||||
// response headers. e.g, X-Minio-* or X-Amz-*.
|
||||
@ -158,6 +161,11 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
|
||||
objInfo.StorageClass = globalMinioDefaultStorageClass
|
||||
}
|
||||
objInfo.VersionPurgeStatus = fi.VersionPurgeStatus
|
||||
// set restore status for transitioned object
|
||||
if ongoing, exp, err := parseRestoreHeaderFromMeta(fi.Metadata); err == nil {
|
||||
objInfo.RestoreOngoing = ongoing
|
||||
objInfo.RestoreExpires = exp
|
||||
}
|
||||
// Success.
|
||||
return objInfo
|
||||
}
|
||||
|
@ -23,11 +23,13 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
"github.com/minio/minio/pkg/bucket/replication"
|
||||
"github.com/minio/minio/pkg/mimedb"
|
||||
"github.com/minio/minio/pkg/sync/errgroup"
|
||||
@ -168,13 +170,18 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
|
||||
ObjInfo: objInfo,
|
||||
}, toObjectErr(errMethodNotAllowed, bucket, object)
|
||||
}
|
||||
|
||||
if objInfo.TransitionStatus == lifecycle.TransitionComplete {
|
||||
// If transitioned, stream from transition tier unless object is restored locally or restore date is past.
|
||||
restoreHdr, ok := objInfo.UserDefined[xhttp.AmzRestore]
|
||||
if !ok || !strings.HasPrefix(restoreHdr, "ongoing-request=false") || (!objInfo.RestoreExpires.IsZero() && time.Now().After(objInfo.RestoreExpires)) {
|
||||
return getTransitionedObjectReader(ctx, bucket, object, rs, h, objInfo, opts)
|
||||
}
|
||||
}
|
||||
unlockOnDefer = false
|
||||
fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts, nsUnlocker)
|
||||
if nErr != nil {
|
||||
return nil, nErr
|
||||
}
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
err := er.getObjectWithFileInfo(ctx, bucket, object, off, length, pw, fi, metaArr, onlineDisks)
|
||||
@ -256,8 +263,8 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
|
||||
if err != nil {
|
||||
return toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
var healOnce sync.Once
|
||||
|
||||
for ; partIndex <= lastPartIndex; partIndex++ {
|
||||
if length == totalBytesRead {
|
||||
break
|
||||
@ -313,12 +320,10 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
|
||||
}
|
||||
// Track total bytes read from disk and written to the client.
|
||||
totalBytesRead += partLength
|
||||
|
||||
// partOffset will be valid only for the first part, hence reset it to 0 for
|
||||
// the remaining parts.
|
||||
partOffset = 0
|
||||
} // End of read all parts loop.
|
||||
|
||||
// Return success.
|
||||
return nil
|
||||
}
|
||||
@ -990,6 +995,8 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
fi.VersionID = opts.VersionID
|
||||
}
|
||||
}
|
||||
fi.TransitionStatus = opts.TransitionStatus
|
||||
|
||||
// versioning suspended means we add `null`
|
||||
// version as delete marker
|
||||
// Add delete marker, since we don't have any version specified explicitly.
|
||||
@ -1010,6 +1017,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
ModTime: modTime,
|
||||
DeleteMarkerReplicationStatus: opts.DeleteMarkerReplicationStatus,
|
||||
VersionPurgeStatus: opts.VersionPurgeStatus,
|
||||
TransitionStatus: opts.TransitionStatus,
|
||||
}); err != nil {
|
||||
return objInfo, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
@ -479,7 +479,6 @@ var supportedDummyObjectAPIs = map[string][]string{
|
||||
|
||||
// List of not implemented object APIs
|
||||
var notImplementedObjectResourceNames = map[string]struct{}{
|
||||
"restore": {},
|
||||
"torrent": {},
|
||||
}
|
||||
|
||||
|
@ -66,6 +66,12 @@ const (
|
||||
AmzTagCount = "x-amz-tagging-count"
|
||||
AmzTagDirective = "X-Amz-Tagging-Directive"
|
||||
|
||||
// S3 transition restore
|
||||
AmzRestore = "x-amz-restore"
|
||||
AmzRestoreExpiryDays = "X-Amz-Restore-Expiry-Days"
|
||||
AmzRestoreRequestDate = "X-Amz-Restore-Request-Date"
|
||||
AmzRestoreOutputPath = "x-amz-restore-output-path"
|
||||
|
||||
// S3 extensions
|
||||
AmzCopySourceIfModifiedSince = "x-amz-copy-source-if-modified-since"
|
||||
AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since"
|
||||
|
@ -163,6 +163,15 @@ type ObjectInfo struct {
|
||||
// to a delete marker on an object.
|
||||
DeleteMarker bool
|
||||
|
||||
// TransitionStatus indicates if transition is complete/pending
|
||||
TransitionStatus string
|
||||
|
||||
// RestoreExpires indicates date a restored object expires
|
||||
RestoreExpires time.Time
|
||||
|
||||
// RestoreOngoing indicates if a restore is in progress
|
||||
RestoreOngoing bool
|
||||
|
||||
// A standard MIME type describing the format of the object.
|
||||
ContentType string
|
||||
|
||||
|
@ -36,18 +36,21 @@ type GetObjectInfoFn func(ctx context.Context, bucket, object string, opts Objec
|
||||
|
||||
// ObjectOptions represents object options for ObjectLayer object operations
|
||||
type ObjectOptions struct {
|
||||
ServerSideEncryption encrypt.ServerSide
|
||||
VersionSuspended bool // indicates if the bucket was previously versioned but is currently suspended.
|
||||
Versioned bool // indicates if the bucket is versioned
|
||||
WalkVersions bool // indicates if the we are interested in walking versions
|
||||
VersionID string // Specifies the versionID which needs to be overwritten or read
|
||||
MTime time.Time // Is only set in POST/PUT operations
|
||||
ServerSideEncryption encrypt.ServerSide
|
||||
VersionSuspended bool // indicates if the bucket was previously versioned but is currently suspended.
|
||||
Versioned bool // indicates if the bucket is versioned
|
||||
WalkVersions bool // indicates if the we are interested in walking versions
|
||||
VersionID string // Specifies the versionID which needs to be overwritten or read
|
||||
MTime time.Time // Is only set in POST/PUT operations
|
||||
Expires time.Time // Is only used in POST/PUT operations
|
||||
|
||||
DeleteMarker bool // Is only set in DELETE operations for delete marker replication
|
||||
UserDefined map[string]string // only set in case of POST/PUT operations
|
||||
PartNumber int // only useful in case of GetObject/HeadObject
|
||||
CheckPrecondFn CheckPreconditionFn // only set during GetObject/HeadObject/CopyObjectPart preconditional valuation
|
||||
DeleteMarkerReplicationStatus string // Is only set in DELETE operations
|
||||
VersionPurgeStatus VersionPurgeStatusType // Is only set in DELETE operations for delete marker version to be permanently deleted.
|
||||
TransitionStatus string // status of the transition
|
||||
}
|
||||
|
||||
// BucketOptions represents bucket options for ObjectLayer bucket operations
|
||||
|
@ -242,6 +242,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
|
||||
}
|
||||
metadata["etag"] = etag
|
||||
}
|
||||
|
||||
// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
|
||||
// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
|
||||
if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {
|
||||
|
@ -43,6 +43,7 @@ import (
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/ioutil"
|
||||
"github.com/minio/minio/pkg/trie"
|
||||
@ -590,7 +591,10 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
|
||||
if err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
|
||||
// if object is encrypted, transition content without decrypting.
|
||||
if opts.TransitionStatus == lifecycle.TransitionPending && isEncrypted {
|
||||
isEncrypted = false
|
||||
}
|
||||
var skipLen int64
|
||||
// Calculate range to read (different for
|
||||
// e.g. encrypted/compressed objects)
|
||||
|
@ -21,8 +21,10 @@ import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
@ -42,6 +44,7 @@ import (
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
|
||||
"github.com/minio/minio/pkg/bucket/policy"
|
||||
"github.com/minio/minio/pkg/bucket/replication"
|
||||
@ -3245,3 +3248,185 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
|
||||
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
// RestoreObjectHandler - POST restore object handler.
|
||||
// ----------
|
||||
func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PostRestoreObject")
|
||||
defer logger.AuditLog(w, r, "PostRestoreObject", mustGetClaimsFromToken(r))
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
object, err := url.PathUnescape(vars["object"])
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Fetch object stat info.
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
// Check for auth type to return S3 compatible error.
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.RestoreObjectAction, bucket, object); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEmptyRequestBody), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
objInfo, err := getObjectInfo(ctx, bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if objInfo.TransitionStatus != lifecycle.TransitionComplete {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
rreq, err := parseRestoreRequest(io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
|
||||
apiErr.Description = err.Error()
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// validate the request
|
||||
if err := rreq.validate(ctx, objectAPI); err != nil {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
|
||||
apiErr.Description = err.Error()
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
statusCode := http.StatusOK
|
||||
alreadyRestored := false
|
||||
if err == nil {
|
||||
if objInfo.RestoreOngoing && rreq.Type != SelectRestoreRequest {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectRestoreAlreadyInProgress), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if !objInfo.RestoreOngoing && !objInfo.RestoreExpires.IsZero() {
|
||||
statusCode = http.StatusAccepted
|
||||
alreadyRestored = true
|
||||
}
|
||||
}
|
||||
// set or upgrade restore expiry
|
||||
restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now(), rreq.Days)
|
||||
metadata := cloneMSS(objInfo.UserDefined)
|
||||
|
||||
// update self with restore metadata
|
||||
if rreq.Type != SelectRestoreRequest {
|
||||
objInfo.metadataOnly = true // Perform only metadata updates.
|
||||
ongoingReq := true
|
||||
if alreadyRestored {
|
||||
ongoingReq = false
|
||||
}
|
||||
metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days)
|
||||
metadata[xhttp.AmzRestoreRequestDate] = time.Now().UTC().Format(http.TimeFormat)
|
||||
if alreadyRestored {
|
||||
metadata[xhttp.AmzRestore] = fmt.Sprintf("ongoing-request=%t, expiry-date=%s", ongoingReq, restoreExpiry.Format(http.TimeFormat))
|
||||
} else {
|
||||
metadata[xhttp.AmzRestore] = fmt.Sprintf("ongoing-request=%t", ongoingReq)
|
||||
}
|
||||
objInfo.UserDefined = metadata
|
||||
if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
}, ObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
}); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// for previously restored object, just update the restore expiry
|
||||
if alreadyRestored {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
restoreObject := mustGetUUID()
|
||||
if rreq.OutputLocation.S3.BucketName != "" {
|
||||
w.Header()[xhttp.AmzRestoreOutputPath] = []string{pathJoin(rreq.OutputLocation.S3.BucketName, rreq.OutputLocation.S3.Prefix, restoreObject)}
|
||||
}
|
||||
w.WriteHeader(statusCode)
|
||||
// Notify object restore started via a POST request.
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectRestorePostInitiated,
|
||||
BucketName: bucket,
|
||||
Object: objInfo,
|
||||
ReqParams: extractReqParams(r),
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: handlers.GetSourceIP(r),
|
||||
})
|
||||
// now process the restore in background
|
||||
go func() {
|
||||
rctx := GlobalContext
|
||||
if !rreq.SelectParameters.IsEmpty() {
|
||||
getObject := func(offset, length int64) (rc io.ReadCloser, err error) {
|
||||
isSuffixLength := false
|
||||
if offset < 0 {
|
||||
isSuffixLength = true
|
||||
}
|
||||
|
||||
rs := &HTTPRangeSpec{
|
||||
IsSuffixLength: isSuffixLength,
|
||||
Start: offset,
|
||||
End: offset + length,
|
||||
}
|
||||
|
||||
return getTransitionedObjectReader(rctx, bucket, object, rs, r.Header, objInfo, ObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
})
|
||||
}
|
||||
if err = rreq.SelectParameters.Open(getObject); err != nil {
|
||||
if serr, ok := err.(s3select.SelectError); ok {
|
||||
encodedErrorResponse := encodeResponse(APIErrorResponse{
|
||||
Code: serr.ErrorCode(),
|
||||
Message: serr.ErrorMessage(),
|
||||
BucketName: bucket,
|
||||
Key: object,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
})
|
||||
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
|
||||
} else {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
nr := httptest.NewRecorder()
|
||||
rw := logger.NewResponseWriter(nr)
|
||||
rw.LogErrBody = true
|
||||
rw.LogAllBody = true
|
||||
rreq.SelectParameters.Evaluate(rw)
|
||||
rreq.SelectParameters.Close()
|
||||
return
|
||||
}
|
||||
if err := restoreTransitionedObject(rctx, bucket, object, objectAPI, objInfo, rreq, restoreExpiry); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Notify object restore completed via a POST request.
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectRestorePostCompleted,
|
||||
BucketName: bucket,
|
||||
Object: objInfo,
|
||||
ReqParams: extractReqParams(r),
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: handlers.GetSourceIP(r),
|
||||
})
|
||||
}()
|
||||
}
|
||||
|
@ -425,6 +425,8 @@ func serverMain(ctx *cli.Context) {
|
||||
globalAllHealState = newHealState()
|
||||
globalBackgroundHealState = newHealState()
|
||||
globalReplicationState = newReplicationState()
|
||||
globalTransitionState = newTransitionState()
|
||||
|
||||
}
|
||||
|
||||
// Configure server.
|
||||
@ -479,6 +481,7 @@ func serverMain(ctx *cli.Context) {
|
||||
if globalIsErasure {
|
||||
initAutoHeal(GlobalContext, newObject)
|
||||
initBackgroundReplication(GlobalContext, newObject)
|
||||
initBackgroundTransition(GlobalContext, newObject)
|
||||
}
|
||||
|
||||
if err = initServer(GlobalContext, newObject); err != nil {
|
||||
|
@ -112,6 +112,10 @@ type FileInfo struct {
|
||||
// a deleted marker for a versioned bucket.
|
||||
Deleted bool
|
||||
|
||||
// TransitionStatus is set to Pending/Complete for transitioned
|
||||
// entries based on state of transition
|
||||
TransitionStatus string
|
||||
|
||||
// DataDir of the file
|
||||
DataDir string
|
||||
|
||||
|
@ -342,8 +342,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if zb0001 != 16 {
|
||||
err = msgp.ArrayError{Wanted: 16, Got: zb0001}
|
||||
if zb0001 != 17 {
|
||||
err = msgp.ArrayError{Wanted: 17, Got: zb0001}
|
||||
return
|
||||
}
|
||||
z.Volume, err = dc.ReadString()
|
||||
@ -371,6 +371,11 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "Deleted")
|
||||
return
|
||||
}
|
||||
z.TransitionStatus, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "TransitionStatus")
|
||||
return
|
||||
}
|
||||
z.DataDir, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "DataDir")
|
||||
@ -472,8 +477,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// array header, size 16
|
||||
err = en.Append(0xdc, 0x0, 0x10)
|
||||
// array header, size 17
|
||||
err = en.Append(0xdc, 0x0, 0x11)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@ -502,6 +507,11 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "Deleted")
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.TransitionStatus)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "TransitionStatus")
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.DataDir)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "DataDir")
|
||||
@ -582,13 +592,14 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// array header, size 16
|
||||
o = append(o, 0xdc, 0x0, 0x10)
|
||||
// array header, size 17
|
||||
o = append(o, 0xdc, 0x0, 0x11)
|
||||
o = msgp.AppendString(o, z.Volume)
|
||||
o = msgp.AppendString(o, z.Name)
|
||||
o = msgp.AppendString(o, z.VersionID)
|
||||
o = msgp.AppendBool(o, z.IsLatest)
|
||||
o = msgp.AppendBool(o, z.Deleted)
|
||||
o = msgp.AppendString(o, z.TransitionStatus)
|
||||
o = msgp.AppendString(o, z.DataDir)
|
||||
o = msgp.AppendBool(o, z.XLV1)
|
||||
o = msgp.AppendTime(o, z.ModTime)
|
||||
@ -626,8 +637,8 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if zb0001 != 16 {
|
||||
err = msgp.ArrayError{Wanted: 16, Got: zb0001}
|
||||
if zb0001 != 17 {
|
||||
err = msgp.ArrayError{Wanted: 17, Got: zb0001}
|
||||
return
|
||||
}
|
||||
z.Volume, bts, err = msgp.ReadStringBytes(bts)
|
||||
@ -655,6 +666,11 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "Deleted")
|
||||
return
|
||||
}
|
||||
z.TransitionStatus, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "TransitionStatus")
|
||||
return
|
||||
}
|
||||
z.DataDir, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "DataDir")
|
||||
@ -757,7 +773,7 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *FileInfo) Msgsize() (s int) {
|
||||
s = 3 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.StringPrefixSize + len(z.VersionID) + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.DataDir) + msgp.BoolSize + msgp.TimeSize + msgp.Int64Size + msgp.Uint32Size + msgp.MapHeaderSize
|
||||
s = 3 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.StringPrefixSize + len(z.VersionID) + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.TransitionStatus) + msgp.StringPrefixSize + len(z.DataDir) + msgp.BoolSize + msgp.TimeSize + msgp.Int64Size + msgp.Uint32Size + msgp.MapHeaderSize
|
||||
if z.Metadata != nil {
|
||||
for za0001, za0002 := range z.Metadata {
|
||||
_ = za0002
|
||||
|
@ -182,7 +182,11 @@ func (m *xlMetaV1Object) ToFileInfo(volume, path string) (FileInfo, error) {
|
||||
if !m.valid() {
|
||||
return FileInfo{}, errFileCorrupt
|
||||
}
|
||||
return FileInfo{
|
||||
var transitionStatus string
|
||||
if st, ok := m.Meta[ReservedMetadataPrefixLower+"transition-status"]; ok {
|
||||
transitionStatus = st
|
||||
}
|
||||
fi := FileInfo{
|
||||
Volume: volume,
|
||||
Name: path,
|
||||
ModTime: m.Stat.ModTime,
|
||||
@ -192,7 +196,11 @@ func (m *xlMetaV1Object) ToFileInfo(volume, path string) (FileInfo, error) {
|
||||
Erasure: m.Erasure,
|
||||
VersionID: m.VersionID,
|
||||
DataDir: m.DataDir,
|
||||
}, nil
|
||||
}
|
||||
if transitionStatus != "" {
|
||||
fi.TransitionStatus = transitionStatus
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
// XL metadata constants.
|
||||
|
@ -425,6 +425,10 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
|
||||
fi.Erasure.Distribution[i] = int(j.ErasureDist[i])
|
||||
}
|
||||
fi.DataDir = uuid.UUID(j.DataDir).String()
|
||||
|
||||
if st, ok := j.MetaSys[ReservedMetadataPrefixLower+"transition-status"]; ok {
|
||||
fi.TransitionStatus = string(st)
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
@ -490,6 +494,11 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
|
||||
switch version.Type {
|
||||
case LegacyType:
|
||||
if version.ObjectV1.VersionID == fi.VersionID {
|
||||
if fi.TransitionStatus != "" {
|
||||
z.Versions[i].ObjectV1.Meta[ReservedMetadataPrefixLower+"transition-status"] = fi.TransitionStatus
|
||||
return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil
|
||||
}
|
||||
|
||||
z.Versions = append(z.Versions[:i], z.Versions[i+1:]...)
|
||||
if fi.Deleted {
|
||||
z.Versions = append(z.Versions, ventry)
|
||||
@ -543,6 +552,10 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
|
||||
switch version.Type {
|
||||
case ObjectType:
|
||||
if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) {
|
||||
if fi.TransitionStatus != "" {
|
||||
z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+"transition-status"] = []byte(fi.TransitionStatus)
|
||||
return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil
|
||||
}
|
||||
z.Versions = append(z.Versions[:i], z.Versions[i+1:]...)
|
||||
if findDataDir(version.ObjectV2.DataDir, z.Versions) > 0 {
|
||||
if fi.Deleted {
|
||||
|
@ -44,6 +44,7 @@ import (
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/config/storageclass"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
xioutil "github.com/minio/minio/pkg/ioutil"
|
||||
@ -1008,8 +1009,10 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
return err
|
||||
}
|
||||
|
||||
// when data-dir is specified.
|
||||
if dataDir != "" {
|
||||
// when data-dir is specified. Transition leverages existing DeleteObject
|
||||
// api call to mark object as deleted.When object is pending transition,
|
||||
// just update the metadata and avoid deleting data dir.
|
||||
if dataDir != "" && fi.TransitionStatus != lifecycle.TransitionPending {
|
||||
filePath := pathJoin(volumeDir, path, dataDir)
|
||||
if err = checkPathLength(filePath); err != nil {
|
||||
return err
|
||||
@ -1019,8 +1022,9 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if !lastVersion {
|
||||
// transitioned objects maintains metadata on the source cluster. When transition
|
||||
// status is set, update the metadata to disk.
|
||||
if !lastVersion || fi.TransitionStatus != "" {
|
||||
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf)
|
||||
}
|
||||
|
||||
@ -1164,7 +1168,11 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
|
||||
if err != nil {
|
||||
return fi, err
|
||||
}
|
||||
|
||||
// skip checking data dir when object is transitioned - after transition, data dir will
|
||||
// be empty with just the metadata left behind.
|
||||
if fi.TransitionStatus != "" {
|
||||
checkDataDir = false
|
||||
}
|
||||
if fi.DataDir != "" && checkDataDir {
|
||||
if _, err = s.StatVol(ctx, pathJoin(volume, path, fi.DataDir, slashSeparator)); err != nil {
|
||||
if err == errVolumeNotFound {
|
||||
|
@ -46,7 +46,7 @@ Lifecycle configuration imported successfully to `play/testbucket`.
|
||||
|
||||
- List the current settings
|
||||
```
|
||||
$ mc ilm list play/testbucket
|
||||
$ mc ilm ls play/testbucket
|
||||
ID | Prefix | Enabled | Expiry | Date/Days | Transition | Date/Days | Storage-Class | Tags
|
||||
------------|----------|------------|--------|--------------|--------------|------------------|------------------|------------------
|
||||
OldPictures | old/ | ✓ | ✓ | 1 Jan 2020 | ✗ | | |
|
||||
@ -100,7 +100,32 @@ When an object has only one version as a delete marker, the latter can be automa
|
||||
}
|
||||
```
|
||||
|
||||
## 4. Enable ILM transition feature
|
||||
|
||||
In Erasure mode, MinIO supports transitioning of older objects to a different cluster by setting up transition rules in the bucket lifecycle configuration. This allows applications to optimize storage costs by moving less frequently accessed data to a cheaper storage without compromising accessibility of data.
|
||||
|
||||
To transition objects in a bucket to a destination bucket on a different cluster, applications need to specify a transition ARN instead of storage class while setting up the ILM lifecycle rule.
|
||||
|
||||
>To create a transition ARN for transitioning objects in srcbucket to a destbucket on cluster endpoint https://transition-endpoint:9000 using `mc`:
|
||||
|
||||
```
|
||||
mc admin bucket remote add myminio/srcbucket https://accessKey:secretKey@transition-endpoint:9000/destbucket --service ilm --region us-east-1 --label "HDDTier"
|
||||
Role ARN = 'arn:minio:ilm:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket'
|
||||
```
|
||||
> The access credentials on the target site needs *s3:GetBucketVersioning* permission if versioning is enabled on the `destbucket` bucket.
|
||||
|
||||
Using above ARN, set up a lifecycle rule with transition:
|
||||
```
|
||||
mc ilm add --expiry-days 365 --transition-days 45 --storage-class "HDDTier" myminio/srcbucket
|
||||
```
|
||||
|
||||
Once transitioned, GET or HEAD on the object will stream the content from the transitioned tier. In the event that the object needs to be restored temporarily to the local cluster, the AWS [RestoreObject API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html) can be utilized.
|
||||
|
||||
```
|
||||
aws s3api restore-object --bucket srcbucket \
|
||||
--key object \
|
||||
--restore-request Days=3
|
||||
```
|
||||
|
||||
## Explore Further
|
||||
- [MinIO | Golang Client API Reference](https://docs.min.io/docs/golang-client-api-reference.html#SetBucketLifecycle)
|
||||
|
@ -30,6 +30,13 @@ var (
|
||||
errXMLNotWellFormed = Errorf("The XML you provided was not well-formed or did not validate against our published schema")
|
||||
)
|
||||
|
||||
const (
|
||||
// TransitionComplete marks completed transition
|
||||
TransitionComplete = "complete"
|
||||
// TransitionPending - transition is yet to be attempted
|
||||
TransitionPending = "pending"
|
||||
)
|
||||
|
||||
// Action represents a delete action or other transition
|
||||
// actions that will be implemented later.
|
||||
type Action int
|
||||
@ -39,10 +46,18 @@ type Action int
|
||||
const (
|
||||
// NoneAction means no action required after evaluting lifecycle rules
|
||||
NoneAction Action = iota
|
||||
// DeleteAction means the object needs to be removed after evaluting lifecycle rules
|
||||
// DeleteAction means the object needs to be removed after evaluating lifecycle rules
|
||||
DeleteAction
|
||||
// DeleteVersionAction deletes a particular version
|
||||
DeleteVersionAction
|
||||
// TransitionAction transitions a particular object after evaluating lifecycle transition rules
|
||||
TransitionAction
|
||||
//TransitionVersionAction transitions a particular object version after evaluating lifecycle transition rules
|
||||
TransitionVersionAction
|
||||
// DeleteRestoredAction means the temporarily restored object needs to be removed after evaluating lifecycle rules
|
||||
DeleteRestoredAction
|
||||
// DeleteRestoredVersionAction deletes a particular version that was temporarily restored
|
||||
DeleteRestoredVersionAction
|
||||
)
|
||||
|
||||
// Lifecycle - Configuration for bucket lifecycle.
|
||||
@ -85,13 +100,18 @@ func (lc Lifecycle) HasActiveRules(prefix string, recursive bool) bool {
|
||||
if rule.NoncurrentVersionTransition.NoncurrentDays > 0 {
|
||||
return true
|
||||
}
|
||||
if rule.Expiration.IsNull() {
|
||||
if rule.Expiration.IsNull() && rule.Transition.IsNull() {
|
||||
continue
|
||||
}
|
||||
if !rule.Expiration.IsDateNull() && rule.Expiration.Date.After(time.Now()) {
|
||||
continue
|
||||
if !rule.Expiration.IsDateNull() && rule.Expiration.Date.Before(time.Now()) {
|
||||
return true
|
||||
}
|
||||
if !rule.Transition.IsDateNull() && rule.Transition.Date.Before(time.Now()) {
|
||||
return true
|
||||
}
|
||||
if !rule.Expiration.IsDaysNull() || !rule.Transition.IsDaysNull() {
|
||||
return true
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@ -160,15 +180,26 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
|
||||
continue
|
||||
}
|
||||
// The NoncurrentVersionExpiration action requests MinIO to expire
|
||||
// noncurrent versions of objects 100 days after the objects become
|
||||
// noncurrent versions of objects x days after the objects become
|
||||
// noncurrent.
|
||||
if !rule.NoncurrentVersionExpiration.IsDaysNull() {
|
||||
rules = append(rules, rule)
|
||||
continue
|
||||
}
|
||||
// The NoncurrentVersionTransition action requests MinIO to transition
|
||||
// noncurrent versions of objects x days after the objects become
|
||||
// noncurrent.
|
||||
if !rule.NoncurrentVersionTransition.IsDaysNull() {
|
||||
rules = append(rules, rule)
|
||||
continue
|
||||
}
|
||||
|
||||
if rule.Filter.TestTags(strings.Split(obj.UserTags, "&")) {
|
||||
rules = append(rules, rule)
|
||||
}
|
||||
if !rule.Transition.IsNull() {
|
||||
rules = append(rules, rule)
|
||||
}
|
||||
}
|
||||
return rules
|
||||
}
|
||||
@ -184,6 +215,9 @@ type ObjectOpts struct {
|
||||
DeleteMarker bool
|
||||
NumVersions int
|
||||
SuccessorModTime time.Time
|
||||
TransitionStatus string
|
||||
RestoreOngoing bool
|
||||
RestoreExpires time.Time
|
||||
}
|
||||
|
||||
// ComputeAction returns the action to perform by evaluating all lifecycle rules
|
||||
@ -207,11 +241,20 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
|
||||
if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() {
|
||||
// Non current versions should be deleted if their age exceeds non current days configuration
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
|
||||
if time.Now().After(expectedExpiryTime(obj.SuccessorModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) {
|
||||
if time.Now().After(ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionExpiration.NoncurrentDays))) {
|
||||
return DeleteVersionAction
|
||||
}
|
||||
}
|
||||
}
|
||||
if !rule.NoncurrentVersionTransition.IsDaysNull() {
|
||||
if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() && obj.TransitionStatus != TransitionComplete {
|
||||
// Non current versions should be deleted if their age exceeds non current days configuration
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
|
||||
if time.Now().After(ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionTransition.NoncurrentDays))) {
|
||||
return TransitionVersionAction
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the object or simply add a delete marker (once) in a versioned bucket
|
||||
if obj.VersionID == "" || obj.IsLatest && !obj.DeleteMarker {
|
||||
@ -221,22 +264,42 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
|
||||
action = DeleteAction
|
||||
}
|
||||
case !rule.Expiration.IsDaysNull():
|
||||
if time.Now().UTC().After(expectedExpiryTime(obj.ModTime, rule.Expiration.Days)) {
|
||||
if time.Now().UTC().After(ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days))) {
|
||||
action = DeleteAction
|
||||
}
|
||||
}
|
||||
if action == NoneAction {
|
||||
if obj.TransitionStatus != TransitionComplete {
|
||||
switch {
|
||||
case !rule.Transition.IsDateNull():
|
||||
if time.Now().UTC().After(rule.Transition.Date.Time) {
|
||||
action = TransitionAction
|
||||
}
|
||||
case !rule.Transition.IsDaysNull():
|
||||
if time.Now().UTC().After(ExpectedExpiryTime(obj.ModTime, int(rule.Transition.Days))) {
|
||||
action = TransitionAction
|
||||
}
|
||||
}
|
||||
}
|
||||
if !obj.RestoreExpires.IsZero() && time.Now().After(obj.RestoreExpires) {
|
||||
if obj.VersionID != "" {
|
||||
action = DeleteRestoredVersionAction
|
||||
} else {
|
||||
action = DeleteRestoredAction
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return action
|
||||
}
|
||||
|
||||
// expectedExpiryTime calculates the expiry date/time based on a object modtime.
|
||||
// The expected expiry time is always a midnight time following the the object
|
||||
// modification time plus the number of expiration days.
|
||||
// ExpectedExpiryTime calculates the expiry, transition or restore date/time based on a object modtime.
|
||||
// The expected transition or restore time is always a midnight time following the the object
|
||||
// modification time plus the number of transition/restore days.
|
||||
// e.g. If the object modtime is `Thu May 21 13:42:50 GMT 2020` and the object should
|
||||
// expire in 1 day, then the expected expiry time is `Fri, 23 May 2020 00:00:00 GMT`
|
||||
func expectedExpiryTime(modTime time.Time, days ExpirationDays) time.Time {
|
||||
// transition in 1 day, then the expected transition time is `Fri, 23 May 2020 00:00:00 GMT`
|
||||
func ExpectedExpiryTime(modTime time.Time, days int) time.Time {
|
||||
t := modTime.UTC().Add(time.Duration(days+1) * 24 * time.Hour)
|
||||
return t.Truncate(24 * time.Hour)
|
||||
}
|
||||
@ -256,7 +319,7 @@ func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
|
||||
// expiration date and its associated rule ID.
|
||||
for _, rule := range lc.FilterActionableRules(obj) {
|
||||
if !rule.NoncurrentVersionExpiration.IsDaysNull() && !obj.IsLatest && obj.VersionID != "" {
|
||||
return rule.ID, expectedExpiryTime(time.Now(), ExpirationDays(rule.NoncurrentVersionExpiration.NoncurrentDays))
|
||||
return rule.ID, ExpectedExpiryTime(time.Now(), int(rule.NoncurrentVersionExpiration.NoncurrentDays))
|
||||
}
|
||||
|
||||
if !rule.Expiration.IsDateNull() {
|
||||
@ -266,7 +329,7 @@ func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
|
||||
}
|
||||
}
|
||||
if !rule.Expiration.IsDaysNull() {
|
||||
expectedExpiry := expectedExpiryTime(obj.ModTime, rule.Expiration.Days)
|
||||
expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days))
|
||||
if finalExpiryDate.IsZero() || finalExpiryDate.After(expectedExpiry) {
|
||||
finalExpiryRuleID = rule.ID
|
||||
finalExpiryDate = expectedExpiry
|
||||
|
@ -240,7 +240,7 @@ func TestExpectedExpiryTime(t *testing.T) {
|
||||
|
||||
for i, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
|
||||
got := expectedExpiryTime(tc.modTime, tc.days)
|
||||
got := ExpectedExpiryTime(tc.modTime, int(tc.days))
|
||||
if !got.Equal(tc.expected) {
|
||||
t.Fatalf("Expected %v to be equal to %v", got, tc.expected)
|
||||
}
|
||||
|
@ -32,10 +32,6 @@ type NoncurrentVersionTransition struct {
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
}
|
||||
|
||||
var (
|
||||
errNoncurrentVersionTransitionUnsupported = Errorf("Specifying <NoncurrentVersionTransition></NoncurrentVersionTransition> is not supported")
|
||||
)
|
||||
|
||||
// MarshalXML if non-current days not set to non zero value
|
||||
func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
if n.IsDaysNull() {
|
||||
@ -50,13 +46,6 @@ func (n NoncurrentVersionExpiration) IsDaysNull() bool {
|
||||
return n.NoncurrentDays == ExpirationDays(0)
|
||||
}
|
||||
|
||||
// UnmarshalXML is extended to indicate lack of support for
|
||||
// NoncurrentVersionTransition xml tag in object lifecycle
|
||||
// configuration
|
||||
func (n NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
|
||||
return errNoncurrentVersionTransitionUnsupported
|
||||
}
|
||||
|
||||
// MarshalXML is extended to leave out
|
||||
// <NoncurrentVersionTransition></NoncurrentVersionTransition> tags
|
||||
func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
@ -65,3 +54,8 @@ func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartE
|
||||
}
|
||||
return e.EncodeElement(&n, start)
|
||||
}
|
||||
|
||||
// IsDaysNull returns true if days field is null
|
||||
func (n NoncurrentVersionTransition) IsDaysNull() bool {
|
||||
return n.NoncurrentDays == ExpirationDays(0)
|
||||
}
|
||||
|
@ -100,6 +100,10 @@ func (r Rule) validateFilter() error {
|
||||
return r.Filter.Validate()
|
||||
}
|
||||
|
||||
func (r Rule) validateTransition() error {
|
||||
return r.Transition.Validate()
|
||||
}
|
||||
|
||||
// Prefix - a rule can either have prefix under <filter></filter> or under
|
||||
// <filter><and></and></filter>. This method returns the prefix from the
|
||||
// location where it is available
|
||||
@ -147,5 +151,8 @@ func (r Rule) Validate() error {
|
||||
if err := r.validateFilter(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.validateTransition(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
@ -22,39 +22,6 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestUnsupportedRules checks if Rule xml with unsuported tags return
|
||||
// appropriate errors on parsing
|
||||
func TestUnsupportedRules(t *testing.T) {
|
||||
// NoncurrentVersionTransition, and Transition tags aren't supported
|
||||
unsupportedTestCases := []struct {
|
||||
inputXML string
|
||||
expectedErr error
|
||||
}{
|
||||
{ // Rule with unsupported NoncurrentVersionTransition
|
||||
inputXML: ` <Rule>
|
||||
<NoncurrentVersionTransition></NoncurrentVersionTransition>
|
||||
</Rule>`,
|
||||
expectedErr: errNoncurrentVersionTransitionUnsupported,
|
||||
},
|
||||
{ // Rule with unsupported Transition action
|
||||
inputXML: ` <Rule>
|
||||
<Transition></Transition>
|
||||
</Rule>`,
|
||||
expectedErr: errTransitionUnsupported,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range unsupportedTestCases {
|
||||
t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
|
||||
var rule Rule
|
||||
err := xml.Unmarshal([]byte(tc.inputXML), &rule)
|
||||
if err != tc.expectedErr {
|
||||
t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestInvalidRules checks if Rule xml with invalid elements returns
|
||||
// appropriate errors on validation
|
||||
func TestInvalidRules(t *testing.T) {
|
||||
|
@ -18,25 +18,147 @@ package lifecycle
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
errTransitionInvalidDays = Errorf("Days must be 0 or greater when used with Transition")
|
||||
errTransitionInvalidDate = Errorf("Date must be provided in ISO 8601 format")
|
||||
errTransitionInvalid = Errorf("Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present inside Expiration.")
|
||||
errTransitionDateNotMidnight = Errorf("'Date' must be at midnight GMT")
|
||||
)
|
||||
|
||||
// TransitionDate is a embedded type containing time.Time to unmarshal
|
||||
// Date in Transition
|
||||
type TransitionDate struct {
|
||||
time.Time
|
||||
}
|
||||
|
||||
// UnmarshalXML parses date from Transition and validates date format
|
||||
func (tDate *TransitionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
|
||||
var dateStr string
|
||||
err := d.DecodeElement(&dateStr, &startElement)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// While AWS documentation mentions that the date specified
|
||||
// must be present in ISO 8601 format, in reality they allow
|
||||
// users to provide RFC 3339 compliant dates.
|
||||
trnDate, err := time.Parse(time.RFC3339, dateStr)
|
||||
if err != nil {
|
||||
return errTransitionInvalidDate
|
||||
}
|
||||
// Allow only date timestamp specifying midnight GMT
|
||||
hr, min, sec := trnDate.Clock()
|
||||
nsec := trnDate.Nanosecond()
|
||||
loc := trnDate.Location()
|
||||
if !(hr == 0 && min == 0 && sec == 0 && nsec == 0 && loc.String() == time.UTC.String()) {
|
||||
return errTransitionDateNotMidnight
|
||||
}
|
||||
|
||||
*tDate = TransitionDate{trnDate}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalXML encodes expiration date if it is non-zero and encodes
|
||||
// empty string otherwise
|
||||
func (tDate TransitionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
|
||||
if tDate.Time.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return e.EncodeElement(tDate.Format(time.RFC3339), startElement)
|
||||
}
|
||||
|
||||
// TransitionDays is a type alias to unmarshal Days in Transition
|
||||
type TransitionDays int
|
||||
|
||||
// UnmarshalXML parses number of days from Transition and validates if
|
||||
// >= 0
|
||||
func (tDays *TransitionDays) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
|
||||
var numDays int
|
||||
err := d.DecodeElement(&numDays, &startElement)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if numDays < 0 {
|
||||
return errTransitionInvalidDays
|
||||
}
|
||||
*tDays = TransitionDays(numDays)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalXML encodes number of days to expire if it is non-zero and
|
||||
// encodes empty string otherwise
|
||||
func (tDays TransitionDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
|
||||
if tDays == 0 {
|
||||
return nil
|
||||
}
|
||||
return e.EncodeElement(int(tDays), startElement)
|
||||
}
|
||||
|
||||
// Transition - transition actions for a rule in lifecycle configuration.
|
||||
type Transition struct {
|
||||
XMLName xml.Name `xml:"Transition"`
|
||||
Days int `xml:"Days,omitempty"`
|
||||
Date string `xml:"Date,omitempty"`
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
XMLName xml.Name `xml:"Transition"`
|
||||
Days TransitionDays `xml:"Days,omitempty"`
|
||||
Date TransitionDate `xml:"Date,omitempty"`
|
||||
StorageClass string `xml:"StorageClass,omitempty"`
|
||||
|
||||
set bool
|
||||
}
|
||||
|
||||
var errTransitionUnsupported = Errorf("Specifying <Transition></Transition> tag is not supported")
|
||||
|
||||
// UnmarshalXML is extended to indicate lack of support for Transition
|
||||
// xml tag in object lifecycle configuration
|
||||
func (t Transition) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
return errTransitionUnsupported
|
||||
// MarshalXML encodes transition field into an XML form.
|
||||
func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
|
||||
if !t.set {
|
||||
return nil
|
||||
}
|
||||
type transitionWrapper Transition
|
||||
return enc.EncodeElement(transitionWrapper(t), start)
|
||||
}
|
||||
|
||||
// MarshalXML is extended to leave out <Transition></Transition> tags
|
||||
func (t Transition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
// UnmarshalXML decodes transition field from the XML form.
|
||||
func (t *Transition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
|
||||
type transitionWrapper Transition
|
||||
var trw transitionWrapper
|
||||
err := d.DecodeElement(&trw, &startElement)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*t = Transition(trw)
|
||||
t.set = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate - validates the "Expiration" element
|
||||
func (t Transition) Validate() error {
|
||||
if !t.set {
|
||||
return nil
|
||||
}
|
||||
|
||||
if t.IsDaysNull() && t.IsDateNull() {
|
||||
return errXMLNotWellFormed
|
||||
}
|
||||
|
||||
// Both transition days and date are specified
|
||||
if !t.IsDaysNull() && !t.IsDateNull() {
|
||||
return errTransitionInvalid
|
||||
}
|
||||
if t.StorageClass == "" {
|
||||
return errXMLNotWellFormed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsDaysNull returns true if days field is null
|
||||
func (t Transition) IsDaysNull() bool {
|
||||
return t.Days == TransitionDays(0)
|
||||
}
|
||||
|
||||
// IsDateNull returns true if date field is null
|
||||
func (t Transition) IsDateNull() bool {
|
||||
return t.Date.Time.IsZero()
|
||||
}
|
||||
|
||||
// IsNull returns true if both date and days fields are null
|
||||
func (t Transition) IsNull() bool {
|
||||
return t.IsDaysNull() && t.IsDateNull()
|
||||
}
|
||||
|
@ -169,6 +169,9 @@ const (
|
||||
|
||||
// GetObjectVersionForReplicationAction - GetObjectVersionForReplication REST API action
|
||||
GetObjectVersionForReplicationAction = "s3:GetObjectVersionForReplication"
|
||||
|
||||
// RestoreObjectAction - RestoreObject REST API action
|
||||
RestoreObjectAction = "s3:RestoreObject"
|
||||
)
|
||||
|
||||
// List of all supported object actions.
|
||||
@ -195,6 +198,7 @@ var supportedObjectActions = map[Action]struct{}{
|
||||
ReplicateDeleteAction: {},
|
||||
ReplicateTagsAction: {},
|
||||
GetObjectVersionForReplicationAction: {},
|
||||
RestoreObjectAction: {},
|
||||
}
|
||||
|
||||
// isObjectAction - returns whether action is object type or not.
|
||||
@ -255,6 +259,7 @@ var supportedActions = map[Action]struct{}{
|
||||
ReplicateDeleteAction: {},
|
||||
ReplicateTagsAction: {},
|
||||
GetObjectVersionForReplicationAction: {},
|
||||
RestoreObjectAction: {},
|
||||
}
|
||||
|
||||
// IsValid - checks if action is valid or not.
|
||||
@ -410,4 +415,5 @@ var actionConditionKeyMap = map[Action]condition.KeySet{
|
||||
ReplicateDeleteAction: condition.NewKeySet(condition.CommonKeys...),
|
||||
ReplicateTagsAction: condition.NewKeySet(condition.CommonKeys...),
|
||||
GetObjectVersionForReplicationAction: condition.NewKeySet(condition.CommonKeys...),
|
||||
RestoreObjectAction: condition.NewKeySet(condition.CommonKeys...),
|
||||
}
|
||||
|
@ -52,6 +52,9 @@ const (
|
||||
ObjectReplicationMissedThreshold
|
||||
ObjectReplicationReplicatedAfterThreshold
|
||||
ObjectReplicationNotTracked
|
||||
ObjectRestorePostInitiated
|
||||
ObjectRestorePostCompleted
|
||||
ObjectRestorePostAll
|
||||
)
|
||||
|
||||
// Expand - returns expanded values of abbreviated event type.
|
||||
@ -86,6 +89,11 @@ func (name Name) Expand() []Name {
|
||||
ObjectReplicationMissedThreshold,
|
||||
ObjectReplicationReplicatedAfterThreshold,
|
||||
}
|
||||
case ObjectRestorePostAll:
|
||||
return []Name{
|
||||
ObjectRestorePostInitiated,
|
||||
ObjectRestorePostCompleted,
|
||||
}
|
||||
default:
|
||||
return []Name{name}
|
||||
}
|
||||
@ -140,6 +148,10 @@ func (name Name) String() string {
|
||||
return "s3:Replication:OperationMissedThreshold"
|
||||
case ObjectReplicationReplicatedAfterThreshold:
|
||||
return "s3:Replication:OperationReplicatedAfterThreshold"
|
||||
case ObjectRestorePostInitiated:
|
||||
return "s3:ObjectRestore:Post"
|
||||
case ObjectRestorePostCompleted:
|
||||
return "s3:ObjectRestore:Completed"
|
||||
}
|
||||
|
||||
return ""
|
||||
@ -236,6 +248,13 @@ func ParseName(s string) (Name, error) {
|
||||
return ObjectReplicationReplicatedAfterThreshold, nil
|
||||
case "s3:Replication:OperationNotTracked":
|
||||
return ObjectReplicationNotTracked, nil
|
||||
case "s3:ObjectRestore:*":
|
||||
return ObjectRestorePostAll, nil
|
||||
case "s3:ObjectRestore:Post":
|
||||
return ObjectRestorePostInitiated, nil
|
||||
case "s3:ObjectRestore:Completed":
|
||||
return ObjectRestorePostCompleted, nil
|
||||
|
||||
default:
|
||||
return 0, &ErrInvalidEventName{s}
|
||||
}
|
||||
|
@ -35,11 +35,13 @@ type ServiceType string
|
||||
const (
|
||||
// ReplicationService specifies replication service
|
||||
ReplicationService ServiceType = "replication"
|
||||
// ILMService specifies ilm service
|
||||
ILMService ServiceType = "ilm"
|
||||
)
|
||||
|
||||
// IsValid returns true if ARN type represents replication
|
||||
// IsValid returns true if ARN type represents replication or ilm
|
||||
func (t ServiceType) IsValid() bool {
|
||||
return t == ReplicationService
|
||||
return t == ReplicationService || t == ILMService
|
||||
}
|
||||
|
||||
// ARN is a struct to define arn.
|
||||
|
Loading…
Reference in New Issue
Block a user