From 35f2552fc58ca05addbffea5f97582265ff4c86a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 27 Aug 2021 17:06:47 -0700 Subject: [PATCH] reduce extra getObjectInfo() calls during ILM transition (#13091) * reduce extra getObjectInfo() calls during ILM transition This PR also changes expiration logic to be non-blocking, scanner is now free from additional costs incurred due to slower object layer calls and hitting the drives. * move verifying expiration inside locks --- cmd/bucket-lifecycle.go | 16 ++++++--- cmd/data-scanner.go | 60 +++++++-------------------------- cmd/erasure-object.go | 66 +++++++++++++++++++++++++++++++------ cmd/object-api-interface.go | 6 ++++ cmd/object-handlers.go | 24 +++++++++++--- 5 files changed, 105 insertions(+), 67 deletions(-) diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index 8a5d84af1..46d5b6947 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -75,8 +75,9 @@ func NewLifecycleSys() *LifecycleSys { } type expiryTask struct { - objInfo ObjectInfo - versionExpiry bool + objInfo ObjectInfo + versionExpiry bool + restoredObject bool } type expiryState struct { @@ -89,13 +90,13 @@ func (es *expiryState) PendingTasks() int { return len(es.expiryCh) } -func (es *expiryState) queueExpiryTask(oi ObjectInfo, rmVersion bool) { +func (es *expiryState) queueExpiryTask(oi ObjectInfo, restoredObject bool, rmVersion bool) { select { case <-GlobalContext.Done(): es.once.Do(func() { close(es.expiryCh) }) - case es.expiryCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion}: + case es.expiryCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion, restoredObject: restoredObject}: default: } } @@ -114,7 +115,11 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) { globalExpiryState = newExpiryState() go func() { for t := range globalExpiryState.expiryCh { - applyExpiryRule(ctx, objectAPI, t.objInfo, false, t.versionExpiry) + if t.objInfo.TransitionedObject.Status != "" { + applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject) + } else { + applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry) + } } }() } @@ -247,6 +252,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob var opts ObjectOptions opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket) opts.VersionID = lcOpts.VersionID + opts.Expiration = ExpirationOptions{Expire: true} switch action { case expireObj: // When an object is past expiry or when a transitioned object is being diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index d9b140251..909625fe5 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -919,50 +919,14 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje } } switch action { - case lifecycle.DeleteAction, lifecycle.DeleteVersionAction: + case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction: + return applyLifecycleAction(action, oi), 0 case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: - case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction: + return applyLifecycleAction(action, oi), size default: // No action. return false, size } - - obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{ - VersionID: versionID, - }) - if err != nil { - switch err.(type) { - case MethodNotAllowed: // This happens usually for a delete marker - if !obj.DeleteMarker { // if this is not a delete marker log and return - // Do nothing - heal in the future. - logger.LogIf(ctx, err) - return false, size - } - case ObjectNotFound, VersionNotFound: - // object not found or version not found return 0 - return false, 0 - default: - // All other errors proceed. - logger.LogIf(ctx, err) - return false, size - } - } - - action = evalActionFromLifecycle(ctx, *i.lifeCycle, obj, i.debug) - if action != lifecycle.NoneAction { - applied = applyLifecycleAction(ctx, action, o, obj) - } - - if applied { - switch action { - case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: - return true, size - } - // For all other lifecycle actions that remove data - return true, 0 - } - - return false, size } // applyTierObjSweep removes remote object pending deletion and the free-version @@ -1081,7 +1045,9 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, } func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool { - opts := ObjectOptions{} + opts := ObjectOptions{ + Expiration: ExpirationOptions{Expire: true}, + } if applyOnVersion { opts.VersionID = obj.VersionID @@ -1120,20 +1086,18 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay } // Apply object, object version, restored object or restored object version action on the given object -func applyExpiryRule(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject, applyOnVersion bool) bool { - if obj.TransitionedObject.Status != "" { - return applyExpiryOnTransitionedObject(ctx, objLayer, obj, restoredObject) - } - return applyExpiryOnNonTransitionedObjects(ctx, objLayer, obj, applyOnVersion) +func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool { + globalExpiryState.queueExpiryTask(obj, restoredObject, applyOnVersion) + return true } // Perform actions (removal or transitioning of objects), return true the action is successfully performed -func applyLifecycleAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) (success bool) { +func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo) (success bool) { switch action { case lifecycle.DeleteVersionAction, lifecycle.DeleteAction: - success = applyExpiryRule(ctx, objLayer, obj, false, action == lifecycle.DeleteVersionAction) + success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction) case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction: - success = applyExpiryRule(ctx, objLayer, obj, true, action == lifecycle.DeleteRestoredVersionAction) + success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction) case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: success = applyTransitionRule(obj) } diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 0b4fec875..a76163e7a 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1191,9 +1191,40 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object) } + var lc *lifecycle.Lifecycle + if opts.Expiration.Expire { + // Check if the current bucket has a configured lifecycle policy + lc, _ = globalLifecycleSys.Get(bucket) + } + + // expiration attempted on a bucket with no lifecycle + // rules shall be rejected. + if lc == nil && opts.Expiration.Expire { + if opts.VersionID != "" { + return objInfo, VersionNotFound{ + Bucket: bucket, + Object: object, + VersionID: opts.VersionID, + } + } + return objInfo, ObjectNotFound{ + Bucket: bucket, + Object: object, + } + } + + // Acquire a write lock before deleting the object. + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx.Cancel) + versionFound := true objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response. - goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts) + goi, gerr := er.getObjectInfo(ctx, bucket, object, opts) if gerr != nil && goi.Name == "" { switch gerr.(type) { case InsufficientReadQuorum: @@ -1207,16 +1238,31 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string } } - defer NSUpdated(bucket, object) - - // Acquire a write lock before deleting the object. - lk := er.NewNSLock(bucket, object) - lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) - if err != nil { - return ObjectInfo{}, err + if opts.Expiration.Expire { + action := evalActionFromLifecycle(ctx, *lc, goi, false) + var isErr bool + switch action { + case lifecycle.NoneAction: + isErr = true + case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: + isErr = true + } + if isErr { + if goi.VersionID != "" { + return goi, VersionNotFound{ + Bucket: bucket, + Object: object, + VersionID: goi.VersionID, + } + } + return goi, ObjectNotFound{ + Bucket: bucket, + Object: object, + } + } } - ctx = lkctx.Context() - defer lk.Unlock(lkctx.Cancel) + + defer NSUpdated(bucket, object) storageDisks := er.getDisks() writeQuorum := len(storageDisks)/2 + 1 diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 6d9368471..237987d7f 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -52,6 +52,7 @@ type ObjectOptions struct { DeleteMarkerReplicationStatus string // Is only set in DELETE operations VersionPurgeStatus VersionPurgeStatusType // Is only set in DELETE operations for delete marker version to be permanently deleted. Transition TransitionOptions + Expiration ExpirationOptions NoLock bool // indicates to lower layers if the caller is expecting to hold locks. ProxyRequest bool // only set for GET/HEAD in active-active replication scenario @@ -63,6 +64,11 @@ type ObjectOptions struct { MaxParity bool } +// ExpirationOptions represents object options for object expiration at objectLayer. +type ExpirationOptions struct { + Expire bool +} + // TransitionOptions represents object options for transition ObjectLayer operation type TransitionOptions struct { Status string diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 122f0b911..df81a2632 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -450,8 +450,16 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj // Automatically remove the object/version is an expiry lifecycle rule can be applied if lc, err := globalLifecycleSys.Get(bucket); err == nil { action := evalActionFromLifecycle(ctx, *lc, objInfo, false) - if action == lifecycle.DeleteAction || action == lifecycle.DeleteVersionAction { - globalExpiryState.queueExpiryTask(objInfo, action == lifecycle.DeleteVersionAction) + var success bool + switch action { + case lifecycle.DeleteVersionAction, lifecycle.DeleteAction: + success = applyExpiryRule(objInfo, false, action == lifecycle.DeleteVersionAction) + case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction: + // Restored object delete would be still allowed to proceed as success + // since transition behavior is slightly different. + applyExpiryRule(objInfo, true, action == lifecycle.DeleteRestoredVersionAction) + } + if success { writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey)) return } @@ -656,8 +664,16 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob // Automatically remove the object/version is an expiry lifecycle rule can be applied if lc, err := globalLifecycleSys.Get(bucket); err == nil { action := evalActionFromLifecycle(ctx, *lc, objInfo, false) - if action == lifecycle.DeleteAction || action == lifecycle.DeleteVersionAction { - globalExpiryState.queueExpiryTask(objInfo, action == lifecycle.DeleteVersionAction) + var success bool + switch action { + case lifecycle.DeleteVersionAction, lifecycle.DeleteAction: + success = applyExpiryRule(objInfo, false, action == lifecycle.DeleteVersionAction) + case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction: + // Restored object delete would be still allowed to proceed as success + // since transition behavior is slightly different. + applyExpiryRule(objInfo, true, action == lifecycle.DeleteRestoredVersionAction) + } + if success { writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey)) return }