reduce extra getObjectInfo() calls during ILM transition (#13091)

* reduce extra getObjectInfo() calls during ILM transition

This PR also changes expiration logic to be non-blocking,
scanner is now free from additional costs incurred due
to slower object layer calls and hitting the drives.

* move verifying expiration inside locks
This commit is contained in:
Harshavardhana 2021-08-27 17:06:47 -07:00 committed by GitHub
parent e05886561d
commit 35f2552fc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 105 additions and 67 deletions

View File

@ -77,6 +77,7 @@ func NewLifecycleSys() *LifecycleSys {
type expiryTask struct { type expiryTask struct {
objInfo ObjectInfo objInfo ObjectInfo
versionExpiry bool versionExpiry bool
restoredObject bool
} }
type expiryState struct { type expiryState struct {
@ -89,13 +90,13 @@ func (es *expiryState) PendingTasks() int {
return len(es.expiryCh) return len(es.expiryCh)
} }
func (es *expiryState) queueExpiryTask(oi ObjectInfo, rmVersion bool) { func (es *expiryState) queueExpiryTask(oi ObjectInfo, restoredObject bool, rmVersion bool) {
select { select {
case <-GlobalContext.Done(): case <-GlobalContext.Done():
es.once.Do(func() { es.once.Do(func() {
close(es.expiryCh) close(es.expiryCh)
}) })
case es.expiryCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion}: case es.expiryCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion, restoredObject: restoredObject}:
default: default:
} }
} }
@ -114,7 +115,11 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
globalExpiryState = newExpiryState() globalExpiryState = newExpiryState()
go func() { go func() {
for t := range globalExpiryState.expiryCh { for t := range globalExpiryState.expiryCh {
applyExpiryRule(ctx, objectAPI, t.objInfo, false, t.versionExpiry) if t.objInfo.TransitionedObject.Status != "" {
applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
} else {
applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
}
} }
}() }()
} }
@ -247,6 +252,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
var opts ObjectOptions var opts ObjectOptions
opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket) opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
opts.VersionID = lcOpts.VersionID opts.VersionID = lcOpts.VersionID
opts.Expiration = ExpirationOptions{Expire: true}
switch action { switch action {
case expireObj: case expireObj:
// When an object is past expiry or when a transitioned object is being // When an object is past expiry or when a transitioned object is being

View File

@ -919,50 +919,14 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
} }
} }
switch action { switch action {
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction: case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
return applyLifecycleAction(action, oi), 0
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction: return applyLifecycleAction(action, oi), size
default: default:
// No action. // No action.
return false, size return false, size
} }
obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{
VersionID: versionID,
})
if err != nil {
switch err.(type) {
case MethodNotAllowed: // This happens usually for a delete marker
if !obj.DeleteMarker { // if this is not a delete marker log and return
// Do nothing - heal in the future.
logger.LogIf(ctx, err)
return false, size
}
case ObjectNotFound, VersionNotFound:
// object not found or version not found return 0
return false, 0
default:
// All other errors proceed.
logger.LogIf(ctx, err)
return false, size
}
}
action = evalActionFromLifecycle(ctx, *i.lifeCycle, obj, i.debug)
if action != lifecycle.NoneAction {
applied = applyLifecycleAction(ctx, action, o, obj)
}
if applied {
switch action {
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
return true, size
}
// For all other lifecycle actions that remove data
return true, 0
}
return false, size
} }
// applyTierObjSweep removes remote object pending deletion and the free-version // applyTierObjSweep removes remote object pending deletion and the free-version
@ -1081,7 +1045,9 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
} }
func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool { func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
opts := ObjectOptions{} opts := ObjectOptions{
Expiration: ExpirationOptions{Expire: true},
}
if applyOnVersion { if applyOnVersion {
opts.VersionID = obj.VersionID opts.VersionID = obj.VersionID
@ -1120,20 +1086,18 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
} }
// Apply object, object version, restored object or restored object version action on the given object // Apply object, object version, restored object or restored object version action on the given object
func applyExpiryRule(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject, applyOnVersion bool) bool { func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
if obj.TransitionedObject.Status != "" { globalExpiryState.queueExpiryTask(obj, restoredObject, applyOnVersion)
return applyExpiryOnTransitionedObject(ctx, objLayer, obj, restoredObject) return true
}
return applyExpiryOnNonTransitionedObjects(ctx, objLayer, obj, applyOnVersion)
} }
// Perform actions (removal or transitioning of objects), return true the action is successfully performed // Perform actions (removal or transitioning of objects), return true the action is successfully performed
func applyLifecycleAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) (success bool) { func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo) (success bool) {
switch action { switch action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction: case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
success = applyExpiryRule(ctx, objLayer, obj, false, action == lifecycle.DeleteVersionAction) success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction: case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
success = applyExpiryRule(ctx, objLayer, obj, true, action == lifecycle.DeleteRestoredVersionAction) success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
success = applyTransitionRule(obj) success = applyTransitionRule(obj)
} }

View File

@ -1191,9 +1191,40 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object) return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
} }
var lc *lifecycle.Lifecycle
if opts.Expiration.Expire {
// Check if the current bucket has a configured lifecycle policy
lc, _ = globalLifecycleSys.Get(bucket)
}
// expiration attempted on a bucket with no lifecycle
// rules shall be rejected.
if lc == nil && opts.Expiration.Expire {
if opts.VersionID != "" {
return objInfo, VersionNotFound{
Bucket: bucket,
Object: object,
VersionID: opts.VersionID,
}
}
return objInfo, ObjectNotFound{
Bucket: bucket,
Object: object,
}
}
// Acquire a write lock before deleting the object.
lk := er.NewNSLock(bucket, object)
lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
if err != nil {
return ObjectInfo{}, err
}
ctx = lkctx.Context()
defer lk.Unlock(lkctx.Cancel)
versionFound := true versionFound := true
objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response. objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts) goi, gerr := er.getObjectInfo(ctx, bucket, object, opts)
if gerr != nil && goi.Name == "" { if gerr != nil && goi.Name == "" {
switch gerr.(type) { switch gerr.(type) {
case InsufficientReadQuorum: case InsufficientReadQuorum:
@ -1207,16 +1238,31 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
} }
} }
defer NSUpdated(bucket, object) if opts.Expiration.Expire {
action := evalActionFromLifecycle(ctx, *lc, goi, false)
// Acquire a write lock before deleting the object. var isErr bool
lk := er.NewNSLock(bucket, object) switch action {
lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) case lifecycle.NoneAction:
if err != nil { isErr = true
return ObjectInfo{}, err case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
isErr = true
} }
ctx = lkctx.Context() if isErr {
defer lk.Unlock(lkctx.Cancel) if goi.VersionID != "" {
return goi, VersionNotFound{
Bucket: bucket,
Object: object,
VersionID: goi.VersionID,
}
}
return goi, ObjectNotFound{
Bucket: bucket,
Object: object,
}
}
}
defer NSUpdated(bucket, object)
storageDisks := er.getDisks() storageDisks := er.getDisks()
writeQuorum := len(storageDisks)/2 + 1 writeQuorum := len(storageDisks)/2 + 1

View File

@ -52,6 +52,7 @@ type ObjectOptions struct {
DeleteMarkerReplicationStatus string // Is only set in DELETE operations DeleteMarkerReplicationStatus string // Is only set in DELETE operations
VersionPurgeStatus VersionPurgeStatusType // Is only set in DELETE operations for delete marker version to be permanently deleted. VersionPurgeStatus VersionPurgeStatusType // Is only set in DELETE operations for delete marker version to be permanently deleted.
Transition TransitionOptions Transition TransitionOptions
Expiration ExpirationOptions
NoLock bool // indicates to lower layers if the caller is expecting to hold locks. NoLock bool // indicates to lower layers if the caller is expecting to hold locks.
ProxyRequest bool // only set for GET/HEAD in active-active replication scenario ProxyRequest bool // only set for GET/HEAD in active-active replication scenario
@ -63,6 +64,11 @@ type ObjectOptions struct {
MaxParity bool MaxParity bool
} }
// ExpirationOptions represents object options for object expiration at objectLayer.
type ExpirationOptions struct {
Expire bool
}
// TransitionOptions represents object options for transition ObjectLayer operation // TransitionOptions represents object options for transition ObjectLayer operation
type TransitionOptions struct { type TransitionOptions struct {
Status string Status string

View File

@ -450,8 +450,16 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
// Automatically remove the object/version is an expiry lifecycle rule can be applied // Automatically remove the object/version is an expiry lifecycle rule can be applied
if lc, err := globalLifecycleSys.Get(bucket); err == nil { if lc, err := globalLifecycleSys.Get(bucket); err == nil {
action := evalActionFromLifecycle(ctx, *lc, objInfo, false) action := evalActionFromLifecycle(ctx, *lc, objInfo, false)
if action == lifecycle.DeleteAction || action == lifecycle.DeleteVersionAction { var success bool
globalExpiryState.queueExpiryTask(objInfo, action == lifecycle.DeleteVersionAction) switch action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
success = applyExpiryRule(objInfo, false, action == lifecycle.DeleteVersionAction)
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
// Restored object delete would be still allowed to proceed as success
// since transition behavior is slightly different.
applyExpiryRule(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
}
if success {
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey)) writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
return return
} }
@ -656,8 +664,16 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
// Automatically remove the object/version is an expiry lifecycle rule can be applied // Automatically remove the object/version is an expiry lifecycle rule can be applied
if lc, err := globalLifecycleSys.Get(bucket); err == nil { if lc, err := globalLifecycleSys.Get(bucket); err == nil {
action := evalActionFromLifecycle(ctx, *lc, objInfo, false) action := evalActionFromLifecycle(ctx, *lc, objInfo, false)
if action == lifecycle.DeleteAction || action == lifecycle.DeleteVersionAction { var success bool
globalExpiryState.queueExpiryTask(objInfo, action == lifecycle.DeleteVersionAction) switch action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
success = applyExpiryRule(objInfo, false, action == lifecycle.DeleteVersionAction)
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
// Restored object delete would be still allowed to proceed as success
// since transition behavior is slightly different.
applyExpiryRule(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
}
if success {
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey)) writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
return return
} }