From cd6dec49c0f8b99d40981a4e95eb28d0371e4d29 Mon Sep 17 00:00:00 2001 From: Poorna Date: Tue, 11 Apr 2023 19:22:32 -0700 Subject: [PATCH] Add trace support for ilm activity (#16993) --- cmd/bucket-lifecycle.go | 28 +++++++++++++++++++++++++++- cmd/data-scanner.go | 9 ++++++--- cmd/erasure-object.go | 3 ++- cmd/object-handlers-common.go | 14 ++++++++------ 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index 2f650b5a0..21358a574 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -32,6 +32,7 @@ import ( "time" "github.com/google/uuid" + "github.com/minio/madmin-go/v2" "github.com/minio/minio-go/v7/pkg/tags" "github.com/minio/minio/internal/amztime" sse "github.com/minio/minio/internal/bucket/encryption" @@ -70,6 +71,30 @@ func NewLifecycleSys() *LifecycleSys { return &LifecycleSys{} } +func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string) madmin.TraceInfo { + return madmin.TraceInfo{ + TraceType: madmin.TraceILM, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: event, + Duration: duration, + Path: pathJoin(oi.Bucket, oi.Name), + Error: "", + Message: getSource(4), + Custom: map[string]string{"version-id": oi.VersionID}, + } +} + +func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string) { + startTime := time.Now() + return func(event string) { + duration := time.Since(startTime) + if globalTrace.NumSubscribers(madmin.TraceILM) > 0 { + globalTrace.Publish(ilmTrace(startTime, duration, oi, event)) + } + } +} + type expiryTask struct { objInfo ObjectInfo versionExpiry bool @@ -360,6 +385,7 @@ const ( // 1. when a restored (via PostRestoreObject API) object expires. // 2. when a transitioned object expires (based on an ILM rule). func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error { + traceFn := globalLifecycleSys.trace(*oi) var opts ObjectOptions opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name) opts.VersionID = lcOpts.VersionID @@ -385,7 +411,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob } // Send audit for the lifecycle delete operation - auditLogLifecycle(ctx, *oi, ILMExpiry) + defer auditLogLifecycle(ctx, *oi, ILMExpiry, traceFn) eventName := event.ObjectRemovedDelete if lcOpts.DeleteMarker { diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index d68116590..2a0a0c005 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -957,6 +957,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje // applyTierObjSweep removes remote object pending deletion and the free-version // tracking this information. func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi ObjectInfo) { + traceFn := globalLifecycleSys.trace(oi) if !oi.TransitionedObject.FreeVersion { // nothing to be done return @@ -982,7 +983,7 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O InclFreeVersions: true, }) if err == nil { - auditLogLifecycle(ctx, oi, ILMFreeVersionDelete) + auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, traceFn) } if ignoreNotFoundErr(err) != nil { logger.LogIf(ctx, err) @@ -1177,6 +1178,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, } func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool { + traceFn := globalLifecycleSys.trace(obj) opts := ObjectOptions{ Expiration: ExpirationOptions{Expire: true}, } @@ -1200,7 +1202,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay } // Send audit for the lifecycle delete operation - auditLogLifecycle(ctx, obj, ILMExpiry) + auditLogLifecycle(ctx, obj, ILMExpiry, traceFn) eventName := event.ObjectRemovedDelete if obj.DeleteMarker { @@ -1443,7 +1445,7 @@ const ( ILMTransition = " ilm:transition" ) -func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) { +func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, traceFn func(event string)) { var apiName string switch event { case ILMExpiry: @@ -1460,4 +1462,5 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) { Object: oi.Name, VersionID: oi.VersionID, }) + traceFn(event) } diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 039378d19..ebe8b5851 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1953,6 +1953,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st return toObjectErr(err, bucket, object) } } + traceFn := globalLifecycleSys.trace(fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) destObj, err := genTransitionObjName(bucket) if err != nil { @@ -2000,7 +2001,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st UserAgent: "Internal: [ILM-Transition]", Host: globalLocalNodeName, }) - auditLogLifecycle(ctx, objInfo, ILMTransition) + auditLogLifecycle(ctx, objInfo, ILMTransition, traceFn) return err } diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go index 41805f906..46c1081e8 100644 --- a/cmd/object-handlers-common.go +++ b/cmd/object-handlers-common.go @@ -367,15 +367,17 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD continue } dobj := deletedObjs[i] + oi := ObjectInfo{ + Bucket: bucket, + Name: dobj.ObjectName, + VersionID: dobj.VersionID, + } + traceFn := globalLifecycleSys.trace(oi) // Send audit for the lifecycle delete operation auditLogLifecycle( ctx, - ObjectInfo{ - Bucket: bucket, - Name: dobj.ObjectName, - VersionID: dobj.VersionID, - }, - ILMExpiry) + oi, + ILMExpiry, traceFn) sendEvent(eventArgs{ EventName: event.ObjectRemovedDelete,