From 443c93c63472ff408c6ac02e5901b676d815601b Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 24 May 2024 12:28:51 -0700 Subject: [PATCH] compute time spent in ILM properly (#19806) --- buildscripts/verify-build.sh | 20 ++++++++++++-------- cmd/bucket-lifecycle.go | 3 ++- cmd/data-scanner.go | 25 ++++++++++--------------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/buildscripts/verify-build.sh b/buildscripts/verify-build.sh index c3e4f3f83..e15d64758 100755 --- a/buildscripts/verify-build.sh +++ b/buildscripts/verify-build.sh @@ -15,13 +15,14 @@ WORK_DIR="$PWD/.verify-$RANDOM" export MINT_MODE=core export MINT_DATA_DIR="$WORK_DIR/data" export SERVER_ENDPOINT="127.0.0.1:9000" +export MC_HOST_verify="http://minio:minio123@${SERVER_ENDPOINT}/" +export MC_HOST_verify_ipv6="http://minio:minio123@[::1]:9000/" export ACCESS_KEY="minio" export SECRET_KEY="minio123" export ENABLE_HTTPS=0 export GO111MODULE=on export GOGC=25 export ENABLE_ADMIN=1 - export MINIO_CI_CD=1 MINIO_CONFIG_DIR="$WORK_DIR/.minio" @@ -36,18 +37,21 @@ function start_minio_fs() { export MINIO_ROOT_USER=$ACCESS_KEY export MINIO_ROOT_PASSWORD=$SECRET_KEY "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 & - sleep 10 + + "${WORK_DIR}/mc" ready verify } function start_minio_erasure() { "${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 & - sleep 15 + + "${WORK_DIR}/mc" ready verify } function start_minio_erasure_sets() { export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}" "${MINIO[@]}" server >"$WORK_DIR/erasure-minio-sets.log" 2>&1 & - sleep 15 + + "${WORK_DIR}/mc" ready verify } function start_minio_pool_erasure_sets() { @@ -57,7 +61,7 @@ function start_minio_pool_erasure_sets() { "${MINIO[@]}" server --address ":9000" >"$WORK_DIR/pool-minio-9000.log" 2>&1 & "${MINIO[@]}" server --address ":9001" >"$WORK_DIR/pool-minio-9001.log" 2>&1 & - sleep 40 + "${WORK_DIR}/mc" ready verify } function start_minio_pool_erasure_sets_ipv6() { @@ -67,7 +71,7 @@ function start_minio_pool_erasure_sets_ipv6() { "${MINIO[@]}" server --address="[::1]:9000" >"$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 & "${MINIO[@]}" server --address="[::1]:9001" >"$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 & - sleep 40 + "${WORK_DIR}/mc" ready verify_ipv6 } function start_minio_dist_erasure() { @@ -78,7 +82,7 @@ function start_minio_dist_erasure() { "${MINIO[@]}" server --address ":900${i}" >"$WORK_DIR/dist-minio-900${i}.log" 2>&1 & done - sleep 40 + "${WORK_DIR}/mc" ready verify } function run_test_fs() { @@ -222,7 +226,7 @@ function __init__() { exit 1 fi - (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + (cd "${MC_BUILD_DIR}" && go build -o "${WORK_DIR}/mc") # remove mc source. purge "${MC_BUILD_DIR}" diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index 2984d23f5..c95ed7d72 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -663,11 +663,12 @@ func genTransitionObjName(bucket string) (string, error) { // is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved // to the transition tier without decrypting or re-encrypting. func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, lae lcAuditEvent) (err error) { + timeILM := globalScannerMetrics.timeILM(lae.Action) defer func() { if err != nil { return } - globalScannerMetrics.timeILM(lae.Action)(1) + timeILM(1) }() opts := ObjectOptions{ diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 5b3c75d25..9ab1bf64f 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -1240,29 +1240,22 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) return true } -func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) bool { - var err error - defer func() { - if err != nil { - return - } - // Note: DeleteAllVersions action is not supported for - // transitioned objects - globalScannerMetrics.timeILM(lcEvent.Action)(1) - }() - - if err = expireTransitionedObject(ctx, objLayer, &obj, lcEvent, src); err != nil { +func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) (ok bool) { + timeILM := globalScannerMetrics.timeILM(lcEvent.Action) + if err := expireTransitionedObject(ctx, objLayer, &obj, lcEvent, src); err != nil { if isErrObjectNotFound(err) || isErrVersionNotFound(err) { return false } ilmLogIf(ctx, err) return false } + timeILM(1) + // Notification already sent in *expireTransitionedObject*, just return 'true' here. return true } -func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) bool { +func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) (ok bool) { traceFn := globalLifecycleSys.trace(obj) opts := ObjectOptions{ Expiration: ExpirationOptions{Expire: true}, @@ -1284,8 +1277,10 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay dobj ObjectInfo err error ) + + timeILM := globalScannerMetrics.timeILM(lcEvent.Action) defer func() { - if err != nil { + if !ok { return } @@ -1294,7 +1289,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay if lcEvent.Action.DeleteAll() { numVersions = uint64(obj.NumVersions) } - globalScannerMetrics.timeILM(lcEvent.Action)(numVersions) + timeILM(numVersions) } }()