From e3eb5c1328fc90e07861528ccad7064aaf8ff862 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 4 Aug 2024 21:55:25 -0700 Subject: [PATCH] batch-exp: Remove 1000 maximum objects per call (#20212) It seems ObjectAPI.DeleteObjects() is clogging up when it is removing 10k versions of a single object. Authored-by: Anis Eleuch --- cmd/batch-expire.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/batch-expire.go b/cmd/batch-expire.go index 3a39f473c..7c7d1e1fb 100644 --- a/cmd/batch-expire.go +++ b/cmd/batch-expire.go @@ -342,8 +342,24 @@ func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versio PrefixEnabledFn: vc.PrefixEnabled, VersionSuspended: vc.Suspended(), } - _, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel, opts) - return errs + + allErrs := make([]error, 0, len(objsToDel)) + + for { + count := len(objsToDel) + if count == 0 { + break + } + if count > maxDeleteList { + count = maxDeleteList + } + _, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts) + allErrs = append(allErrs, errs...) + // Next batch of deletion + objsToDel = objsToDel[count:] + } + + return allErrs } const (