replication: attempt abort multipart-upload at max 3 times on remote (#14087)

this is mainly an attempt to relinquish space on the remote
site, if this still doesn't do it we give and let the admin
know with a log message.
This commit is contained in:
Harshavardhana 2022-01-11 22:32:29 -08:00 committed by GitHub
parent d50442da01
commit cc3f139d1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 12 additions and 3 deletions

View File

@ -28,6 +28,7 @@ import (
"sync" "sync"
"time" "time"
"github.com/dustin/go-humanize"
"github.com/minio/madmin-go" "github.com/minio/madmin-go"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
miniogo "github.com/minio/minio-go/v7" miniogo "github.com/minio/minio-go/v7"
@ -1184,9 +1185,17 @@ func replicateObjectWithMultipart(ctx context.Context, c *miniogo.Core, bucket,
defer func() { defer func() {
if err != nil { if err != nil {
// block and abort remote upload upon failure. // block and abort remote upload upon failure.
if aerr := c.AbortMultipartUpload(ctx, bucket, object, uploadID); aerr != nil { attempts := 1
aerr = fmt.Errorf("Unable to cleanup failed multipart replication %s on remote %s/%s: %w", uploadID, bucket, object, aerr) for attempts <= 3 {
logger.LogIf(ctx, aerr) aerr := c.AbortMultipartUpload(ctx, bucket, object, uploadID)
if aerr == nil {
return
}
logger.LogIf(ctx,
fmt.Errorf("Trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
humanize.Ordinal(attempts), uploadID, bucket, object, aerr))
attempts++
time.Sleep(time.Second)
} }
} }
}() }()