MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

allow multipart uploads for single part multipart (#12821)

Browse Source 
its possible that some multipart uploads would have
uploaded only single parts so relying on `len(o.Parts)`
alone is not sufficient, we need to look for ETag
pattern to be absolutely sure.
This commit is contained in:
Harshavardhana
2021-07-28 22:11:55 -07:00
committed by GitHub
parent b6cd54779c
commit 3c34e18a4e
5 changed files with 68 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
cmd/bucket-replication.go
View File

@@ -801,14 +801,11 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
defer cancel()
}
r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
if len(objInfo.Parts) > 1 {
if uploadID, err := replicateObjectWithMultipart(ctx, c, dest.Bucket, object, r, objInfo, putOpts); err != nil {
if objInfo.Multipart {
if err := replicateObjectWithMultipart(ctx, c, dest.Bucket, object,
r, objInfo, putOpts); err != nil {
replicationStatus = replication.Failed
logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
// block and abort remote upload upon failure.
if err = c.AbortMultipartUpload(ctx, dest.Bucket, object, uploadID); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to clean up failed upload %s on remote %s/%s: %w", uploadID, dest.Bucket, object, err))
}
}
} else {
if _, err = c.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts); err != nil {
@@ -876,38 +873,52 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
}
}
func replicateObjectWithMultipart(ctx context.Context, c *miniogo.Core, bucket, object string, r io.Reader, objInfo ObjectInfo, opts miniogo.PutObjectOptions) (uploadID string, err error) {
func replicateObjectWithMultipart(ctx context.Context, c *miniogo.Core, bucket, object string, r io.Reader, objInfo ObjectInfo, opts miniogo.PutObjectOptions) (err error) {
var uploadedParts []miniogo.CompletePart
uploadID, err = c.NewMultipartUpload(context.Background(), bucket, object, opts)
uploadID, err := c.NewMultipartUpload(context.Background(), bucket, object, opts)
if err != nil {
return
return err
}
defer func() {
if err != nil {
// block and abort remote upload upon failure.
if aerr := c.AbortMultipartUpload(ctx, bucket, object, uploadID); aerr != nil {
aerr = fmt.Errorf("Unable to cleanup failed multipart replication %s on remote %s/%s: %w", uploadID, bucket, object, aerr)
logger.LogIf(ctx, aerr)
}
}
}()
var (
hr *hash.Reader
pInfo miniogo.ObjectPart
)
for _, partInfo := range objInfo.Parts {
hr, err = hash.NewReader(r, partInfo.Size, "", "", partInfo.Size)
if err != nil {
return
return err
}
pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.Size, "", "", opts.ServerSideEncryption)
if err != nil {
return
return err
}
if pInfo.Size != partInfo.Size {
return uploadID, fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.Size)
return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.Size)
}
uploadedParts = append(uploadedParts, miniogo.CompletePart{
PartNumber: pInfo.PartNumber,
ETag: pInfo.ETag,
})
}
_, err = c.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{
SourceMTime: objInfo.ModTime,
ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
}})
return
_, err = c.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, miniogo.PutObjectOptions{
Internal: miniogo.AdvancedPutOptions{
SourceMTime: objInfo.ModTime,
// always set this to distinguish between `mc mirror` replication and serverside
ReplicationRequest: true,
}})
return err
}
// filterReplicationStatusMetadata filters replication status metadata for COPY