From 80e3dce6318a68f7f3a8540605b085178e0c499f Mon Sep 17 00:00:00 2001 From: poornas Date: Tue, 15 Sep 2020 16:31:47 -0700 Subject: [PATCH] azure: update content-md5 to metadata after upload (#10482) Fixes #10453 --- cmd/gateway/azure/gateway-azure.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index a12008443..4bf96bf03 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -70,7 +70,6 @@ const ( azureChunkSizeEnvVar = "MINIO_AZURE_CHUNK_SIZE_MB" azureDownloadRetryAttempts = 5 - azureBlockSize = 100 * humanize.MiByte azureS3MinPartSize = 5 * humanize.MiByte metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json" azureMarkerPrefix = "{minio}" @@ -880,20 +879,15 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string, // uses Azure equivalent `UploadStreamToBlockBlob`. func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { data := r.Reader - - if data.Size() > azureBlockSize/2 { - if opts.UserDefined == nil { - opts.UserDefined = map[string]string{} - } - - // Save md5sum for future processing on the object. - opts.UserDefined["x-amz-meta-md5sum"] = r.MD5CurrentHexString() + if opts.UserDefined == nil { + opts.UserDefined = map[string]string{} } metadata, properties, err := s3MetaToAzureProperties(ctx, opts.UserDefined) if err != nil { return objInfo, azureToObjectError(err, bucket, object) } + blobURL := a.client.NewContainerURL(bucket).NewBlockBlobURL(object) _, err = azblob.UploadStreamToBlockBlob(ctx, data, blobURL, azblob.UploadStreamToBlockBlobOptions{ @@ -905,7 +899,18 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r * if err != nil { return objInfo, azureToObjectError(err, bucket, object) } - + // Query the blob's properties and metadata + get, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{}) + if err != nil { + return objInfo, azureToObjectError(err, bucket, object) + } + // Update the blob's metadata with Content-MD5 after the upload + metadata = get.NewMetadata() + metadata["md5sum"] = r.MD5CurrentHexString() + _, err = blobURL.SetMetadata(ctx, metadata, azblob.BlobAccessConditions{}) + if err != nil { + return objInfo, azureToObjectError(err, bucket, object) + } return a.GetObjectInfo(ctx, bucket, object, opts) } @@ -929,6 +934,7 @@ func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, des return objInfo, azureToObjectError(err, srcBucket, srcObject) } props.ContentMD5 = srcProps.ContentMD5() + azureMeta["md5sum"] = srcInfo.ETag res, err := destBlob.StartCopyFromURL(ctx, srcBlobURL, azureMeta, azblob.ModifiedAccessConditions{}, azblob.BlobAccessConditions{}) if err != nil { return objInfo, azureToObjectError(err, srcBucket, srcObject)