From 28c26a9e597f9fb3d33f531c3b8007679bd60cec Mon Sep 17 00:00:00 2001 From: Krishna Srinivas Date: Mon, 22 May 2017 15:42:00 -0700 Subject: [PATCH] Generate random ETag if client does not provide MD5 for PutObjectPart (#4385) fixes #4289 fixes #4290 --- cmd/gateway-azure.go | 65 ++++++++++++++++++++++++++++++++++++----- cmd/gateway-handlers.go | 5 ++++ 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/cmd/gateway-azure.go b/cmd/gateway-azure.go index 2c5e87949..0c5be7b11 100644 --- a/cmd/gateway-azure.go +++ b/cmd/gateway-azure.go @@ -17,6 +17,7 @@ package cmd import ( + "crypto/md5" "encoding/base64" "encoding/hex" "fmt" @@ -313,24 +314,47 @@ func canonicalMetadata(metadata map[string]string) (canonical map[string]string) // uses Azure equivalent CreateBlockBlobFromReader. func (a *azureObjects) PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) { var sha256Writer hash.Hash + var md5sumWriter hash.Hash + + var writers []io.Writer + + md5sum := metadata["etag"] + delete(metadata, "etag") + teeReader := data + if sha256sum != "" { sha256Writer = sha256.New() - teeReader = io.TeeReader(data, sha256Writer) + writers = append(writers, sha256Writer) } - delete(metadata, "etag") + if md5sum != "" { + md5sumWriter = md5.New() + writers = append(writers, md5sumWriter) + } + + if len(writers) > 0 { + teeReader = io.TeeReader(data, io.MultiWriter(writers...)) + } err = a.client.CreateBlockBlobFromReader(bucket, object, uint64(size), teeReader, canonicalMetadata(metadata)) if err != nil { return objInfo, azureToObjectError(traceError(err), bucket, object) } + if md5sum != "" { + newMD5sum := hex.EncodeToString(md5sumWriter.Sum(nil)) + if newMD5sum != md5sum { + a.client.DeleteBlob(bucket, object, nil) + return ObjectInfo{}, azureToObjectError(traceError(BadDigest{md5sum, newMD5sum})) + } + } + if sha256sum != "" { newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil)) if newSHA256sum != sha256sum { a.client.DeleteBlob(bucket, object, nil) - return ObjectInfo{}, traceError(SHA256Mismatch{}) + return ObjectInfo{}, azureToObjectError(traceError(SHA256Mismatch{})) } } @@ -426,27 +450,54 @@ func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int return info, traceError(InvalidUploadID{}) } var sha256Writer hash.Hash + var md5sumWriter hash.Hash + var etag string + + var writers []io.Writer + if sha256sum != "" { sha256Writer = sha256.New() + writers = append(writers, sha256Writer) } - teeReader := io.TeeReader(data, sha256Writer) + if md5Hex != "" { + md5sumWriter = md5.New() + writers = append(writers, md5sumWriter) + etag = md5Hex + } else { + // Generate random ETag. + etag = getMD5Hash([]byte(mustGetUUID())) + } - id := azureGetBlockID(partID, md5Hex) + teeReader := data + + if len(writers) > 0 { + teeReader = io.TeeReader(data, io.MultiWriter(writers...)) + } + + id := azureGetBlockID(partID, etag) err = a.client.PutBlockWithLength(bucket, object, id, uint64(size), teeReader, nil) if err != nil { return info, azureToObjectError(traceError(err), bucket, object) } + if md5Hex != "" { + newMD5sum := hex.EncodeToString(md5sumWriter.Sum(nil)) + if newMD5sum != md5Hex { + a.client.DeleteBlob(bucket, object, nil) + return PartInfo{}, azureToObjectError(traceError(BadDigest{md5Hex, newMD5sum})) + } + } + if sha256sum != "" { newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil)) if newSHA256sum != sha256sum { - return PartInfo{}, traceError(SHA256Mismatch{}) + return PartInfo{}, azureToObjectError(traceError(SHA256Mismatch{})) } } info.PartNumber = partID - info.ETag = md5Hex + info.ETag = etag info.LastModified = UTCNow() info.Size = size return info, nil diff --git a/cmd/gateway-handlers.go b/cmd/gateway-handlers.go index 456b4446d..553de071b 100644 --- a/cmd/gateway-handlers.go +++ b/cmd/gateway-handlers.go @@ -282,6 +282,11 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re return } + if err != nil { + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + w.Header().Set("ETag", "\""+objInfo.ETag+"\"") writeSuccessResponseHeadersOnly(w) }