Move to latest release of minio-go (#4886)

- Region handling can now use region endpoints directly.
- All uploads are streaming no more large buffer needed.
- Major API overhaul for CopyObject(dst, src)
- Fixes bugs present in existing code for copying
  - metadata replace directive CopyObject
  - PutObjectPart doesn't require md5Sum and sha256

cmd/gateway-s3.go

@@ -19,7 +19,7 @@ package cmd
 import (
 	"io"
 	"net/http"
-	"path"
+	"strings"
 
 	"encoding/hex"
 
@@ -330,29 +330,32 @@ func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectI
 	return fromMinioClientObjectInfo(bucket, oi), nil
 }
 
+// Decodes hex encoded md5, sha256 into their raw byte representations.
+func getMD5AndSha256SumBytes(md5Hex, sha256Hex string) (md5Bytes, sha256Bytes []byte, err error) {
+	if md5Hex != "" {
+		md5Bytes, err = hex.DecodeString(md5Hex)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	if sha256Hex != "" {
+		sha256Bytes, err = hex.DecodeString(sha256Hex)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	return md5Bytes, sha256Bytes, nil
+}
+
 // PutObject creates a new object with the incoming data,
 func (l *s3Objects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, e error) {
-	var sha256sumBytes []byte
-
-	var err error
-	if sha256sum != "" {
-		sha256sumBytes, err = hex.DecodeString(sha256sum)
-		if err != nil {
-			return objInfo, s3ToObjectError(traceError(err), bucket, object)
-		}
+	md5Bytes, sha256Bytes, err := getMD5AndSha256SumBytes(metadata["etag"], sha256sum)
+	if err != nil {
+		return objInfo, s3ToObjectError(traceError(err), bucket, object)
 	}
+	delete(metadata, "etag")
 
-	var md5sumBytes []byte
-	md5sum := metadata["etag"]
-	if md5sum != "" {
-		md5sumBytes, err = hex.DecodeString(md5sum)
-		if err != nil {
-			return objInfo, s3ToObjectError(traceError(err), bucket, object)
-		}
-		delete(metadata, "etag")
-	}
-
-	oi, err := l.Client.PutObject(bucket, object, size, data, md5sumBytes, sha256sumBytes, toMinioClientMetadata(metadata))
+	oi, err := l.Client.PutObject(bucket, object, size, data, md5Bytes, sha256Bytes, toMinioClientMetadata(metadata))
 	if err != nil {
 		return objInfo, s3ToObjectError(traceError(err), bucket, object)
 	}
@@ -361,15 +364,31 @@ func (l *s3Objects) PutObject(bucket string, object string, size int64, data io.
 }
 
 // CopyObject copies a blob from source container to destination container.
-func (l *s3Objects) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string, metadata map[string]string) (objInfo ObjectInfo, e error) {
-	err := l.Client.CopyObject(destBucket, destObject, path.Join(srcBucket, srcObject), minio.CopyConditions{})
+func (l *s3Objects) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
+	// Source object
+	src := minio.NewSourceInfo(srcBucket, srcObject, nil)
+
+	// Destination object
+	var xamzMeta = map[string]string{}
+	for key := range metadata {
+		for _, prefix := range userMetadataKeyPrefixes {
+			if strings.HasPrefix(key, prefix) {
+				xamzMeta[key] = metadata[key]
+			}
+		}
+	}
+	dst, err := minio.NewDestinationInfo(dstBucket, dstObject, nil, xamzMeta)
 	if err != nil {
+		return objInfo, s3ToObjectError(traceError(err), dstBucket, dstObject)
+	}
+
+	if err = l.Client.CopyObject(dst, src); err != nil {
 		return objInfo, s3ToObjectError(traceError(err), srcBucket, srcObject)
 	}
 
-	oi, err := l.GetObjectInfo(destBucket, destObject)
+	oi, err := l.GetObjectInfo(dstBucket, dstObject)
 	if err != nil {
-		return objInfo, s3ToObjectError(traceError(err), destBucket, destObject)
+		return objInfo, s3ToObjectError(traceError(err), dstBucket, dstObject)
 	}
 
 	return oi, nil
@@ -474,17 +493,12 @@ func fromMinioClientObjectPart(op minio.ObjectPart) PartInfo {
 
 // PutObjectPart puts a part of object in bucket
 func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (pi PartInfo, e error) {
-	md5HexBytes, err := hex.DecodeString(md5Hex)
+	md5Bytes, sha256Bytes, err := getMD5AndSha256SumBytes(md5Hex, sha256sum)
 	if err != nil {
-		return pi, err
+		return pi, s3ToObjectError(traceError(err), bucket, object)
 	}
 
-	sha256sumBytes, err := hex.DecodeString(sha256sum)
-	if err != nil {
-		return pi, err
-	}
-
-	info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, size, data, md5HexBytes, sha256sumBytes)
+	info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, size, data, md5Bytes, sha256Bytes)
 	if err != nil {
 		return pi, err
 	}

cmd/gateway-s3_test.go

@@ -15,3 +15,44 @@
  */
 
 package cmd
+
+import (
+	"testing"
+)
+
+// Tests extracting md5/sha256 bytes.
+func TestGetMD5AndSha256Bytes(t *testing.T) {
+	testCases := []struct {
+		md5Hex    string
+		sha256Hex string
+		success   bool
+	}{
+		// Test 1: Hex encoding failure.
+		{
+			md5Hex:    "a",
+			sha256Hex: "b",
+			success:   false,
+		},
+		// Test 2: Hex encoding success.
+		{
+			md5Hex:    "91be0b892e47ede9de06aac14ca0369e",
+			sha256Hex: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+			success:   true,
+		},
+		// Test 3: hex values are empty should return success.
+		{
+			md5Hex:    "",
+			sha256Hex: "",
+			success:   true,
+		},
+	}
+	for i, testCase := range testCases {
+		_, _, err := getMD5AndSha256SumBytes(testCase.md5Hex, testCase.sha256Hex)
+		if err != nil && testCase.success {
+			t.Errorf("Test %d: Expected success, but got failure %s", i+1, err)
+		}
+		if err == nil && !testCase.success {
+			t.Errorf("Test %d: Expected failure, but got success", i+1)
+		}
+	}
+}