Reply back CompleteMultipartUploadResult properly with final ETag computed

- Now s3 libraries and also objectstorage-go work properly

pkg/storage/drivers/donut/donut.go

@@ -30,6 +30,7 @@ import (
 	"io/ioutil"
 
 	"errors"
+
 	"github.com/minio-io/minio/pkg/iodine"
 	"github.com/minio-io/minio/pkg/storage/donut"
 	"github.com/minio-io/minio/pkg/storage/drivers"
@@ -407,6 +408,6 @@ func (d donutDriver) CreateObjectPart(bucket, key, uploadID string, partID int,
 	return "", iodine.New(errors.New("Not Implemented"), nil)
 }
 
-func (d donutDriver) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) error {
-	return iodine.New(errors.New("Not Implemented"), nil)
+func (d donutDriver) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (string, error) {
+	return "", iodine.New(errors.New("Not Implemented"), nil)
 }

pkg/storage/drivers/driver.go

@@ -42,7 +42,7 @@ type Driver interface {
 	// Object Multipart Operations
 	NewMultipartUpload(bucket string, key string, contentType string) (string, error)
 	CreateObjectPart(bucket string, key string, uploadID string, partID int, contentType string, md5sum string, size int64, data io.Reader) (string, error)
-	CompleteMultipartUpload(bucket string, key string, uploadID string, parts map[int]string) error
+	CompleteMultipartUpload(bucket string, key string, uploadID string, parts map[int]string) (string, error)
 }
 
 // BucketACL - bucket level access control

pkg/storage/drivers/memory/memory.go

@@ -195,10 +195,10 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
 }
 
 func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
-	humanReadableErr, err := memory.createObject(bucket, key, contentType, expectedMD5Sum, size, data)
+	md5sum, err := memory.createObject(bucket, key, contentType, expectedMD5Sum, size, data)
 	// free
 	debug.FreeOSMemory()
-	return humanReadableErr, iodine.New(err, nil)
+	return md5sum, iodine.New(err, nil)
 }
 
 // getMD5AndData - this is written as a wrapper to capture md5sum and data in a more memory efficient way
@@ -497,15 +497,18 @@ func (memory *memoryDriver) GetObjectMetadata(bucket, key, prefix string) (drive
 
 func (memory *memoryDriver) evictObject(a ...interface{}) {
 	cacheStats := memory.objects.Stats()
-	log.Printf("CurrenSize: %d, CurrentItems: %d, TotalEvictions: %d",
+	log.Printf("CurrentSize: %d, CurrentItems: %d, TotalEvictions: %d",
 		cacheStats.Bytes, cacheStats.Items, cacheStats.Evictions)
 	key := a[0].(string)
 	// loop through all buckets
-	for bucket, storedBucket := range memory.storedBuckets {
+	for _, storedBucket := range memory.storedBuckets {
 		delete(storedBucket.objectMetadata, key)
 		// remove bucket if no objects found anymore
 		if len(storedBucket.objectMetadata) == 0 {
-			delete(memory.storedBuckets, bucket)
+			// TODO (y4m4)
+			// for now refrain from deleting buckets, due to multipart deletes before fullobject being written
+			// this case gets trigerred and we can't store the actual data at all receiving 404 on the client
+			// delete(memory.storedBuckets, bucket)
 		}
 	}
 	debug.FreeOSMemory()
@@ -531,7 +534,7 @@ func (memory *memoryDriver) CreateObjectPart(bucket, key, uploadID string, partI
 	return memory.CreateObject(bucket, getMultipartKey(key, uploadID, partID), "", expectedMD5Sum, size, data)
 }
 
-func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) error {
+func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (string, error) {
 	// TODO verify upload id exists
 	memory.lock.Lock()
 	size := int64(0)
@@ -542,7 +545,7 @@ func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string
 			}
 		} else {
 			memory.lock.Unlock()
-			return iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil)
+			return "", iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil)
 		}
 	}
 
@@ -552,13 +555,16 @@ func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string
 		if _, ok := parts[i]; ok {
 			if object, ok := memory.objects.Get(bucket + "/" + getMultipartKey(key, uploadID, i)); ok == true {
 				obj := object.([]byte)
-				io.Copy(&fullObject, bytes.NewBuffer(obj))
+				_, err := io.Copy(&fullObject, bytes.NewBuffer(obj))
+				if err != nil {
+					return "", iodine.New(err, nil)
+				}
 			} else {
 				log.Println("Cannot fetch: ", getMultipartKey(key, uploadID, i))
 			}
 		} else {
 			memory.lock.Unlock()
-			return iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil)
+			return "", iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil)
 		}
 	}
 
@@ -573,6 +579,5 @@ func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string
 
 	md5sumSlice := md5.Sum(fullObject.Bytes())
 	md5sum := base64.StdEncoding.EncodeToString(md5sumSlice[:])
-	_, err := memory.CreateObject(bucket, key, "", md5sum, size, &fullObject)
-	return err
+	return memory.CreateObject(bucket, key, "", md5sum, size, &fullObject)
 }

pkg/storage/drivers/mocks/Driver.go

@@ -146,10 +146,11 @@ func (m *Driver) CreateObjectPart(bucket string, key string, uploadID string, pa
 }
 
 // CompleteMultipartUpload is a mock
-func (m *Driver) CompleteMultipartUpload(bucket string, key string, uploadID string, parts map[int]string) error {
+func (m *Driver) CompleteMultipartUpload(bucket string, key string, uploadID string, parts map[int]string) (string, error) {
 	ret := m.Called(bucket, key, uploadID, parts)
 
-	r0 := ret.Error(0)
+	r0 := ret.Get(0).(string)
+	r1 := ret.Error(1)
 
-	return r0
+	return r0, r1
 }