api: Increase the maximum object size limit from 5GiB to 16GiB. (#3834)

The globalMaxObjectSize limit is instilled in S3 spec perhaps
due to certain limitations on S3 infrastructure. For minio we
don't have such limitations and we can stream a larger file
instead.

So we are going to bump this limit to 16GiB.

Fixes #3825

cmd/admin-handlers_test.go

@@ -1239,16 +1239,16 @@ func TestWriteSetConfigResponse(t *testing.T) {
 	}
 
 	testPeers := []adminPeer{
-		adminPeer{
+		{
 			addr: "localhost:9001",
 		},
-		adminPeer{
+		{
 			addr: "localhost:9002",
 		},
-		adminPeer{
+		{
 			addr: "localhost:9003",
 		},
-		adminPeer{
+		{
 			addr: "localhost:9004",
 		},
 	}

cmd/bucket-handlers.go

@@ -486,7 +486,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			return
 		}
 
-		if fileSize > lengthRange.Max || fileSize > maxObjectSize {
+		if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
 			errorIf(err, "Unable to create object.")
 			writeErrorResponse(w, toAPIErrorCode(errDataTooLarge), r.URL)
 			return

cmd/generic-handlers.go

@@ -46,9 +46,9 @@ func registerHandlers(mux *router.Router, handlerFns ...HandlerFunc) http.Handle
 // which is more than enough to accommodate any form data fields and headers.
 const requestFormDataSize = 64 * humanize.MiByte
 
-// For any HTTP request, request body should be not more than 5GiB + requestFormDataSize
-// where, 5GiB is the maximum allowed object size for object upload.
-const requestMaxBodySize = 5*humanize.GiByte + requestFormDataSize
+// For any HTTP request, request body should be not more than 16GiB + requestFormDataSize
+// where, 16GiB is the maximum allowed object size for object upload.
+const requestMaxBodySize = globalMaxObjectSize + requestFormDataSize
 
 type requestSizeLimitHandler struct {
 	handler     http.Handler

cmd/object-handlers.go

@@ -628,7 +628,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	}
 
 	/// maximum copy size for multipart objects in a single operation
-	if isMaxObjectSize(length) {
+	if isMaxAllowedPartSize(length) {
 		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
 		return
 	}
@@ -637,6 +637,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	// object is same then only metadata is updated.
 	partInfo, err := objectAPI.CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID, partID, startOffset, length)
 	if err != nil {
+		errorIf(err, "Unable to perform CopyObjectPart %s/%s", srcBucket, srcObject)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -687,7 +688,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	}
 
 	/// maximum Upload size for multipart objects in a single operation
-	if isMaxObjectSize(size) {
+	if isMaxAllowedPartSize(size) {
 		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
 		return
 	}

cmd/object-handlers_test.go

@@ -840,7 +840,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			req.ContentLength = -1
 			req.TransferEncoding = []string{}
 		case TooBigObject:
-			req.ContentLength = maxObjectSize + 1
+			req.ContentLength = globalMaxObjectSize + 1
 		}
 		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
 		// Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler`  handles the request.
@@ -883,7 +883,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			reqV2.ContentLength = -1
 			reqV2.TransferEncoding = []string{}
 		case TooBigObject:
-			reqV2.ContentLength = maxObjectSize + 1
+			reqV2.ContentLength = globalMaxObjectSize + 1
 		}
 
 		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
@@ -2722,7 +2722,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
 		{
 			objectName: testObject,
 			reader:     bytes.NewReader([]byte("hello")),
-			partNumber: strconv.Itoa(maxPartID + 1),
+			partNumber: strconv.Itoa(globalMaxPartID + 1),
 			fault:      None,
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
@@ -2880,7 +2880,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
 					// Setting the content length to a value greater than the max allowed size of a part.
 					// Used in test case  4.
 				case TooBigObject:
-					req.ContentLength = maxObjectSize + 1
+					req.ContentLength = globalMaxObjectSize + 1
 					// Malformed signature.
 					// Used in test case  6.
 				case BadSignature:

cmd/utils.go

@@ -170,27 +170,41 @@ func checkValidMD5(md5 string) ([]byte, error) {
 
 /// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
 const (
-	// maximum object size per PUT request is 5GiB
-	maxObjectSize = 5 * humanize.GiByte
-	// minimum Part size for multipart upload is 5MiB
-	minPartSize = 5 * humanize.MiByte
-	// maximum Part ID for multipart upload is 10000 (Acceptable values range from 1 to 10000 inclusive)
-	maxPartID = 10000
+	// Maximum object size per PUT request is 16GiB.
+	// This is a divergence from S3 limit on purpose to support
+	// use cases where users are going to upload large files
+	// using 'curl' and presigned URL.
+	globalMaxObjectSize = 16 * humanize.GiByte
+
+	// Minimum Part size for multipart upload is 5MiB
+	globalMinPartSize = 5 * humanize.MiByte
+
+	// Maximum Part size for multipart upload is 5GiB
+	globalMaxPartSize = 5 * humanize.GiByte
+
+	// Maximum Part ID for multipart upload is 10000
+	// (Acceptable values range from 1 to 10000 inclusive)
+	globalMaxPartID = 10000
 )
 
 // isMaxObjectSize - verify if max object size
 func isMaxObjectSize(size int64) bool {
-	return size > maxObjectSize
+	return size > globalMaxObjectSize
+}
+
+// // Check if part size is more than maximum allowed size.
+func isMaxAllowedPartSize(size int64) bool {
+	return size > globalMaxPartSize
 }
 
 // Check if part size is more than or equal to minimum allowed size.
 func isMinAllowedPartSize(size int64) bool {
-	return size >= minPartSize
+	return size >= globalMinPartSize
 }
 
 // isMaxPartNumber - Check if part ID is greater than the maximum allowed ID.
 func isMaxPartID(partID int) bool {
-	return partID > maxPartID
+	return partID > globalMaxPartID
 }
 
 func contains(stringList []string, element string) bool {

cmd/utils_test.go

@@ -110,12 +110,12 @@ func TestMaxObjectSize(t *testing.T) {
 		// Test - 1 - maximum object size.
 		{
 			true,
-			maxObjectSize + 1,
+			globalMaxObjectSize + 1,
 		},
 		// Test - 2 - not maximum object size.
 		{
 			false,
-			maxObjectSize - 1,
+			globalMaxObjectSize - 1,
 		},
 	}
 	for i, s := range sizes {
@@ -135,12 +135,12 @@ func TestMinAllowedPartSize(t *testing.T) {
 		// Test - 1 - within minimum part size.
 		{
 			true,
-			minPartSize + 1,
+			globalMinPartSize + 1,
 		},
 		// Test - 2 - smaller than minimum part size.
 		{
 			false,
-			minPartSize - 1,
+			globalMinPartSize - 1,
 		},
 	}
 
@@ -161,12 +161,12 @@ func TestMaxPartID(t *testing.T) {
 		// Test - 1 part number within max part number.
 		{
 			false,
-			maxPartID - 1,
+			globalMaxPartID - 1,
 		},
 		// Test - 2 part number bigger than max part number.
 		{
 			true,
-			maxPartID + 1,
+			globalMaxPartID + 1,
 		},
 	}