api: Increase the maximum object size limit from 5GiB to 16GiB. (#3834)

The globalMaxObjectSize limit is instilled in S3 spec perhaps
due to certain limitations on S3 infrastructure. For minio we
don't have such limitations and we can stream a larger file
instead.

So we are going to bump this limit to 16GiB.

Fixes #3825
This commit is contained in:
Harshavardhana 2017-03-03 10:14:17 -08:00 committed by GitHub
parent 28c53a3555
commit bc52d911ef
7 changed files with 44 additions and 29 deletions

View File

@ -1239,16 +1239,16 @@ func TestWriteSetConfigResponse(t *testing.T) {
}
testPeers := []adminPeer{
adminPeer{
{
addr: "localhost:9001",
},
adminPeer{
{
addr: "localhost:9002",
},
adminPeer{
{
addr: "localhost:9003",
},
adminPeer{
{
addr: "localhost:9004",
},
}

View File

@ -486,7 +486,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
return
}
if fileSize > lengthRange.Max || fileSize > maxObjectSize {
if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
errorIf(err, "Unable to create object.")
writeErrorResponse(w, toAPIErrorCode(errDataTooLarge), r.URL)
return

View File

@ -46,9 +46,9 @@ func registerHandlers(mux *router.Router, handlerFns ...HandlerFunc) http.Handle
// which is more than enough to accommodate any form data fields and headers.
const requestFormDataSize = 64 * humanize.MiByte
// For any HTTP request, request body should be not more than 5GiB + requestFormDataSize
// where, 5GiB is the maximum allowed object size for object upload.
const requestMaxBodySize = 5*humanize.GiByte + requestFormDataSize
// For any HTTP request, request body should be not more than 16GiB + requestFormDataSize
// where, 16GiB is the maximum allowed object size for object upload.
const requestMaxBodySize = globalMaxObjectSize + requestFormDataSize
type requestSizeLimitHandler struct {
handler http.Handler

View File

@ -628,7 +628,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
}
/// maximum copy size for multipart objects in a single operation
if isMaxObjectSize(length) {
if isMaxAllowedPartSize(length) {
writeErrorResponse(w, ErrEntityTooLarge, r.URL)
return
}
@ -637,6 +637,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
// object is same then only metadata is updated.
partInfo, err := objectAPI.CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID, partID, startOffset, length)
if err != nil {
errorIf(err, "Unable to perform CopyObjectPart %s/%s", srcBucket, srcObject)
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
return
}
@ -687,7 +688,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
}
/// maximum Upload size for multipart objects in a single operation
if isMaxObjectSize(size) {
if isMaxAllowedPartSize(size) {
writeErrorResponse(w, ErrEntityTooLarge, r.URL)
return
}

View File

@ -840,7 +840,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
req.ContentLength = -1
req.TransferEncoding = []string{}
case TooBigObject:
req.ContentLength = maxObjectSize + 1
req.ContentLength = globalMaxObjectSize + 1
}
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
// Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler` handles the request.
@ -883,7 +883,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
reqV2.ContentLength = -1
reqV2.TransferEncoding = []string{}
case TooBigObject:
reqV2.ContentLength = maxObjectSize + 1
reqV2.ContentLength = globalMaxObjectSize + 1
}
// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
@ -2722,7 +2722,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
{
objectName: testObject,
reader: bytes.NewReader([]byte("hello")),
partNumber: strconv.Itoa(maxPartID + 1),
partNumber: strconv.Itoa(globalMaxPartID + 1),
fault: None,
accessKey: credentials.AccessKey,
secretKey: credentials.SecretKey,
@ -2880,7 +2880,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
// Setting the content length to a value greater than the max allowed size of a part.
// Used in test case 4.
case TooBigObject:
req.ContentLength = maxObjectSize + 1
req.ContentLength = globalMaxObjectSize + 1
// Malformed signature.
// Used in test case 6.
case BadSignature:

View File

@ -170,27 +170,41 @@ func checkValidMD5(md5 string) ([]byte, error) {
/// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
const (
// maximum object size per PUT request is 5GiB
maxObjectSize = 5 * humanize.GiByte
// minimum Part size for multipart upload is 5MiB
minPartSize = 5 * humanize.MiByte
// maximum Part ID for multipart upload is 10000 (Acceptable values range from 1 to 10000 inclusive)
maxPartID = 10000
// Maximum object size per PUT request is 16GiB.
// This is a divergence from S3 limit on purpose to support
// use cases where users are going to upload large files
// using 'curl' and presigned URL.
globalMaxObjectSize = 16 * humanize.GiByte
// Minimum Part size for multipart upload is 5MiB
globalMinPartSize = 5 * humanize.MiByte
// Maximum Part size for multipart upload is 5GiB
globalMaxPartSize = 5 * humanize.GiByte
// Maximum Part ID for multipart upload is 10000
// (Acceptable values range from 1 to 10000 inclusive)
globalMaxPartID = 10000
)
// isMaxObjectSize - verify if max object size
func isMaxObjectSize(size int64) bool {
return size > maxObjectSize
return size > globalMaxObjectSize
}
// // Check if part size is more than maximum allowed size.
func isMaxAllowedPartSize(size int64) bool {
return size > globalMaxPartSize
}
// Check if part size is more than or equal to minimum allowed size.
func isMinAllowedPartSize(size int64) bool {
return size >= minPartSize
return size >= globalMinPartSize
}
// isMaxPartNumber - Check if part ID is greater than the maximum allowed ID.
func isMaxPartID(partID int) bool {
return partID > maxPartID
return partID > globalMaxPartID
}
func contains(stringList []string, element string) bool {

View File

@ -110,12 +110,12 @@ func TestMaxObjectSize(t *testing.T) {
// Test - 1 - maximum object size.
{
true,
maxObjectSize + 1,
globalMaxObjectSize + 1,
},
// Test - 2 - not maximum object size.
{
false,
maxObjectSize - 1,
globalMaxObjectSize - 1,
},
}
for i, s := range sizes {
@ -135,12 +135,12 @@ func TestMinAllowedPartSize(t *testing.T) {
// Test - 1 - within minimum part size.
{
true,
minPartSize + 1,
globalMinPartSize + 1,
},
// Test - 2 - smaller than minimum part size.
{
false,
minPartSize - 1,
globalMinPartSize - 1,
},
}
@ -161,12 +161,12 @@ func TestMaxPartID(t *testing.T) {
// Test - 1 part number within max part number.
{
false,
maxPartID - 1,
globalMaxPartID - 1,
},
// Test - 2 part number bigger than max part number.
{
true,
maxPartID + 1,
globalMaxPartID + 1,
},
}