When object whose size is greater than 5G is uploaded using presigned POST we should return error. (#3033)

fixes #2961

cmd/api-errors.go

@@ -627,6 +627,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooSmall
 	case SHA256Mismatch:
 		apiErr = ErrContentSHA256Mismatch
+	case ObjectTooLarge:
+		apiErr = ErrEntityTooLarge
 	default:
 		apiErr = ErrInternalError
 	}

cmd/handler-utils.go

@@ -114,7 +114,7 @@ func extractPostPolicyFormValues(reader *multipart.Reader) (filePart io.Reader,
 				}
 				formValues[canonicalFormName] = string(buffer)
 			} else {
-				filePart = io.LimitReader(part, maxObjectSize)
+				filePart = limitReader(part, maxObjectSize)
 				fileName = part.FileName()
 				// As described in S3 spec, we expect file to be the last form field
 				break

cmd/object-errors.go

@@ -74,6 +74,13 @@ func toObjectErr(err error, params ...string) error {
 				Object: params[1],
 			}
 		}
+	case errDataTooLarge:
+		if len(params) >= 2 {
+			err = ObjectTooLarge{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		}
 	case errXLReadQuorum:
 		err = InsufficientReadQuorum{}
 	case errXLWriteQuorum:
@@ -252,6 +259,13 @@ func (e InvalidRange) Error() string {
 	return fmt.Sprintf("The requested range \"bytes %d-%d/%d\" is not satisfiable.", e.offsetBegin, e.offsetEnd, e.resourceSize)
 }
 
+// ObjectTooLarge error returned when the size of the object > max object size allowed (5G) per request.
+type ObjectTooLarge GenericError
+
+func (e ObjectTooLarge) Error() string {
+	return "size of the object greater than what is allowed(5G)"
+}
+
 /// Multipart related errors.
 
 // MalformedUploadID malformed upload id.

cmd/object-utils.go

@@ -20,6 +20,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"path"
 	"regexp"
 	"strings"
@@ -161,3 +162,24 @@ type byBucketName []BucketInfo
 func (d byBucketName) Len() int           { return len(d) }
 func (d byBucketName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
 func (d byBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }
+
+// Copied from io.LimitReader()
+// limitReader returns a Reader that reads from r
+// but returns error after n bytes.
+// The underlying implementation is a *LimitedReader.
+type limitedReader struct {
+	R io.Reader // underlying reader
+	N int64     // max bytes remaining
+}
+
+func limitReader(r io.Reader, n int64) io.Reader { return &limitedReader{r, n} }
+
+func (l *limitedReader) Read(p []byte) (n int, err error) {
+	n, err = l.R.Read(p)
+	l.N -= int64(n)
+	if l.N < 0 {
+		// If more data is available than what is expected we return error.
+		return 0, errDataTooLarge
+	}
+	return
+}

cmd/object-utils_test.go

@@ -17,6 +17,8 @@
 package cmd
 
 import (
+	"io/ioutil"
+	"strings"
 	"testing"
 )
 
@@ -113,3 +115,24 @@ func TestIsValidObjectName(t *testing.T) {
 		}
 	}
 }
+
+// Tests limitReader
+func TestLimitReader(t *testing.T) {
+	testCases := []struct {
+		data   string
+		maxLen int64
+		err    error
+	}{
+		{"1234567890", 15, nil},
+		{"1234567890", 10, nil},
+		{"1234567890", 5, errDataTooLarge},
+	}
+
+	for i, test := range testCases {
+		r := strings.NewReader(test.data)
+		_, err := ioutil.ReadAll(limitReader(r, test.maxLen))
+		if err != test.err {
+			t.Fatalf("test %d failed: expected %v, got %v", i+1, test.err, err)
+		}
+	}
+}

cmd/typed-errors.go

@@ -35,3 +35,6 @@ var errContentSHA256Mismatch = errors.New("Content checksum SHA256 mismatch")
 
 // used when we deal with data larger than expected
 var errSizeUnexpected = errors.New("Data size larger than expected")
+
+// When upload object size is greater than 5G in a single PUT/POST operation.
+var errDataTooLarge = errors.New("Object size larger than allowed limit")