restirct max size of http header and user metadata (#4634) (#4680)

S3 only allows http headers with a size of 8 KB and user-defined metadata
with a size of 2 KB. This change adds a new API error and returns this
error to clients which sends to large http requests.

Fixes #4634

cmd/api-errors.go

@@ -116,6 +116,7 @@ const (
 	ErrInvalidDuration
 	ErrNotSupported
 	ErrBucketAlreadyExists
+	ErrMetadataTooLarge
 	// Add new error codes here.
 
 	// Bucket notification related errors.
@@ -630,7 +631,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
+	ErrMetadataTooLarge: {
+		Code:           "InvalidArgument",
+		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 

cmd/generic-handlers.go

@@ -62,6 +62,52 @@ func (h requestSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 	h.handler.ServeHTTP(w, r)
 }
 
+const (
+	// Maximum size for http headers - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
+	maxHeaderSize = 8 * 1024
+	// Maximum size for user-defined metadata - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
+	maxUserDataSize = 2 * 1024
+)
+
+type requestHeaderSizeLimitHandler struct {
+	http.Handler
+}
+
+func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
+	return requestHeaderSizeLimitHandler{h}
+}
+
+// ServeHTTP restricts the size of the http header to 8 KB and the size
+// of the user-defined metadata to 2 KB.
+func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if isHTTPHeaderSizeTooLarge(r.Header) {
+		writeErrorResponse(w, ErrMetadataTooLarge, r.URL)
+		return
+	}
+	h.Handler.ServeHTTP(w, r)
+}
+
+// isHTTPHeaderSizeTooLarge returns true if the provided
+// header is larger than 8 KB or the user-defined metadata
+// is larger than 2 KB.
+func isHTTPHeaderSizeTooLarge(header http.Header) bool {
+	var size, usersize int
+	for key := range header {
+		length := len(key) + len(header.Get(key))
+		size += length
+		for _, prefix := range userMetadataKeyPrefixes {
+			if strings.HasPrefix(key, prefix) {
+				usersize += length
+				break
+			}
+		}
+		if usersize > maxUserDataSize || size > maxHeaderSize {
+			return true
+		}
+	}
+	return false
+}
+
 // Reserved bucket.
 const (
 	minioReservedBucket     = "minio"

cmd/generic-handlers_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"net/http"
+	"strconv"
 	"testing"
 )
 
@@ -88,3 +89,37 @@ func TestGuessIsBrowser(t *testing.T) {
 		t.Fatal("Test shouldn't report as browser for a non browser request.")
 	}
 }
+
+var isHTTPHeaderSizeTooLargeTests = []struct {
+	header     http.Header
+	shouldFail bool
+}{
+	{header: generateHeader(0, 0), shouldFail: false},
+	{header: generateHeader(1024, 0), shouldFail: false},
+	{header: generateHeader(2048, 0), shouldFail: false},
+	{header: generateHeader(8*1024+1, 0), shouldFail: true},
+	{header: generateHeader(0, 1024), shouldFail: false},
+	{header: generateHeader(0, 2048), shouldFail: true},
+	{header: generateHeader(0, 2048+1), shouldFail: true},
+}
+
+func generateHeader(size, usersize int) http.Header {
+	header := http.Header{}
+	for i := 0; i < size; i++ {
+		header.Add(strconv.Itoa(i), "")
+	}
+	userlength := 0
+	for i := 0; userlength < usersize; i++ {
+		userlength += len(userMetadataKeyPrefixes[0] + strconv.Itoa(i))
+		header.Add(userMetadataKeyPrefixes[0]+strconv.Itoa(i), "")
+	}
+	return header
+}
+
+func TestIsHTTPHeaderSizeTooLarge(t *testing.T) {
+	for i, test := range isHTTPHeaderSizeTooLargeTests {
+		if res := isHTTPHeaderSizeTooLarge(test.header); res != test.shouldFail {
+			t.Errorf("Test %d: Expected %v got %v", i, res, test.shouldFail)
+		}
+	}
+}

cmd/handler-utils.go

@@ -97,6 +97,14 @@ func path2BucketAndObject(path string) (bucket, object string) {
 	return bucket, object
 }
 
+// userMetadataKeyPrefixes contains the prefixes of used-defined metadata keys.
+// All values stored with a key starting with one of the following prefixes
+// must be extracted from the header.
+var userMetadataKeyPrefixes = []string{
+	"X-Amz-Meta-",
+	"X-Minio-Meta-",
+}
+
 // extractMetadataFromHeader extracts metadata from HTTP header.
 func extractMetadataFromHeader(header http.Header) (map[string]string, error) {
 	if header == nil {
@@ -119,11 +127,11 @@ func extractMetadataFromHeader(header http.Header) (map[string]string, error) {
 		if key != http.CanonicalHeaderKey(key) {
 			return nil, traceError(errInvalidArgument)
 		}
-		if strings.HasPrefix(key, "X-Amz-Meta-") {
+		for _, prefix := range userMetadataKeyPrefixes {
-			metadata[key] = header.Get(key)
+			if strings.HasPrefix(key, prefix) {
-		}
+				metadata[key] = header.Get(key)
-		if strings.HasPrefix(key, "X-Minio-Meta-") {
+				break
-			metadata[key] = header.Get(key)
+			}
 		}
 	}
 	return metadata, nil

cmd/routers.go

@@ -91,6 +91,8 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
 		setHTTPStatsHandler,
 		// Limits all requests size to a maximum fixed limit
 		setRequestSizeLimitHandler,
+		// Limits all header sizes to a maximum fixed limit
+		setRequestHeaderSizeLimitHandler,
 		// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
 		setCrossDomainPolicy,
 		// Redirect some pre-defined browser request paths to a static location prefix.