mirror of
https://github.com/minio/minio.git
synced 2024-12-24 22:25:54 -05:00
S3 only allows http headers with a size of 8 KB and user-defined metadata with a size of 2 KB. This change adds a new API error and returns this error to clients which sends to large http requests. Fixes #4634
This commit is contained in:
parent
b694c1a4d7
commit
3a73c675a6
@ -116,6 +116,7 @@ const (
|
|||||||
ErrInvalidDuration
|
ErrInvalidDuration
|
||||||
ErrNotSupported
|
ErrNotSupported
|
||||||
ErrBucketAlreadyExists
|
ErrBucketAlreadyExists
|
||||||
|
ErrMetadataTooLarge
|
||||||
// Add new error codes here.
|
// Add new error codes here.
|
||||||
|
|
||||||
// Bucket notification related errors.
|
// Bucket notification related errors.
|
||||||
@ -630,7 +631,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
|||||||
Description: "Cannot respond to plain-text request from TLS-encrypted server",
|
Description: "Cannot respond to plain-text request from TLS-encrypted server",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrMetadataTooLarge: {
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: "Your metadata headers exceed the maximum allowed metadata size.",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
// Add your error structure here.
|
// Add your error structure here.
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -62,6 +62,52 @@ func (h requestSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
|
|||||||
h.handler.ServeHTTP(w, r)
|
h.handler.ServeHTTP(w, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Maximum size for http headers - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
|
||||||
|
maxHeaderSize = 8 * 1024
|
||||||
|
// Maximum size for user-defined metadata - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
|
||||||
|
maxUserDataSize = 2 * 1024
|
||||||
|
)
|
||||||
|
|
||||||
|
type requestHeaderSizeLimitHandler struct {
|
||||||
|
http.Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
|
||||||
|
return requestHeaderSizeLimitHandler{h}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServeHTTP restricts the size of the http header to 8 KB and the size
|
||||||
|
// of the user-defined metadata to 2 KB.
|
||||||
|
func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if isHTTPHeaderSizeTooLarge(r.Header) {
|
||||||
|
writeErrorResponse(w, ErrMetadataTooLarge, r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.Handler.ServeHTTP(w, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isHTTPHeaderSizeTooLarge returns true if the provided
|
||||||
|
// header is larger than 8 KB or the user-defined metadata
|
||||||
|
// is larger than 2 KB.
|
||||||
|
func isHTTPHeaderSizeTooLarge(header http.Header) bool {
|
||||||
|
var size, usersize int
|
||||||
|
for key := range header {
|
||||||
|
length := len(key) + len(header.Get(key))
|
||||||
|
size += length
|
||||||
|
for _, prefix := range userMetadataKeyPrefixes {
|
||||||
|
if strings.HasPrefix(key, prefix) {
|
||||||
|
usersize += length
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if usersize > maxUserDataSize || size > maxHeaderSize {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// Reserved bucket.
|
// Reserved bucket.
|
||||||
const (
|
const (
|
||||||
minioReservedBucket = "minio"
|
minioReservedBucket = "minio"
|
||||||
|
@ -18,6 +18,7 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -88,3 +89,37 @@ func TestGuessIsBrowser(t *testing.T) {
|
|||||||
t.Fatal("Test shouldn't report as browser for a non browser request.")
|
t.Fatal("Test shouldn't report as browser for a non browser request.")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var isHTTPHeaderSizeTooLargeTests = []struct {
|
||||||
|
header http.Header
|
||||||
|
shouldFail bool
|
||||||
|
}{
|
||||||
|
{header: generateHeader(0, 0), shouldFail: false},
|
||||||
|
{header: generateHeader(1024, 0), shouldFail: false},
|
||||||
|
{header: generateHeader(2048, 0), shouldFail: false},
|
||||||
|
{header: generateHeader(8*1024+1, 0), shouldFail: true},
|
||||||
|
{header: generateHeader(0, 1024), shouldFail: false},
|
||||||
|
{header: generateHeader(0, 2048), shouldFail: true},
|
||||||
|
{header: generateHeader(0, 2048+1), shouldFail: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateHeader(size, usersize int) http.Header {
|
||||||
|
header := http.Header{}
|
||||||
|
for i := 0; i < size; i++ {
|
||||||
|
header.Add(strconv.Itoa(i), "")
|
||||||
|
}
|
||||||
|
userlength := 0
|
||||||
|
for i := 0; userlength < usersize; i++ {
|
||||||
|
userlength += len(userMetadataKeyPrefixes[0] + strconv.Itoa(i))
|
||||||
|
header.Add(userMetadataKeyPrefixes[0]+strconv.Itoa(i), "")
|
||||||
|
}
|
||||||
|
return header
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsHTTPHeaderSizeTooLarge(t *testing.T) {
|
||||||
|
for i, test := range isHTTPHeaderSizeTooLargeTests {
|
||||||
|
if res := isHTTPHeaderSizeTooLarge(test.header); res != test.shouldFail {
|
||||||
|
t.Errorf("Test %d: Expected %v got %v", i, res, test.shouldFail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -97,6 +97,14 @@ func path2BucketAndObject(path string) (bucket, object string) {
|
|||||||
return bucket, object
|
return bucket, object
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// userMetadataKeyPrefixes contains the prefixes of used-defined metadata keys.
|
||||||
|
// All values stored with a key starting with one of the following prefixes
|
||||||
|
// must be extracted from the header.
|
||||||
|
var userMetadataKeyPrefixes = []string{
|
||||||
|
"X-Amz-Meta-",
|
||||||
|
"X-Minio-Meta-",
|
||||||
|
}
|
||||||
|
|
||||||
// extractMetadataFromHeader extracts metadata from HTTP header.
|
// extractMetadataFromHeader extracts metadata from HTTP header.
|
||||||
func extractMetadataFromHeader(header http.Header) (map[string]string, error) {
|
func extractMetadataFromHeader(header http.Header) (map[string]string, error) {
|
||||||
if header == nil {
|
if header == nil {
|
||||||
@ -119,11 +127,11 @@ func extractMetadataFromHeader(header http.Header) (map[string]string, error) {
|
|||||||
if key != http.CanonicalHeaderKey(key) {
|
if key != http.CanonicalHeaderKey(key) {
|
||||||
return nil, traceError(errInvalidArgument)
|
return nil, traceError(errInvalidArgument)
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(key, "X-Amz-Meta-") {
|
for _, prefix := range userMetadataKeyPrefixes {
|
||||||
|
if strings.HasPrefix(key, prefix) {
|
||||||
metadata[key] = header.Get(key)
|
metadata[key] = header.Get(key)
|
||||||
|
break
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(key, "X-Minio-Meta-") {
|
|
||||||
metadata[key] = header.Get(key)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return metadata, nil
|
return metadata, nil
|
||||||
|
@ -91,6 +91,8 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
|
|||||||
setHTTPStatsHandler,
|
setHTTPStatsHandler,
|
||||||
// Limits all requests size to a maximum fixed limit
|
// Limits all requests size to a maximum fixed limit
|
||||||
setRequestSizeLimitHandler,
|
setRequestSizeLimitHandler,
|
||||||
|
// Limits all header sizes to a maximum fixed limit
|
||||||
|
setRequestHeaderSizeLimitHandler,
|
||||||
// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
|
// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
|
||||||
setCrossDomainPolicy,
|
setCrossDomainPolicy,
|
||||||
// Redirect some pre-defined browser request paths to a static location prefix.
|
// Redirect some pre-defined browser request paths to a static location prefix.
|
||||||
|
Loading…
Reference in New Issue
Block a user