MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

reduce number of middleware handlers (#13546)

Browse Source 
- combine similar looking functionalities into single
  handlers, and remove unnecessary proxying of the
  requests at handler layer.

- remove bucket forwarding handler as part of default setup
  add it only if bucket federation is enabled.

Improvements observed for 1kiB object reads.
```
-------------------
Operation: GET
Operations: 4538555 -> 4595804
* Average: +1.26% (+0.2 MiB/s) throughput, +1.26% (+190.2) obj/s
* Fastest: +4.67% (+0.7 MiB/s) throughput, +4.67% (+739.8) obj/s
* 50% Median: +1.15% (+0.2 MiB/s) throughput, +1.15% (+173.9) obj/s
```
This commit is contained in:
Harshavardhana
2021-11-01 08:04:03 -07:00
committed by GitHub
parent 8ed7346273
commit 6d53e3c2d7
10 changed files with 125 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
cmd/generic-handlers.go
View File

@@ -18,7 +18,6 @@
package cmd
import (
"context"
"net"
"net/http"
"path"
@@ -37,42 +36,39 @@ import (
"github.com/minio/minio/internal/logger"
)
// Adds limiting body size middleware
// Maximum allowed form data field values. 64MiB is a guessed practical value
// which is more than enough to accommodate any form data fields and headers.
const requestFormDataSize = 64 * humanize.MiByte
// For any HTTP request, request body should be not more than 16GiB + requestFormDataSize
// where, 16GiB is the maximum allowed object size for object upload.
const requestMaxBodySize = globalMaxObjectSize + requestFormDataSize
func setRequestSizeLimitHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Restricting read data to a given maximum length
r.Body = http.MaxBytesReader(w, r.Body, requestMaxBodySize)
h.ServeHTTP(w, r)
})
}
const (
// Maximum allowed form data field values. 64MiB is a guessed practical value
// which is more than enough to accommodate any form data fields and headers.
requestFormDataSize = 64 * humanize.MiByte
// For any HTTP request, request body should be not more than 16GiB + requestFormDataSize
// where, 16GiB is the maximum allowed object size for object upload.
requestMaxBodySize = globalMaxObjectSize + requestFormDataSize
// Maximum size for http headers - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
maxHeaderSize = 8 * 1024
// Maximum size for user-defined metadata - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
maxUserDataSize = 2 * 1024
)
// ServeHTTP restricts the size of the http header to 8 KB and the size
// of the user-defined metadata to 2 KB.
func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if isHTTPHeaderSizeTooLarge(r.Header) {
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL)
atomic.AddUint64(&globalHTTPStats.rejectedRequestsHeader, 1)
return
// ReservedMetadataPrefix is the prefix of a metadata key which
// is reserved and for internal use only.
const (
ReservedMetadataPrefix = "X-Minio-Internal-"
ReservedMetadataPrefixLower = "x-minio-internal-"
)
// containsReservedMetadata returns true if the http.Header contains
// keys which are treated as metadata but are reserved for internal use
// and must not set by clients
func containsReservedMetadata(header http.Header) bool {
for key := range header {
if strings.HasPrefix(strings.ToLower(key), ReservedMetadataPrefixLower) {
return true
}
h.ServeHTTP(w, r)
})
}
return false
}
// isHTTPHeaderSizeTooLarge returns true if the provided
@@ -96,37 +92,25 @@ func isHTTPHeaderSizeTooLarge(header http.Header) bool {
return false
}
// ReservedMetadataPrefix is the prefix of a metadata key which
// is reserved and for internal use only.
const (
ReservedMetadataPrefix = "X-Minio-Internal-"
ReservedMetadataPrefixLower = "x-minio-internal-"
)
// ServeHTTP fails if the request contains at least one reserved header which
// would be treated as metadata.
func filterReservedMetadata(h http.Handler) http.Handler {
// Limits body and header to specific allowed maximum limits as per S3/MinIO API requirements.
func setRequestLimitHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Reject unsupported reserved metadata first before validation.
if containsReservedMetadata(r.Header) {
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrUnsupportedMetadata), r.URL)
return
}
if isHTTPHeaderSizeTooLarge(r.Header) {
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL)
atomic.AddUint64(&globalHTTPStats.rejectedRequestsHeader, 1)
return
}
// Restricting read data to a given maximum length
r.Body = http.MaxBytesReader(w, r.Body, requestMaxBodySize)
h.ServeHTTP(w, r)
})
}
// containsReservedMetadata returns true if the http.Header contains
// keys which are treated as metadata but are reserved for internal use
// and must not set by clients
func containsReservedMetadata(header http.Header) bool {
for key := range header {
if strings.HasPrefix(strings.ToLower(key), ReservedMetadataPrefixLower) {
return true
}
}
return false
}
// Reserved bucket.
const (
minioReservedBucket = "minio"
@@ -134,24 +118,6 @@ const (
loginPathPrefix = SlashSeparator + "login"
)
func setRedirectHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !shouldProxy() || guessIsRPCReq(r) || guessIsBrowserReq(r) ||
guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || isAdminReq(r) {
h.ServeHTTP(w, r)
return
}
// if this server is still initializing, proxy the request
// to any other online servers to avoid 503 for any incoming
// API calls.
if idx := getOnlineProxyEndpointIdx(); idx >= 0 {
proxyRequest(context.TODO(), w, r, globalProxyEndpoints[idx])
return
}
h.ServeHTTP(w, r)
})
}
func guessIsBrowserReq(r *http.Request) bool {
aType := getRequestAuthType(r)
return strings.Contains(r.Header.Get("User-Agent"), "Mozilla") &&
@@ -174,10 +140,6 @@ func setBrowserRedirectHandler(h http.Handler) http.Handler {
})
}
func shouldProxy() bool {
return newObjectLayerFn() == nil
}
// Fetch redirect location if urlPath satisfies certain
// criteria. Some special names are considered to be
// redirectable, this is purely internal function and
@@ -261,31 +223,21 @@ func isAdminReq(r *http.Request) bool {
return strings.HasPrefix(r.URL.Path, adminPathPrefix)
}
// Adds verification for incoming paths.
func setReservedBucketHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// For all other requests reject access to reserved buckets
bucketName, _ := request2BucketObjectName(r)
if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
if !guessIsRPCReq(r) && !guessIsBrowserReq(r) && !guessIsHealthCheckReq(r) && !guessIsMetricsReq(r) && !isAdminReq(r) {
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL)
return
}
}
h.ServeHTTP(w, r)
})
}
// Supported Amz date formats.
// Supported amz date formats.
var amzDateFormats = []string{
// Do not change this order, x-amz-date format is usually in
// iso8601Format rest are meant for relaxed handling of other
// odd SDKs that might be out there.
iso8601Format,
time.RFC1123,
time.RFC1123Z,
iso8601Format,
// Add new AMZ date formats here.
}
// Supported Amz date headers.
var amzDateHeaders = []string{
// Do not chane this order, x-amz-date value should be
// validated first.
"x-amz-date",
"date",
}
@@ -314,34 +266,6 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
return time.Time{}, ErrMissingDateHeader
}
// setTimeValidityHandler to validate parsable time over http header
func setTimeValidityHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
aType := getRequestAuthType(r)
if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
// Verify if date headers are set, if not reject the request
amzDate, errCode := parseAmzDateHeader(r)
if errCode != ErrNone {
// All our internal APIs are sensitive towards Date
// header, for all requests where Date header is not
// present we will reject such clients.
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL)
atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
return
}
// Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past
// or in the future, reject request otherwise.
curTime := UTCNow()
if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
return
}
}
h.ServeHTTP(w, r)
})
}
// setHttpStatsHandler sets a http Stats handler to gather HTTP statistics
func setHTTPStatsHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -420,6 +344,23 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
return
}
// For all other requests reject access to reserved buckets
bucketName, _ := request2BucketObjectName(r)
if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
if !guessIsRPCReq(r) && !guessIsBrowserReq(r) && !guessIsHealthCheckReq(r) && !guessIsMetricsReq(r) && !isAdminReq(r) {
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL)
return
}
}
// Deny SSE-C requests if not made over TLS
if !globalIsTLS && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
if r.Method == http.MethodHead {
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest))
} else {
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL)
}
return
}
h.ServeHTTP(w, r)
})
}
@@ -429,8 +370,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
// is obtained from centralized etcd configuration service.
func setBucketForwardingHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if globalDNSConfig == nil || len(globalDomainNames) == 0 || !globalBucketFederation ||
guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
if guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) {
h.ServeHTTP(w, r)
return
@@ -491,29 +431,23 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
})
}
// customHeaderHandler sets x-amz-request-id header.
// Previously, this value was set right before a response was sent to
// the client. So, logger and Error response XML were not using this
// value. This is set here so that this header can be logged as
// part of the log entry, Error response XML and auditing.
func addCustomHeaders(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Set custom headers such as x-amz-request-id for each request.
w.Header().Set(xhttp.AmzRequestID, mustGetRequestID(UTCNow()))
h.ServeHTTP(logger.NewResponseWriter(w), r)
})
}
// addSecurityHeaders adds various HTTP(S) response headers.
// addCustomHeaders adds various HTTP(S) response headers.
// Security Headers enable various security protections behaviors in the client's browser.
func addSecurityHeaders(h http.Handler) http.Handler {
func addCustomHeaders(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
header := w.Header()
header.Set("X-XSS-Protection", "1; mode=block") // Prevents against XSS attacks
header.Set("Content-Security-Policy", "block-all-mixed-content") // prevent mixed (HTTP / HTTPS content)
header.Set("X-Content-Type-Options", "nosniff") // Prevent mime-sniff
header.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains") // HSTS mitigates variants of MITM attacks
h.ServeHTTP(w, r)
// Previously, this value was set right before a response was sent to
// the client. So, logger and Error response XML were not using this
// value. This is set here so that this header can be logged as
// part of the log entry, Error response XML and auditing.
// Set custom headers such as x-amz-request-id for each request.
w.Header().Set(xhttp.AmzRequestID, mustGetRequestID(UTCNow()))
h.ServeHTTP(logger.NewResponseWriter(w), r)
})
}
@@ -521,32 +455,16 @@ func addSecurityHeaders(h http.Handler) http.Handler {
// `panic(logger.ErrCritical)` as done by `logger.CriticalIf`.
//
// It should be always the first / highest HTTP handler.
type criticalErrorHandler struct{ handler http.Handler }
func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err == logger.ErrCritical { // handle
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
return
} else if err != nil {
panic(err) // forward other panic calls
}
}()
h.handler.ServeHTTP(w, r)
}
// sseTLSHandler enforces certain rules for SSE requests which are made / must be made over TLS.
func setSSETLSHandler(h http.Handler) http.Handler {
func setCriticalErrorHandler(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Deny SSE-C requests if not made over TLS
if !globalIsTLS && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
if r.Method == http.MethodHead {
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest))
} else {
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL)
defer func() {
if err := recover(); err == logger.ErrCritical { // handle
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
return
} else if err != nil {
panic(err) // forward other panic calls
}
return
}
}()
h.ServeHTTP(w, r)
})
}