Add rate limiter for S3 API layer (#9196)

- total number of S3 API calls per server
- maximum wait duration for any S3 API call

This implementation is primarily meant for situations
where HDDs are not capable enough to handle the incoming
workload and there is no way to throttle the client.

This feature allows MinIO server to throttle itself
such that we do not overwhelm the HDDs.

cmd/handler-utils.go

@@ -28,6 +28,7 @@ import (
 	"net/url"
 	"regexp"
 	"strings"
+	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -358,6 +359,31 @@ func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
+// maxClients throttles the S3 API calls
+func maxClients(f http.HandlerFunc, enabled bool, requestsMaxCh chan struct{}, requestsDeadline time.Duration) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !enabled {
+			f.ServeHTTP(w, r)
+			return
+		}
+
+		select {
+		case requestsMaxCh <- struct{}{}:
+			defer func() { <-requestsMaxCh }()
+			f.ServeHTTP(w, r)
+		case <-time.NewTimer(requestsDeadline).C:
+
+			// Send a http timeout message
+			writeErrorResponse(r.Context(), w,
+				errorCodes.ToAPIErr(ErrOperationMaxedOut),
+				r.URL, guessIsBrowserReq(r))
+			return
+		case <-r.Context().Done():
+			return
+		}
+	}
+}
+
 func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {