remove requests deadline, instead just reject the requests (#20272)

Additionally set

 - x-ratelimit-limit
 - x-ratelimit-remaining

To indicate the request rates.

cmd/api-errors.go

@@ -1486,7 +1486,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrTooManyRequests: {
 		Code:           "TooManyRequests",
-		Description:    "Deadline exceeded while waiting in incoming queue, please reduce your request rate",
+		Description:    "Please reduce your request rate",
 		HTTPStatusCode: http.StatusTooManyRequests,
 	},
 	ErrUnsupportedMetadata: {

cmd/api-response.go

@@ -947,19 +947,10 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 // writeErrorResponse writes error headers
 func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	switch err.HTTPStatusCode {
-	case http.StatusServiceUnavailable:
+	case http.StatusServiceUnavailable, http.StatusTooManyRequests:
 		// Set retry-after header to indicate user-agents to retry request after 60 seconds.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "60")
-	case http.StatusTooManyRequests:
-		_, deadline := globalAPIConfig.getRequestsPool()
-		if deadline <= 0 {
-			// Set retry-after header to indicate user-agents to retry request after 10 seconds.
-			// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-			w.Header().Set(xhttp.RetryAfter, "10")
-		} else {
-			w.Header().Set(xhttp.RetryAfter, strconv.Itoa(int(deadline.Seconds())))
-		}
 	}
 
 	switch err.Code {

cmd/handler-api.go

@@ -39,7 +39,6 @@ import (
 type apiConfig struct {
 	mu sync.RWMutex
 
-	requestsDeadline       time.Duration
 	requestsPool           chan struct{}
 	clusterDeadline        time.Duration
 	listQuorum             string
@@ -164,7 +163,6 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 		// but this shouldn't last long.
 		t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
 	}
-	t.requestsDeadline = cfg.RequestsDeadline
 	listQuorum := cfg.ListQuorum
 	if listQuorum == "" {
 		listQuorum = "strict"
@@ -290,19 +288,15 @@ func (t *apiConfig) getRequestsPoolCapacity() int {
 	return cap(t.requestsPool)
 }
 
-func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
+func (t *apiConfig) getRequestsPool() chan struct{} {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
 	if t.requestsPool == nil {
-		return nil, 10 * time.Second
+		return nil
 	}
 
-	if t.requestsDeadline <= 0 {
-		return t.requestsPool, 10 * time.Second
-	}
-
-	return t.requestsPool, t.requestsDeadline
+	return t.requestsPool
 }
 
 // maxClients throttles the S3 API calls
@@ -325,27 +319,19 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 		}
 
 		globalHTTPStats.addRequestsInQueue(1)
-		pool, deadline := globalAPIConfig.getRequestsPool()
+		pool := globalAPIConfig.getRequestsPool()
 		if pool == nil {
 			globalHTTPStats.addRequestsInQueue(-1)
 			f.ServeHTTP(w, r)
 			return
 		}
 
-		// No deadline to wait, there is nothing to queue
-		// perform the API call immediately.
-		if deadline <= 0 {
-			globalHTTPStats.addRequestsInQueue(-1)
-			f.ServeHTTP(w, r)
-			return
-		}
-
 		if tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt); ok {
 			tc.FuncName = "s3.MaxClients"
 		}
 
-		deadlineTimer := time.NewTimer(deadline)
-		defer deadlineTimer.Stop()
+		w.Header().Set("X-RateLimit-Limit", strconv.Itoa(cap(pool)))
+		w.Header().Set("X-RateLimit-Remaining", strconv.Itoa(cap(pool)-len(pool)))
 
 		ctx := r.Context()
 		select {
@@ -357,7 +343,13 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 				return
 			}
 			f.ServeHTTP(w, r)
-		case <-deadlineTimer.C:
+		case <-r.Context().Done():
+			globalHTTPStats.addRequestsInQueue(-1)
+			// When the client disconnects before getting the S3 handler
+			// status code response, set the status code to 499 so this request
+			// will be properly audited and traced.
+			w.WriteHeader(499)
+		default:
 			globalHTTPStats.addRequestsInQueue(-1)
 			if contextCanceled(ctx) {
 				w.WriteHeader(499)
@@ -367,12 +359,7 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 			writeErrorResponse(ctx, w,
 				errorCodes.ToAPIErr(ErrTooManyRequests),
 				r.URL)
-		case <-r.Context().Done():
-			globalHTTPStats.addRequestsInQueue(-1)
-			// When the client disconnects before getting the S3 handler
-			// status code response, set the status code to 499 so this request
-			// will be properly audited and traced.
-			w.WriteHeader(499)
+
 		}
 	}
 }