diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index bee4d9449..bf1660aa7 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -284,6 +284,7 @@ type ServerHTTPAPIStats struct { // including their average execution time. type ServerHTTPStats struct { S3RequestsInQueue int32 `json:"s3RequestsInQueue"` + S3RequestsIncoming uint64 `json:"s3RequestsIncoming"` CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"` TotalS3Requests ServerHTTPAPIStats `json:"totalS3Requests"` TotalS3Errors ServerHTTPAPIStats `json:"totalS3Errors"` diff --git a/cmd/handler-api.go b/cmd/handler-api.go index 29b41c2a7..22b6015be 100644 --- a/cmd/handler-api.go +++ b/cmd/handler-api.go @@ -235,6 +235,8 @@ func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) { // maxClients throttles the S3 API calls func maxClients(f http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + globalHTTPStats.incS3RequestsIncoming() + if val := globalServiceFreeze.Load(); val != nil { if unlock, ok := val.(chan struct{}); ok && unlock != nil { // Wait until unfrozen. diff --git a/cmd/http-stats.go b/cmd/http-stats.go index 4db09b571..980a6e828 100644 --- a/cmd/http-stats.go +++ b/cmd/http-stats.go @@ -143,6 +143,7 @@ type HTTPStats struct { rejectedRequestsHeader uint64 rejectedRequestsInvalid uint64 s3RequestsInQueue int32 + s3RequestsIncoming uint64 currentS3Requests HTTPAPIStats totalS3Requests HTTPAPIStats totalS3Errors HTTPAPIStats @@ -153,9 +154,15 @@ func (st *HTTPStats) addRequestsInQueue(i int32) { atomic.AddInt32(&st.s3RequestsInQueue, i) } +func (st *HTTPStats) incS3RequestsIncoming() { + // Golang automatically resets to zero if this overflows + atomic.AddUint64(&st.s3RequestsIncoming, 1) +} + // Converts http stats into struct to be sent back to the client. func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats { serverStats := ServerHTTPStats{} + serverStats.S3RequestsIncoming = atomic.SwapUint64(&st.s3RequestsIncoming, 0) serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue) serverStats.TotalS3RejectedAuth = atomic.LoadUint64(&st.rejectedRequestsAuth) serverStats.TotalS3RejectedTime = atomic.LoadUint64(&st.rejectedRequestsTime) diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index 1f4bc39cc..457176a3a 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -137,6 +137,7 @@ const ( limitTotal MetricName = "limit_total" missedTotal MetricName = "missed_total" waitingTotal MetricName = "waiting_total" + incomingTotal MetricName = "incoming_total" objectTotal MetricName = "object_total" offlineTotal MetricName = "offline_total" onlineTotal MetricName = "online_total" @@ -570,6 +571,16 @@ func getS3RequestsInQueueMD() MetricDescription { } } +func getIncomingS3RequestsMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: incomingTotal, + Help: "Volatile number of total incoming S3 requests", + Type: gaugeMetric, + } +} + func getS3RequestsTotalMD() MetricDescription { return MetricDescription{ Namespace: s3MetricNamespace, @@ -1435,6 +1446,11 @@ func getHTTPMetrics() *MetricsGroup { Description: getS3RequestsInQueueMD(), Value: float64(httpStats.S3RequestsInQueue), }) + metrics = append(metrics, Metric{ + Description: getIncomingS3RequestsMD(), + Value: float64(httpStats.S3RequestsIncoming), + }) + for api, value := range httpStats.CurrentS3Requests.APIStats { metrics = append(metrics, Metric{ Description: getS3RequestsInFlightMD(),