From 2ee337ead5784d60971973b5399c20ebd9fdce44 Mon Sep 17 00:00:00 2001
From: Anis Elleuch <vadmeste@users.noreply.github.com>
Date: Tue, 8 Feb 2022 01:30:14 +0100
Subject: [PATCH] prometheus: Add incoming requests metrics since last scrape
 (#14261)

Some users running MinIO claim that their system became slow. One
way to investigate is to look at this Prometheus history of the number of
the requests reaching the server. The existing current S3 requests metric
is not enough because it can increase of the system really becomes slow,
due to disk issues for example.
---
 cmd/admin-handlers.go |  1 +
 cmd/handler-api.go    |  2 ++
 cmd/http-stats.go     |  7 +++++++
 cmd/metrics-v2.go     | 16 ++++++++++++++++
 4 files changed, 26 insertions(+)

diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go
index bee4d9449..bf1660aa7 100644
--- a/cmd/admin-handlers.go
+++ b/cmd/admin-handlers.go
@@ -284,6 +284,7 @@ type ServerHTTPAPIStats struct {
 // including their average execution time.
 type ServerHTTPStats struct {
 	S3RequestsInQueue      int32              `json:"s3RequestsInQueue"`
+	S3RequestsIncoming     uint64             `json:"s3RequestsIncoming"`
 	CurrentS3Requests      ServerHTTPAPIStats `json:"currentS3Requests"`
 	TotalS3Requests        ServerHTTPAPIStats `json:"totalS3Requests"`
 	TotalS3Errors          ServerHTTPAPIStats `json:"totalS3Errors"`
diff --git a/cmd/handler-api.go b/cmd/handler-api.go
index 29b41c2a7..22b6015be 100644
--- a/cmd/handler-api.go
+++ b/cmd/handler-api.go
@@ -235,6 +235,8 @@ func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
 // maxClients throttles the S3 API calls
 func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		globalHTTPStats.incS3RequestsIncoming()
+
 		if val := globalServiceFreeze.Load(); val != nil {
 			if unlock, ok := val.(chan struct{}); ok && unlock != nil {
 				// Wait until unfrozen.
diff --git a/cmd/http-stats.go b/cmd/http-stats.go
index 4db09b571..980a6e828 100644
--- a/cmd/http-stats.go
+++ b/cmd/http-stats.go
@@ -143,6 +143,7 @@ type HTTPStats struct {
 	rejectedRequestsHeader  uint64
 	rejectedRequestsInvalid uint64
 	s3RequestsInQueue       int32
+	s3RequestsIncoming      uint64
 	currentS3Requests       HTTPAPIStats
 	totalS3Requests         HTTPAPIStats
 	totalS3Errors           HTTPAPIStats
@@ -153,9 +154,15 @@ func (st *HTTPStats) addRequestsInQueue(i int32) {
 	atomic.AddInt32(&st.s3RequestsInQueue, i)
 }
 
+func (st *HTTPStats) incS3RequestsIncoming() {
+	// Golang automatically resets to zero if this overflows
+	atomic.AddUint64(&st.s3RequestsIncoming, 1)
+}
+
 // Converts http stats into struct to be sent back to the client.
 func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 	serverStats := ServerHTTPStats{}
+	serverStats.S3RequestsIncoming = atomic.SwapUint64(&st.s3RequestsIncoming, 0)
 	serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue)
 	serverStats.TotalS3RejectedAuth = atomic.LoadUint64(&st.rejectedRequestsAuth)
 	serverStats.TotalS3RejectedTime = atomic.LoadUint64(&st.rejectedRequestsTime)
diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go
index 1f4bc39cc..457176a3a 100644
--- a/cmd/metrics-v2.go
+++ b/cmd/metrics-v2.go
@@ -137,6 +137,7 @@ const (
 	limitTotal     MetricName = "limit_total"
 	missedTotal    MetricName = "missed_total"
 	waitingTotal   MetricName = "waiting_total"
+	incomingTotal  MetricName = "incoming_total"
 	objectTotal    MetricName = "object_total"
 	offlineTotal   MetricName = "offline_total"
 	onlineTotal    MetricName = "online_total"
@@ -570,6 +571,16 @@ func getS3RequestsInQueueMD() MetricDescription {
 	}
 }
 
+func getIncomingS3RequestsMD() MetricDescription {
+	return MetricDescription{
+		Namespace: s3MetricNamespace,
+		Subsystem: requestsSubsystem,
+		Name:      incomingTotal,
+		Help:      "Volatile number of total incoming S3 requests",
+		Type:      gaugeMetric,
+	}
+}
+
 func getS3RequestsTotalMD() MetricDescription {
 	return MetricDescription{
 		Namespace: s3MetricNamespace,
@@ -1435,6 +1446,11 @@ func getHTTPMetrics() *MetricsGroup {
 			Description: getS3RequestsInQueueMD(),
 			Value:       float64(httpStats.S3RequestsInQueue),
 		})
+		metrics = append(metrics, Metric{
+			Description: getIncomingS3RequestsMD(),
+			Value:       float64(httpStats.S3RequestsIncoming),
+		})
+
 		for api, value := range httpStats.CurrentS3Requests.APIStats {
 			metrics = append(metrics, Metric{
 				Description:    getS3RequestsInFlightMD(),