prometheus: Add incoming requests metrics since last scrape (#14261)

Some users running MinIO claim that their system became slow. One 
way to investigate is to look at this Prometheus history of the number of
the requests reaching the server. The existing current S3 requests metric
is not enough because it can increase of the system really becomes slow, 
due to disk issues for example.

cmd/admin-handlers.go

@@ -284,6 +284,7 @@ type ServerHTTPAPIStats struct {
 // including their average execution time.
 type ServerHTTPStats struct {
 	S3RequestsInQueue      int32              `json:"s3RequestsInQueue"`
+	S3RequestsIncoming     uint64             `json:"s3RequestsIncoming"`
 	CurrentS3Requests      ServerHTTPAPIStats `json:"currentS3Requests"`
 	TotalS3Requests        ServerHTTPAPIStats `json:"totalS3Requests"`
 	TotalS3Errors          ServerHTTPAPIStats `json:"totalS3Errors"`

cmd/handler-api.go

@@ -235,6 +235,8 @@ func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
 // maxClients throttles the S3 API calls
 func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		globalHTTPStats.incS3RequestsIncoming()
+
 		if val := globalServiceFreeze.Load(); val != nil {
 			if unlock, ok := val.(chan struct{}); ok && unlock != nil {
 				// Wait until unfrozen.

cmd/http-stats.go

@@ -143,6 +143,7 @@ type HTTPStats struct {
 	rejectedRequestsHeader  uint64
 	rejectedRequestsInvalid uint64
 	s3RequestsInQueue       int32
+	s3RequestsIncoming      uint64
 	currentS3Requests       HTTPAPIStats
 	totalS3Requests         HTTPAPIStats
 	totalS3Errors           HTTPAPIStats
@@ -153,9 +154,15 @@ func (st *HTTPStats) addRequestsInQueue(i int32) {
 	atomic.AddInt32(&st.s3RequestsInQueue, i)
 }
 
+func (st *HTTPStats) incS3RequestsIncoming() {
+	// Golang automatically resets to zero if this overflows
+	atomic.AddUint64(&st.s3RequestsIncoming, 1)
+}
+
 // Converts http stats into struct to be sent back to the client.
 func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 	serverStats := ServerHTTPStats{}
+	serverStats.S3RequestsIncoming = atomic.SwapUint64(&st.s3RequestsIncoming, 0)
 	serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue)
 	serverStats.TotalS3RejectedAuth = atomic.LoadUint64(&st.rejectedRequestsAuth)
 	serverStats.TotalS3RejectedTime = atomic.LoadUint64(&st.rejectedRequestsTime)

cmd/metrics-v2.go

@@ -137,6 +137,7 @@ const (
 	limitTotal     MetricName = "limit_total"
 	missedTotal    MetricName = "missed_total"
 	waitingTotal   MetricName = "waiting_total"
+	incomingTotal  MetricName = "incoming_total"
 	objectTotal    MetricName = "object_total"
 	offlineTotal   MetricName = "offline_total"
 	onlineTotal    MetricName = "online_total"
@@ -570,6 +571,16 @@ func getS3RequestsInQueueMD() MetricDescription {
 	}
 }
 
+func getIncomingS3RequestsMD() MetricDescription {
+	return MetricDescription{
+		Namespace: s3MetricNamespace,
+		Subsystem: requestsSubsystem,
+		Name:      incomingTotal,
+		Help:      "Volatile number of total incoming S3 requests",
+		Type:      gaugeMetric,
+	}
+}
+
 func getS3RequestsTotalMD() MetricDescription {
 	return MetricDescription{
 		Namespace: s3MetricNamespace,
@@ -1435,6 +1446,11 @@ func getHTTPMetrics() *MetricsGroup {
 			Description: getS3RequestsInQueueMD(),
 			Value:       float64(httpStats.S3RequestsInQueue),
 		})
+		metrics = append(metrics, Metric{
+			Description: getIncomingS3RequestsMD(),
+			Value:       float64(httpStats.S3RequestsIncoming),
+		})
+
 		for api, value := range httpStats.CurrentS3Requests.APIStats {
 			metrics = append(metrics, Metric{
 				Description:    getS3RequestsInFlightMD(),