From 5b7c83341b2d0eb877c1bcced2285fe320cd586f Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Tue, 11 Jul 2023 07:46:24 -0700
Subject: [PATCH] move per bucket metrics to peer location (#17627)

---
 cmd/http-stats.go |  32 ++++++-------
 cmd/metrics-v2.go | 120 +++++++++++++++++++++++-----------------------
 2 files changed, 77 insertions(+), 75 deletions(-)

diff --git a/cmd/http-stats.go b/cmd/http-stats.go
index 7b22ce9df..8fd9442fc 100644
--- a/cmd/http-stats.go
+++ b/cmd/http-stats.go
@@ -236,28 +236,28 @@ func (s *bucketConnStats) incS3OutputBytes(bucket string, n int64) {
 	s.stats[bucket] = stats
 }
 
-// Return S3 total input bytes for input bucket
-func (s *bucketConnStats) getS3InputBytes(bucket string) uint64 {
-	s.RLock()
-	defer s.RUnlock()
-
-	stats := s.stats[bucket]
-	if stats == nil {
-		return 0
-	}
-	return stats.s3InputBytes
+type inOutBytes struct {
+	In  uint64
+	Out uint64
 }
 
-// Return S3 total output bytes
-func (s *bucketConnStats) getS3OutputBytes(bucket string) uint64 {
+// Return S3 total input bytes for input bucket
+func (s *bucketConnStats) getS3InOutBytes() map[string]inOutBytes {
 	s.RLock()
 	defer s.RUnlock()
 
-	stats := s.stats[bucket]
-	if stats == nil {
-		return 0
+	if len(s.stats) == 0 {
+		return nil
 	}
-	return stats.s3OutputBytes
+
+	bucketStats := make(map[string]inOutBytes, len(s.stats))
+	for k, v := range s.stats {
+		bucketStats[k] = inOutBytes{
+			In:  v.s3InputBytes,
+			Out: v.s3OutputBytes,
+		}
+	}
+	return bucketStats
 }
 
 // delete metrics once bucket is deleted.
diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go
index 7933d8ac8..4363b94c2 100644
--- a/cmd/metrics-v2.go
+++ b/cmd/metrics-v2.go
@@ -1990,6 +1990,67 @@ func getHTTPMetrics() *MetricsGroup {
 				VariableLabels: map[string]string{"api": api},
 			})
 		}
+
+		for bucket, inOut := range globalBucketConnStats.getS3InOutBytes() {
+			recvBytes := inOut.In
+			if recvBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficReceivedBytes(),
+					Value:          float64(recvBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+			sentBytes := inOut.Out
+			if sentBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficSentBytes(),
+					Value:          float64(sentBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+
+			httpStats := globalBucketHTTPStats.load(bucket)
+			for k, v := range httpStats.currentS3Requests.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsInFlightMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS3Requests.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsTotalMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS3Canceled.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsCanceledMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS34xxErrors.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3Requests4xxErrorsMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS35xxErrors.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3Requests5xxErrorsMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+		}
+
 		return
 	})
 	return mg
@@ -2104,65 +2165,6 @@ func getBucketUsageMetrics() *MetricsGroup {
 				})
 			}
 
-			recvBytes := globalBucketConnStats.getS3InputBytes(bucket)
-			if recvBytes > 0 {
-				metrics = append(metrics, Metric{
-					Description:    getBucketTrafficReceivedBytes(),
-					Value:          float64(recvBytes),
-					VariableLabels: map[string]string{"bucket": bucket},
-				})
-			}
-
-			sentBytes := globalBucketConnStats.getS3OutputBytes(bucket)
-			if sentBytes > 0 {
-				metrics = append(metrics, Metric{
-					Description:    getBucketTrafficSentBytes(),
-					Value:          float64(sentBytes),
-					VariableLabels: map[string]string{"bucket": bucket},
-				})
-			}
-
-			httpStats := globalBucketHTTPStats.load(bucket)
-			for k, v := range httpStats.currentS3Requests.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3RequestsInFlightMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS3Requests.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3RequestsTotalMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS3Canceled.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3RequestsCanceledMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS34xxErrors.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3Requests4xxErrorsMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS35xxErrors.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3Requests5xxErrorsMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
 			if stats.hasReplicationUsage() {
 				for arn, stat := range stats.Stats {
 					metrics = append(metrics, Metric{