From 20761e053e7ee4022225cbe981c0f82f561f1017 Mon Sep 17 00:00:00 2001
From: Anis Elleuch <vadmeste@users.noreply.github.com>
Date: Sat, 23 Oct 2021 03:13:50 +0100
Subject: [PATCH] replication: Fix replica stats during crawling (#13499)

Also show replica stats with an ARN in Prometheus output.
---
 cmd/data-usage-cache.go | 46 +++++++++++++++++++++++------------------
 cmd/metrics-v2.go       | 11 +++++-----
 cmd/metrics.go          |  2 +-
 3 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go
index 1e0764ebe..eb79a36b0 100644
--- a/cmd/data-usage-cache.go
+++ b/cmd/data-usage-cache.go
@@ -219,20 +219,26 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.Versions += summary.versions
 	e.ObjSizes.add(summary.totalSize)
 
-	if summary.replTargetStats != nil {
-		if e.ReplicationStats == nil {
-			e.ReplicationStats = &replicationAllStats{Targets: make(map[string]replicationStats)}
+	if e.ReplicationStats == nil {
+		e.ReplicationStats = &replicationAllStats{
+			Targets: make(map[string]replicationStats),
 		}
+	} else if e.ReplicationStats.Targets == nil {
+		e.ReplicationStats.Targets = make(map[string]replicationStats)
+	}
+	e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
+
+	if summary.replTargetStats != nil {
 		for arn, st := range summary.replTargetStats {
 			tgtStat, ok := e.ReplicationStats.Targets[arn]
 			if !ok {
 				tgtStat = replicationStats{}
 			}
-			tgtStat.PendingSize = tgtStat.PendingSize + uint64(st.pendingSize)
-			tgtStat.FailedSize = tgtStat.FailedSize + uint64(st.failedSize)
-			tgtStat.ReplicatedSize = tgtStat.ReplicatedSize + uint64(st.replicatedSize)
-			tgtStat.FailedCount = tgtStat.FailedCount + st.failedCount
-			tgtStat.PendingCount = tgtStat.PendingCount + st.pendingCount
+			tgtStat.PendingSize += uint64(st.pendingSize)
+			tgtStat.FailedSize += uint64(st.failedSize)
+			tgtStat.ReplicatedSize += uint64(st.replicatedSize)
+			tgtStat.FailedCount += st.failedCount
+			tgtStat.PendingCount += st.pendingCount
 			e.ReplicationStats.Targets[arn] = tgtStat
 		}
 	}
@@ -243,21 +249,21 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Objects += other.Objects
 	e.Versions += other.Versions
 	e.Size += other.Size
-	ors := other.ReplicationStats
-	if ors != nil && len(ors.Targets) > 0 {
+	if other.ReplicationStats != nil {
 		if e.ReplicationStats == nil {
 			e.ReplicationStats = &replicationAllStats{Targets: make(map[string]replicationStats)}
+		} else if e.ReplicationStats.Targets == nil {
+			e.ReplicationStats.Targets = make(map[string]replicationStats)
 		}
-		if other.ReplicationStats != nil {
-			for arn, stat := range other.ReplicationStats.Targets {
-				st := e.ReplicationStats.Targets[arn]
-				e.ReplicationStats.Targets[arn] = replicationStats{
-					PendingSize:    stat.PendingSize + st.PendingSize,
-					FailedSize:     stat.FailedSize + st.FailedSize,
-					ReplicatedSize: stat.ReplicatedSize + st.ReplicatedSize,
-					PendingCount:   stat.PendingCount + st.PendingCount,
-					FailedCount:    stat.FailedCount + st.FailedCount,
-				}
+		e.ReplicationStats.ReplicaSize += other.ReplicationStats.ReplicaSize
+		for arn, stat := range other.ReplicationStats.Targets {
+			st := e.ReplicationStats.Targets[arn]
+			e.ReplicationStats.Targets[arn] = replicationStats{
+				PendingSize:    stat.PendingSize + st.PendingSize,
+				FailedSize:     stat.FailedSize + st.FailedSize,
+				ReplicatedSize: stat.ReplicatedSize + st.ReplicatedSize,
+				PendingCount:   stat.PendingCount + st.PendingCount,
+				FailedCount:    stat.FailedCount + st.FailedCount,
 			}
 		}
 	}
diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go
index 09657cb2d..8b0300283 100644
--- a/cmd/metrics-v2.go
+++ b/cmd/metrics-v2.go
@@ -1459,6 +1459,12 @@ func getBucketUsageMetrics() MetricsGroup {
 					VariableLabels: map[string]string{"bucket": bucket},
 				})
 
+				metrics = append(metrics, Metric{
+					Description:    getBucketRepReceivedBytesMD(),
+					Value:          float64(stats.ReplicaSize),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+
 				if stats.hasReplicationUsage() {
 					for arn, stat := range stats.Stats {
 						metrics = append(metrics, Metric{
@@ -1471,11 +1477,6 @@ func getBucketUsageMetrics() MetricsGroup {
 							Value:          float64(stat.ReplicatedSize),
 							VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn},
 						})
-						metrics = append(metrics, Metric{
-							Description:    getBucketRepReceivedBytesMD(),
-							Value:          float64(stat.ReplicaSize),
-							VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn},
-						})
 						metrics = append(metrics, Metric{
 							Description:    getBucketRepFailedOperationsMD(),
 							Value:          float64(stat.FailedCount),
diff --git a/cmd/metrics.go b/cmd/metrics.go
index 195d72909..1ec4000c8 100644
--- a/cmd/metrics.go
+++ b/cmd/metrics.go
@@ -456,8 +456,8 @@ func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplic
 
 	// add initial usage stat to cluster stats
 	usageStat := globalReplicationStats.GetInitialUsage(bucket)
+	totReplicaSize += usageStat.ReplicaSize
 	if usageStat.Stats != nil {
-		totReplicaSize += usageStat.ReplicaSize
 		for arn, stat := range usageStat.Stats {
 			st := stats[arn]
 			if st == nil {