From 6ca6788bb7bb18da1d85e2a73927e90f7db730d7 Mon Sep 17 00:00:00 2001
From: jiuker <2818723467@qq.com>
Date: Fri, 8 Dec 2023 08:21:17 +0800
Subject: [PATCH] feat: add events_errors_total metric (#18610)

---
 cmd/metrics-v2.go            | 10 ++++++++++
 internal/event/targetlist.go | 10 +++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go
index ef2803bf5..4fcd33b3f 100644
--- a/cmd/metrics-v2.go
+++ b/cmd/metrics-v2.go
@@ -2475,6 +2475,16 @@ func getNotificationMetrics() *MetricsGroup {
 				},
 				Value: float64(nstats.EventsSkipped),
 			})
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: notifySubsystem,
+					Name:      "events_errors_total",
+					Help:      "Events that were failed while sending to target",
+					Type:      counterMetric,
+				},
+				Value: float64(nstats.EventsErrorsTotal),
+			})
 			metrics = append(metrics, Metric{
 				Description: MetricDescription{
 					Namespace: minioNamespace,
diff --git a/internal/event/targetlist.go b/internal/event/targetlist.go
index 44a3f5971..9c3c62ab5 100644
--- a/internal/event/targetlist.go
+++ b/internal/event/targetlist.go
@@ -57,6 +57,7 @@ type TargetStats struct {
 	TotalEvents        int64
 	EventsSkipped      int64
 	CurrentQueuedCalls int64
+	EventsErrorsTotal  int64
 
 	TargetStats map[string]TargetStat
 }
@@ -70,9 +71,10 @@ type TargetStat struct {
 // TargetList - holds list of targets indexed by target ID.
 type TargetList struct {
 	// The number of concurrent async Send calls to all targets
-	currentSendCalls atomic.Int64
-	totalEvents      atomic.Int64
-	eventsSkipped    atomic.Int64
+	currentSendCalls  atomic.Int64
+	totalEvents       atomic.Int64
+	eventsSkipped     atomic.Int64
+	eventsErrorsTotal atomic.Int64
 
 	sync.RWMutex
 	targets map[TargetID]Target
@@ -205,6 +207,7 @@ func (list *TargetList) sendSync(event Event, targetIDset TargetIDSet) {
 			defer wg.Done()
 
 			if err := target.Save(event); err != nil {
+				list.eventsErrorsTotal.Add(1)
 				reqInfo := &logger.ReqInfo{}
 				reqInfo.AppendTags("targetID", id.String())
 				logger.LogOnceIf(logger.SetReqInfo(context.Background(), reqInfo), err, id.String())
@@ -246,6 +249,7 @@ func (list *TargetList) Stats() TargetStats {
 	t.EventsSkipped = list.eventsSkipped.Load()
 	t.TotalEvents = list.totalEvents.Load()
 	t.CurrentQueuedCalls = int64(len(list.queue))
+	t.EventsErrorsTotal = list.eventsErrorsTotal.Load()
 
 	list.RLock()
 	defer list.RUnlock()