Add cluster notification metrics in metrics-v3 (#19533)

Signed-off-by: Bala.FA <bala@minio.io>

cmd/metrics-v3-cluster-notification.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2024 MinIO, Inc.
+//
+// # This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+)
+
+const (
+	notificationCurrentSendInProgress = "current_send_in_progress"
+	notificationEventsErrorsTotal     = "events_errors_total"
+	notificationEventsSentTotal       = "events_sent_total"
+	notificationEventsSkippedTotal    = "events_skipped_total"
+)
+
+var (
+	notificationCurrentSendInProgressMD = NewCounterMD(notificationCurrentSendInProgress, "Number of concurrent async Send calls active to all targets")
+	notificationEventsErrorsTotalMD     = NewCounterMD(notificationEventsErrorsTotal, "Events that were failed to be sent to the targets")
+	notificationEventsSentTotalMD       = NewCounterMD(notificationEventsSentTotal, "Total number of events sent to the targets")
+	notificationEventsSkippedTotalMD    = NewCounterMD(notificationEventsSkippedTotal, "Events that were skipped to be sent to the targets due to the in-memory queue being full")
+)
+
+// loadClusterNotificationMetrics - `MetricsLoaderFn` for cluster notification metrics.
+func loadClusterNotificationMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
+	if globalEventNotifier == nil {
+		return nil
+	}
+
+	nstats := globalEventNotifier.targetList.Stats()
+	m.Set(notificationCurrentSendInProgress, float64(nstats.CurrentSendCalls))
+	m.Set(notificationEventsErrorsTotal, float64(nstats.EventsErrorsTotal))
+	m.Set(notificationEventsSentTotal, float64(nstats.TotalEvents))
+	m.Set(notificationEventsSkippedTotal, float64(nstats.EventsSkipped))
+
+	return nil
+}

cmd/metrics-v3.go

@@ -45,6 +45,7 @@ const (
 	clusterUsageBucketsCollectorPath collectorPath = "/cluster/usage/buckets"
 	clusterErasureSetCollectorPath   collectorPath = "/cluster/erasure-set"
 	clusterAuditCollectorPath        collectorPath = "/cluster/audit"
+	clusterNotificationCollectorPath collectorPath = "/cluster/notification"
 )
 
 const (
@@ -243,6 +244,16 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		loadClusterAuditMetrics,
 	)
 
+	clusterNotificationMG := NewMetricsGroup(clusterNotificationCollectorPath,
+		[]MetricDescriptor{
+			notificationCurrentSendInProgressMD,
+			notificationEventsErrorsTotalMD,
+			notificationEventsSentTotalMD,
+			notificationEventsSkippedTotalMD,
+		},
+		loadClusterNotificationMetrics,
+	)
+
 	allMetricGroups := []*MetricsGroup{
 		apiRequestsMG,
 		apiBucketMG,
@@ -257,6 +268,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		clusterUsageBucketsMG,
 		clusterErasureSetMG,
 		clusterAuditMG,
+		clusterNotificationMG,
 	}
 
 	// Bucket metrics are special, they always include the bucket label. These

docs/metrics/v3.md

@@ -221,3 +221,12 @@ The standard metrics groups for ProcessCollector and GoCollector are not shown b
 | `minio_cluster_erasure_set_online_drives_count`  | `gauge` | Count of online drives in the erasure set in a pool           | `pool_id,set_id` |
 | `minio_cluster_erasure_set_healing_drives_count` | `gauge` | Count of healing drives in the erasure set in a pool          | `pool_id,set_id` |
 | `minio_cluster_erasure_set_health`               | `gauge` | Health of the erasure set in a pool (1=healthy, 0=unhealthy)  | `pool_id,set_id` |
+
+### `/cluster/notification`
+
+| Name                                                  | Type      | Help                                                                                     | Labels |
+|-------------------------------------------------------|-----------|------------------------------------------------------------------------------------------|--------|
+| `minio_cluster_notification_current_send_in_progress` | `counter` | Number of concurrent async Send calls active to all targets                              |        |
+| `minio_cluster_notification_events_errors_total`      | `counter` | Events that were failed to be sent to the targets                                        |        |
+| `minio_cluster_notification_events_sent_total`        | `counter` | Total number of events sent to the targets                                               |        |
+| `minio_cluster_notification_events_skipped_total`     | `counter` | Events that were skipped to be sent to the targets due to the in-memory queue being full |        |