Add logger webhook metrics in metrics-v3 (#19515)

endpoint: /minio/metrics/v3/cluster/webhook
metrics:
- failed_messages (counter)
- online (gauge)
- queue_length (gauge)
- total_messages (counter)

cmd/metrics-v2.go

@@ -296,9 +296,6 @@ const (
 	kmsUptime          = "uptime"
 
 	webhookOnline = "online"
-	webhookQueueLength    = "queue_length"
-	webhookTotalMessages  = "total_messages"
-	webhookFailedMessages = "failed_messages"
 )
 
 const (

cmd/metrics-v3-logger-webhook.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+
+	"github.com/minio/minio/internal/logger"
+)
+
+const (
+	webhookQueueLength    = "queue_length"
+	webhookTotalMessages  = "total_messages"
+	webhookFailedMessages = "failed_messages"
+	nameL                 = "name"
+	endpointL             = "endpoint"
+)
+
+var (
+	allWebhookLabels        = []string{nameL, endpointL}
+	webhookFailedMessagesMD = NewCounterMD(webhookFailedMessages,
+		"Number of messages that failed to send",
+		allWebhookLabels...)
+	webhookQueueLengthMD = NewGaugeMD(webhookQueueLength,
+		"Webhook queue length",
+		allWebhookLabels...)
+	webhookTotalMessagesMD = NewCounterMD(webhookTotalMessages,
+		"Total number of messages sent to this target",
+		allWebhookLabels...)
+)
+
+// loadLoggerWebhookMetrics - `MetricsLoaderFn` for logger webhook
+// such as failed messages and total messages.
+func loadLoggerWebhookMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
+	tgts := append(logger.SystemTargets(), logger.AuditTargets()...)
+	for _, t := range tgts {
+		labels := []string{nameL, t.String(), endpointL, t.Endpoint()}
+		m.Set(webhookFailedMessages, float64(t.Stats().FailedMessages), labels...)
+		m.Set(webhookQueueLength, float64(t.Stats().QueueLength), labels...)
+		m.Set(webhookTotalMessages, float64(t.Stats().TotalMessages), labels...)
+	}
+
+	return nil
+}

cmd/metrics-v3.go

@@ -53,6 +53,7 @@ const (
 	clusterIAMCollectorPath          collectorPath = "/cluster/iam"
 
 	auditCollectorPath         collectorPath = "/audit"
+	loggerWebhookCollectorPath collectorPath = "/logger/webhook"
 )
 
 const (
@@ -295,6 +296,15 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		loadClusterIAMMetrics,
 	)
 
+	loggerWebhookMG := NewMetricsGroup(loggerWebhookCollectorPath,
+		[]MetricDescriptor{
+			webhookFailedMessagesMD,
+			webhookQueueLengthMD,
+			webhookTotalMessagesMD,
+		},
+		loadLoggerWebhookMetrics,
+	)
+
 	auditMG := NewMetricsGroup(auditCollectorPath,
 		[]MetricDescriptor{
 			auditFailedMessagesMD,
@@ -322,6 +332,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		clusterIAMMG,
 
 		auditMG,
+		loggerWebhookMG,
 	}
 
 	// Bucket metrics are special, they always include the bucket label. These

docs/metrics/v3.md

@@ -42,6 +42,14 @@ These are metrics about the minio process and the node.
 |----------|----------------------------------------|
 | `/audit` | Metrics related to audit functionality |
 
+### Logger webhook metrics
+
+These are metrics about the minio logger webhooks
+
+| Path              | Description                        |
+|-------------------|------------------------------------|
+| `/logger/webhook` | Metrics related to logger webhooks |
+
 ### System metrics
 
 These are metrics about the minio process and the node.
@@ -286,3 +294,11 @@ The standard metrics group for GoCollector is not shown below.
 | `minio_cluster_iam_since_last_sync_millis`                      | `counter` | Time (in milliseconds) since last successful IAM data sync                                                               |        |
 | `minio_cluster_iam_sync_failures`                               | `counter` | Number of failed IAM data syncs since server start                                                                       |        |
 | `minio_cluster_iam_sync_successes`                              | `counter` | Number of successful IAM data syncs since server start                                                                   |        |
+
+### `/logger/webhook`
+
+| Name                                    | Type      | Help                                         | Labels                 |
+|-----------------------------------------|-----------|----------------------------------------------|------------------------|
+| `minio_logger_webhook_failed_messages`  | `counter` | Number of messages that failed to send       | `server,name,endpoint` |
+| `minio_logger_webhook_queue_length`     | `gauge`   | Webhook queue length                         | `server,name,endpoint` |
+| `minio_logger_webhook_total_message`    | `counter` | Total number of messages sent to this target | `server,name,endpoint` |