Add Audit target metrics (#16044)

cmd/consolelogger.go

@@ -21,6 +21,7 @@ import (
 	"container/ring"
 	"context"
 	"sync"
+	"sync/atomic"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
@@ -36,6 +37,9 @@ const defaultLogBufferCount = 10000
 
 // HTTPConsoleLoggerSys holds global console logger state
 type HTTPConsoleLoggerSys struct {
+	totalMessages  int64
+	failedMessages int64
+
 	sync.RWMutex
 	pubsub   *pubsub.PubSub[log.Info, madmin.LogMask]
 	console  *console.Target
@@ -133,6 +137,15 @@ func (sys *HTTPConsoleLoggerSys) String() string {
 	return logger.ConsoleLoggerTgt
 }
 
+// Stats returns the target statistics.
+func (sys *HTTPConsoleLoggerSys) Stats() types.TargetStats {
+	return types.TargetStats{
+		TotalMessages:  atomic.LoadInt64(&sys.totalMessages),
+		FailedMessages: atomic.LoadInt64(&sys.failedMessages),
+		QueueLength:    0,
+	}
+}
+
 // Content returns the console stdout log
 func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
 	sys.RLock()
@@ -170,6 +183,7 @@ func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
 	case string:
 		lg = log.Info{ConsoleMsg: e, NodeName: sys.nodeName}
 	}
+	atomic.AddInt64(&sys.totalMessages, 1)
 
 	sys.pubsub.Publish(lg)
 	sys.Lock()
@@ -177,6 +191,9 @@ func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
 	sys.logBuf.Value = lg
 	sys.logBuf = sys.logBuf.Next()
 	sys.Unlock()
-
-	return sys.console.Send(entry, string(logger.All))
+	err := sys.console.Send(entry, string(logger.All))
+	if err != nil {
+		atomic.AddInt64(&sys.failedMessages, 1)
+	}
+	return err
 }

cmd/data-update-tracker_test.go

@@ -62,6 +62,11 @@ func (t *testingLogger) Type() types.TargetType {
 	return types.TargetHTTP
 }
 
+// Stats returns the target statistics.
+func (t *testingLogger) Stats() types.TargetStats {
+	return types.TargetStats{}
+}
+
 func (t *testingLogger) Send(entry interface{}) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()

cmd/metrics-v2.go

@@ -130,6 +130,7 @@ const (
 	iamSubsystem              MetricSubsystem = "iam"
 	kmsSubsystem              MetricSubsystem = "kms"
 	notifySubsystem           MetricSubsystem = "notify"
+	auditSubsystem            MetricSubsystem = "audit"
 )
 
 // MetricName are the individual names for the metric.
@@ -1567,6 +1568,43 @@ func getNotificationMetrics() *MetricsGroup {
 				Value:          float64(st.CurrentQueue),
 			})
 		}
+		// Audit and system:
+		audit := logger.CurrentStats()
+		for id, st := range audit {
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: auditSubsystem,
+					Name:      "target_queue_length",
+					Help:      "Number of unsent messages in queue for target",
+					Type:      gaugeMetric,
+				},
+				VariableLabels: map[string]string{"target_id": id},
+				Value:          float64(st.QueueLength),
+			})
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: auditSubsystem,
+					Name:      "total_messages",
+					Help:      "Total number of messages sent since start",
+					Type:      counterMetric,
+				},
+				VariableLabels: map[string]string{"target_id": id},
+				Value:          float64(st.TotalMessages),
+			})
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: auditSubsystem,
+					Name:      "failed_messages",
+					Help:      "Total number of messages that failed to send since start",
+					Type:      counterMetric,
+				},
+				VariableLabels: map[string]string{"target_id": id},
+				Value:          float64(st.FailedMessages),
+			})
+		}
 		return metrics
 	})
 	return mg