metric: add KMS-related metrics (#15258)

This commit adds a minimal set of KMS-related metrics:
```
 # HELP minio_cluster_kms_online Reports whether the KMS is online (1) or offline (0)
 # TYPE minio_cluster_kms_online gauge
 minio_cluster_kms_online{server="127.0.0.1:9000"} 1
 # HELP minio_cluster_kms_request_error Number of KMS requests that failed with a well-defined error
 # TYPE minio_cluster_kms_request_error counter
 minio_cluster_kms_request_error{server="127.0.0.1:9000"} 16790
 # HELP minio_cluster_kms_request_success Number of KMS requests that succeeded
 # TYPE minio_cluster_kms_request_success counter
 minio_cluster_kms_request_success{server="127.0.0.1:9000"} 348031
```

Currently, we report whether the KMS is available and how many requests
succeeded/failed. However, KES exposes much more metrics that can be
exposed if necessary. See: https://pkg.go.dev/github.com/minio/kes#Metric

Signed-off-by: Andreas Auernhammer <hi@aead.dev>

cmd/metrics-v2.go

@@ -27,6 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/minio/kes"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/logger"
@@ -49,6 +50,7 @@ func init() {
 		getNodeHealthMetrics(),
 		getClusterStorageMetrics(),
 		getClusterTierMetrics(),
+		getKMSMetrics(),
 	}
 
 	peerMetricsGroups = []*MetricsGroup{
@@ -63,6 +65,7 @@ func init() {
 		getILMNodeMetrics(),
 		getScannerNodeMetrics(),
 		getIAMNodeMetrics(),
+		getKMSNodeMetrics(),
 	}
 
 	allMetricsGroups := func() (allMetrics []*MetricsGroup) {
@@ -80,7 +83,6 @@ func init() {
 		getMinioVersionMetrics(),
 		getS3TTFBMetric(),
 	})
-
 	clusterCollector = newMinioClusterCollector(allMetricsGroups)
 }
 
@@ -123,6 +125,7 @@ const (
 	ilmSubsystem              MetricSubsystem = "ilm"
 	scannerSubsystem          MetricSubsystem = "scanner"
 	iamSubsystem              MetricSubsystem = "iam"
+	kmsSubsystem              MetricSubsystem = "kms"
 )
 
 // MetricName are the individual names for the metric.
@@ -188,6 +191,12 @@ const (
 	transitionedBytes    MetricName = "transitioned_bytes"
 	transitionedObjects  MetricName = "transitioned_objects"
 	transitionedVersions MetricName = "transitioned_versions"
+
+	kmsOnline          = "online"
+	kmsRequestsSuccess = "request_success"
+	kmsRequestsError   = "request_error"
+	kmsRequestsFail    = "request_failure"
+	kmsUptime          = "uptime"
 )
 
 const (
@@ -1931,6 +1940,107 @@ func getClusterStorageMetrics() *MetricsGroup {
 	return mg
 }
 
+func getKMSNodeMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: 10 * time.Second,
+	}
+
+	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
+		objLayer := newObjectLayerFn()
+		// Service not initialized yet
+		if objLayer == nil || globalIsGateway || GlobalKMS == nil {
+			return
+		}
+
+		const (
+			Online  = 1
+			Offline = 0
+		)
+		desc := MetricDescription{
+			Namespace: clusterMetricNamespace,
+			Subsystem: kmsSubsystem,
+			Name:      kmsOnline,
+			Help:      "Reports whether the KMS is online (1) or offline (0)",
+			Type:      gaugeMetric,
+		}
+		_, err := GlobalKMS.Metrics(ctx)
+		if _, ok := kes.IsConnError(err); ok {
+			return []Metric{{
+				Description: desc,
+				Value:       float64(Offline),
+			}}
+		}
+		return []Metric{{
+			Description: desc,
+			Value:       float64(Online),
+		}}
+	})
+	return mg
+}
+
+func getKMSMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: 10 * time.Second,
+	}
+
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		objLayer := newObjectLayerFn()
+		// Service not initialized yet
+		if objLayer == nil || globalIsGateway || GlobalKMS == nil {
+			return []Metric{}
+		}
+
+		metrics := make([]Metric, 0, 4)
+		metric, err := GlobalKMS.Metrics(ctx)
+		if err != nil {
+			return metrics
+		}
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: clusterMetricNamespace,
+				Subsystem: kmsSubsystem,
+				Name:      kmsRequestsSuccess,
+				Help:      "Number of KMS requests that succeeded",
+				Type:      counterMetric,
+			},
+			Value: float64(metric.RequestOK),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: clusterMetricNamespace,
+				Subsystem: kmsSubsystem,
+				Name:      kmsRequestsError,
+				Help:      "Number of KMS requests that failed due to some error. (HTTP 4xx status code)",
+				Type:      counterMetric,
+			},
+			Value: float64(metric.RequestErr),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: clusterMetricNamespace,
+				Subsystem: kmsSubsystem,
+				Name:      kmsRequestsFail,
+				Help:      "Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code)",
+				Type:      counterMetric,
+			},
+			Value: float64(metric.RequestFail),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: clusterMetricNamespace,
+				Subsystem: kmsSubsystem,
+				Name:      kmsUptime,
+				Help:      "The time the KMS has been up and running in seconds.",
+				Type:      counterMetric,
+			},
+			Value: metric.UpTime.Seconds(),
+		})
+
+		return metrics
+	})
+	return mg
+}
+
 type minioClusterCollector struct {
 	metricsGroups []*MetricsGroup
 	desc          *prometheus.Desc

go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dperf v0.4.2
 	github.com/minio/highwayhash v1.0.2
-	github.com/minio/kes v0.19.2
+	github.com/minio/kes v0.20.0
 	github.com/minio/madmin-go v1.4.3
 	github.com/minio/minio-go/v7 v7.0.30
 	github.com/minio/pkg v1.1.26
@@ -65,7 +65,7 @@ require (
 	github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9
 	github.com/pierrec/lz4 v2.6.1+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.12.1
+	github.com/prometheus/client_golang v1.12.2
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.34.0
 	github.com/prometheus/procfs v0.7.3

go.sum

@@ -618,8 +618,8 @@ github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEX
 github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
-github.com/minio/kes v0.19.2 h1:0kdMAgLMSkiDA33k8pMHC7d6erDuseuLrZF+N3017SM=
+github.com/minio/kes v0.20.0 h1:1tyC51Rr8zTregTESuT/QN/iebNMX7B9t7d3xLNMEpE=
-github.com/minio/kes v0.19.2/go.mod h1:X2fMkDbAkjbSKDGOQZvyPkHxoG7nuzP6R78Jw+TzXtM=
+github.com/minio/kes v0.20.0/go.mod h1:3FW1BQkMGQW78yhy+69tUq5bdcf5rnXJizyeKB9a/tc=
 github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
 github.com/minio/madmin-go v1.4.3 h1:5/kBHjKTjYOQQHvyznu51weN5hJtFW67LB2VLz+hmzU=
 github.com/minio/madmin-go v1.4.3/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
@@ -745,8 +745,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
+github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

internal/kms/kes.go

@@ -115,6 +115,10 @@ func (c *kesClient) Stat() (Status, error) {
 	}, nil
 }
 
+func (c *kesClient) Metrics(ctx context.Context) (kes.Metric, error) {
+	return c.client.Metrics(ctx)
+}
+
 // CreateKey tries to create a new key at the KMS with the
 // given key ID.
 //

internal/kms/kms.go

@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/kes"
 )
 
 // KMS is the generic interface that abstracts over
@@ -31,6 +32,9 @@ type KMS interface {
 	// Stat returns the current KMS status.
 	Stat() (Status, error)
 
+	// Metrics returns a KMS metric snapshot.
+	Metrics(ctx context.Context) (kes.Metric, error)
+
 	// CreateKey creates a new key at the KMS with the given key ID.
 	CreateKey(keyID string) error
 

internal/kms/single-key.go

@@ -33,6 +33,7 @@ import (
 	"golang.org/x/crypto/chacha20"
 	"golang.org/x/crypto/chacha20poly1305"
 
+	"github.com/minio/kes"
 	"github.com/minio/minio/internal/hash/sha256"
 )
 
@@ -89,6 +90,10 @@ func (kms secretKey) Stat() (Status, error) {
 	}, nil
 }
 
+func (secretKey) Metrics(ctx context.Context) (kes.Metric, error) {
+	return kes.Metric{}, errors.New("kms: metrics are not supported")
+}
+
 func (secretKey) CreateKey(string) error {
 	return errors.New("kms: creating keys is not supported")
 }