diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index 3116ea071..24efe52d1 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -27,6 +27,7 @@ import ( "sync/atomic" "time" + "github.com/minio/kes" "github.com/minio/madmin-go" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/logger" @@ -49,6 +50,7 @@ func init() { getNodeHealthMetrics(), getClusterStorageMetrics(), getClusterTierMetrics(), + getKMSMetrics(), } peerMetricsGroups = []*MetricsGroup{ @@ -63,6 +65,7 @@ func init() { getILMNodeMetrics(), getScannerNodeMetrics(), getIAMNodeMetrics(), + getKMSNodeMetrics(), } allMetricsGroups := func() (allMetrics []*MetricsGroup) { @@ -80,7 +83,6 @@ func init() { getMinioVersionMetrics(), getS3TTFBMetric(), }) - clusterCollector = newMinioClusterCollector(allMetricsGroups) } @@ -123,6 +125,7 @@ const ( ilmSubsystem MetricSubsystem = "ilm" scannerSubsystem MetricSubsystem = "scanner" iamSubsystem MetricSubsystem = "iam" + kmsSubsystem MetricSubsystem = "kms" ) // MetricName are the individual names for the metric. @@ -188,6 +191,12 @@ const ( transitionedBytes MetricName = "transitioned_bytes" transitionedObjects MetricName = "transitioned_objects" transitionedVersions MetricName = "transitioned_versions" + + kmsOnline = "online" + kmsRequestsSuccess = "request_success" + kmsRequestsError = "request_error" + kmsRequestsFail = "request_failure" + kmsUptime = "uptime" ) const ( @@ -1931,6 +1940,107 @@ func getClusterStorageMetrics() *MetricsGroup { return mg } +func getKMSNodeMetrics() *MetricsGroup { + mg := &MetricsGroup{ + cacheInterval: 10 * time.Second, + } + + mg.RegisterRead(func(ctx context.Context) (metrics []Metric) { + objLayer := newObjectLayerFn() + // Service not initialized yet + if objLayer == nil || globalIsGateway || GlobalKMS == nil { + return + } + + const ( + Online = 1 + Offline = 0 + ) + desc := MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: kmsSubsystem, + Name: kmsOnline, + Help: "Reports whether the KMS is online (1) or offline (0)", + Type: gaugeMetric, + } + _, err := GlobalKMS.Metrics(ctx) + if _, ok := kes.IsConnError(err); ok { + return []Metric{{ + Description: desc, + Value: float64(Offline), + }} + } + return []Metric{{ + Description: desc, + Value: float64(Online), + }} + }) + return mg +} + +func getKMSMetrics() *MetricsGroup { + mg := &MetricsGroup{ + cacheInterval: 10 * time.Second, + } + + mg.RegisterRead(func(ctx context.Context) []Metric { + objLayer := newObjectLayerFn() + // Service not initialized yet + if objLayer == nil || globalIsGateway || GlobalKMS == nil { + return []Metric{} + } + + metrics := make([]Metric, 0, 4) + metric, err := GlobalKMS.Metrics(ctx) + if err != nil { + return metrics + } + metrics = append(metrics, Metric{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: kmsSubsystem, + Name: kmsRequestsSuccess, + Help: "Number of KMS requests that succeeded", + Type: counterMetric, + }, + Value: float64(metric.RequestOK), + }) + metrics = append(metrics, Metric{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: kmsSubsystem, + Name: kmsRequestsError, + Help: "Number of KMS requests that failed due to some error. (HTTP 4xx status code)", + Type: counterMetric, + }, + Value: float64(metric.RequestErr), + }) + metrics = append(metrics, Metric{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: kmsSubsystem, + Name: kmsRequestsFail, + Help: "Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code)", + Type: counterMetric, + }, + Value: float64(metric.RequestFail), + }) + metrics = append(metrics, Metric{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: kmsSubsystem, + Name: kmsUptime, + Help: "The time the KMS has been up and running in seconds.", + Type: counterMetric, + }, + Value: metric.UpTime.Seconds(), + }) + + return metrics + }) + return mg +} + type minioClusterCollector struct { metricsGroups []*MetricsGroup desc *prometheus.Desc diff --git a/go.mod b/go.mod index 66349efe9..024d22dc8 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/minio/csvparser v1.0.0 github.com/minio/dperf v0.4.2 github.com/minio/highwayhash v1.0.2 - github.com/minio/kes v0.19.2 + github.com/minio/kes v0.20.0 github.com/minio/madmin-go v1.4.3 github.com/minio/minio-go/v7 v7.0.30 github.com/minio/pkg v1.1.26 @@ -65,7 +65,7 @@ require ( github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 github.com/pierrec/lz4 v2.6.1+incompatible github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.1 + github.com/prometheus/client_golang v1.12.2 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.34.0 github.com/prometheus/procfs v0.7.3 diff --git a/go.sum b/go.sum index 9532a6df5..53b58e561 100644 --- a/go.sum +++ b/go.sum @@ -618,8 +618,8 @@ github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEX github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= -github.com/minio/kes v0.19.2 h1:0kdMAgLMSkiDA33k8pMHC7d6erDuseuLrZF+N3017SM= -github.com/minio/kes v0.19.2/go.mod h1:X2fMkDbAkjbSKDGOQZvyPkHxoG7nuzP6R78Jw+TzXtM= +github.com/minio/kes v0.20.0 h1:1tyC51Rr8zTregTESuT/QN/iebNMX7B9t7d3xLNMEpE= +github.com/minio/kes v0.20.0/go.mod h1:3FW1BQkMGQW78yhy+69tUq5bdcf5rnXJizyeKB9a/tc= github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc= github.com/minio/madmin-go v1.4.3 h1:5/kBHjKTjYOQQHvyznu51weN5hJtFW67LB2VLz+hmzU= github.com/minio/madmin-go v1.4.3/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98= @@ -745,8 +745,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/internal/kms/kes.go b/internal/kms/kes.go index 21813b3b4..a7e3a9740 100644 --- a/internal/kms/kes.go +++ b/internal/kms/kes.go @@ -115,6 +115,10 @@ func (c *kesClient) Stat() (Status, error) { }, nil } +func (c *kesClient) Metrics(ctx context.Context) (kes.Metric, error) { + return c.client.Metrics(ctx) +} + // CreateKey tries to create a new key at the KMS with the // given key ID. // diff --git a/internal/kms/kms.go b/internal/kms/kms.go index 2978bc370..e59827194 100644 --- a/internal/kms/kms.go +++ b/internal/kms/kms.go @@ -23,6 +23,7 @@ import ( "encoding/json" jsoniter "github.com/json-iterator/go" + "github.com/minio/kes" ) // KMS is the generic interface that abstracts over @@ -31,6 +32,9 @@ type KMS interface { // Stat returns the current KMS status. Stat() (Status, error) + // Metrics returns a KMS metric snapshot. + Metrics(ctx context.Context) (kes.Metric, error) + // CreateKey creates a new key at the KMS with the given key ID. CreateKey(keyID string) error diff --git a/internal/kms/single-key.go b/internal/kms/single-key.go index aed31526d..b94ed29ef 100644 --- a/internal/kms/single-key.go +++ b/internal/kms/single-key.go @@ -33,6 +33,7 @@ import ( "golang.org/x/crypto/chacha20" "golang.org/x/crypto/chacha20poly1305" + "github.com/minio/kes" "github.com/minio/minio/internal/hash/sha256" ) @@ -89,6 +90,10 @@ func (kms secretKey) Stat() (Status, error) { }, nil } +func (secretKey) Metrics(ctx context.Context) (kes.Metric, error) { + return kes.Metric{}, errors.New("kms: metrics are not supported") +} + func (secretKey) CreateKey(string) error { return errors.New("kms: creating keys is not supported") }