diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 60fb74e96..3dea1b628 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -2286,6 +2286,7 @@ type HealthResult struct { ESHealth []struct { Maintenance bool PoolID, SetID int + Healthy bool HealthyDrives int HealingDrives int ReadQuorum int @@ -2409,23 +2410,25 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea result.ESHealth = append(result.ESHealth, struct { Maintenance bool PoolID, SetID int + Healthy bool HealthyDrives, HealingDrives int ReadQuorum, WriteQuorum int }{ Maintenance: opts.Maintenance, SetID: setIdx, PoolID: poolIdx, + Healthy: erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx], HealthyDrives: erasureSetUpCount[poolIdx][setIdx].online, HealingDrives: erasureSetUpCount[poolIdx][setIdx].healing, ReadQuorum: poolReadQuorums[poolIdx], WriteQuorum: poolWriteQuorums[poolIdx], }) - if erasureSetUpCount[poolIdx][setIdx].online < poolWriteQuorums[poolIdx] { + result.Healthy = erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx] + if !result.Healthy { logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d", poolIdx, setIdx, poolWriteQuorums[poolIdx])) - result.Healthy = false } } } diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index c2f1aa7e8..052bdc407 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -60,13 +60,13 @@ func init() { getClusterHealthMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}), getReplicationSiteMetrics(MetricsGroupOpts{dependGlobalSiteReplicationSys: true}), + getBatchJobsMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), } peerMetricsGroups = []*MetricsGroup{ getGoMetrics(), getHTTPMetrics(MetricsGroupOpts{}), getNotificationMetrics(MetricsGroupOpts{dependGlobalLambdaTargetList: true}), - getLocalStorageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), getMinioProcMetrics(), getMinioVersionMetrics(), getNetworkMetrics(), @@ -77,7 +77,8 @@ func init() { getKMSNodeMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependGlobalKMS: true}), getMinioHealingMetrics(MetricsGroupOpts{dependGlobalBackgroundHealState: true}), getWebhookMetrics(), - getReplicationClusterMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependBucketTargetSys: true}), + getTierMetrics(), + getReplicationNodeMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependBucketTargetSys: true}), } allMetricsGroups := func() (allMetrics []*MetricsGroup) { @@ -97,13 +98,13 @@ func init() { getDistLockMetrics(MetricsGroupOpts{dependGlobalIsDistErasure: true, dependGlobalLockServer: true}), getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}), getLocalStorageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), + getReplicationNodeMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependBucketTargetSys: true}), } bucketMetricsGroups := []*MetricsGroup{ getBucketUsageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), getHTTPMetrics(MetricsGroupOpts{bucketOnly: true}), getBucketTTFBMetric(), - getBatchJobsMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), } bucketPeerMetricsGroups = []*MetricsGroup{ @@ -2137,7 +2138,7 @@ func getIAMNodeMetrics(opts MetricsGroupOpts) *MetricsGroup { } // replication metrics for each node - published to the cluster endpoint with nodename as label -func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup { +func getReplicationNodeMetrics(opts MetricsGroupOpts) *MetricsGroup { mg := &MetricsGroup{ cacheInterval: 1 * time.Minute, metricsGroupOpts: opts, @@ -3375,6 +3376,16 @@ func getClusterHealthStatusMD() MetricDescription { } } +func getClusterErasureSetHealthStatusMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: "health", + Name: "erasure_set_status", + Help: "Get current health status for this erasure set", + Type: gaugeMetric, + } +} + func getClusterErasureSetReadQuorumMD() MetricDescription { return MetricDescription{ Namespace: clusterMetricNamespace, @@ -3468,6 +3479,17 @@ func getClusterHealthMetrics(opts MetricsGroupOpts) *MetricsGroup { VariableLabels: labels, Value: float64(h.HealingDrives), }) + + health := 1 + if !h.Healthy { + health = 0 + } + + metrics = append(metrics, Metric{ + Description: getClusterErasureSetHealthStatusMD(), + VariableLabels: labels, + Value: float64(health), + }) } return diff --git a/docs/metrics/prometheus/alerts.md b/docs/metrics/prometheus/alerts.md index f898c4850..8ab9f6231 100644 --- a/docs/metrics/prometheus/alerts.md +++ b/docs/metrics/prometheus/alerts.md @@ -51,13 +51,13 @@ groups: - name: example rules: - alert: MinIOClusterTolerance - expr: minio_cluster_health_erasure_set_tolerance <= 0 + expr: minio_cluster_health_erasure_set_status < 1 for: 5m labels: severity: critical annotations: - summary: "Instance {{ $labels.server }} unable to tolerate node failures" - description: "MinIO instance {{ $labels.server }} of job {{ $labels.job }} has tolerance <=0 for more than 5 minutes." + summary: "Instance {{ $labels.server }} has lost quorum on pool {{ $labels.pool }} on set {{ $labels.set }}" + description: "MinIO instance {{ $labels.server }} of job {{ $labels.job }} has lost quorum on pool {{ $labels.pool }} on set {{ $labels.set }} for more than 5 minutes." ``` ## Verify the configuration and alerts @@ -65,7 +65,7 @@ To verify the above sample alert follow below steps 1. Start a distributed MinIO instance (4 nodes setup) 2. Start Prometheus server and AlertManager -3. Bring down couple of MinIO instances to bring down the Erasure Set tolerance to -1 and verify the same with `mc admin prometheus metrics ALIAS | grep minio_cluster_health_erasure_set_tolerance` +3. Bring down couple of MinIO instances to bring down the Erasure Set tolerance to -1 and verify the same with `mc admin prometheus metrics ALIAS | grep minio_cluster_health_erasure_set_status` 4. Wait for 5 mins (as alert is configured to be firing after 5 mins), and verify that you see an entry in webhook for the alert as well as in Prometheus console as shown below ```json @@ -90,7 +90,7 @@ To verify the above sample alert follow below steps }, "startsAt": "2023-11-18T06:20:09.456Z", "endsAt": "0001-01-01T00:00:00Z", - "generatorURL": "http://fedora-shubhendu:9090/graph?g0.expr=minio_cluster_health_erasure_set_tolerance+%3C%3D+0&g0.tab=1", + "generatorURL": "http://fedora-minio:9090/graph?g0.expr=minio_cluster_health_erasure_set_tolerance+%3C%3D+0&g0.tab=1", "fingerprint": "2255608b0da28ca3" } ], @@ -107,10 +107,10 @@ To verify the above sample alert follow below steps "severity": "critical" }, "commonAnnotations": { - "description": "MinIO instance 127.0.0.1:9000 of job minio-job has tolerance <=0 for more than 5 minutes.", - "summary": "Instance 127.0.0.1:9000 unable to tolerate node failures" + "description": "MinIO instance 127.0.0.1:9000 of job minio-job has lost quorum on pool 0 on set 0 for more than 5 minutes.", + "summary": "Instance 127.0.0.1:9000 has lot quorum on pool 0 on set 0" }, - "externalURL": "http://fedora-shubhendu:9093", + "externalURL": "http://fedora-minio:9093", "version": "4", "groupKey": "{}:{alertname=\"MinIOClusterTolerance\"}", "truncatedAlerts": 0 diff --git a/docs/metrics/prometheus/grafana/grafana-minio.png b/docs/metrics/prometheus/grafana/grafana-minio.png index 59ca432db..d1c56cef3 100644 Binary files a/docs/metrics/prometheus/grafana/grafana-minio.png and b/docs/metrics/prometheus/grafana/grafana-minio.png differ diff --git a/docs/metrics/prometheus/grafana/minio-dashboard.json b/docs/metrics/prometheus/grafana/minio-dashboard.json index a3d8b7681..3efc1afab 100644 --- a/docs/metrics/prometheus/grafana/minio-dashboard.json +++ b/docs/metrics/prometheus/grafana/minio-dashboard.json @@ -17,11 +17,17 @@ "name": "Bar gauge", "version": "" }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "10.0.2" + "version": "10.3.1" }, { "type": "panel", @@ -123,7 +129,8 @@ } ] }, - "unit": "dtdurations" + "unit": "dtdurations", + "unitScale": true }, "overrides": [] }, @@ -148,10 +155,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -201,7 +210,8 @@ } ] }, - "unit": "bytes" + "unit": "bytes", + "unitScale": true }, "overrides": [] }, @@ -226,10 +236,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -249,7 +261,7 @@ "step": 60 } ], - "title": "Total S3 Traffic Inbound", + "title": "Total S3 Ingress", "type": "stat" }, { @@ -281,7 +293,8 @@ "type": "special" } ], - "unit": "bytes" + "unit": "bytes", + "unitScale": true }, "overrides": [ { @@ -393,6 +406,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -406,6 +420,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -436,7 +451,8 @@ } ] }, - "unit": "bytes" + "unit": "bytes", + "unitScale": true }, "overrides": [ { @@ -525,6 +541,9 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, "mappings": [], "thresholds": { "mode": "absolute", @@ -532,13 +551,10 @@ { "color": "green", "value": null - }, - { - "color": "semi-dark-red", - "value": 80 } ] - } + }, + "unitScale": true }, "overrides": [] }, @@ -551,41 +567,47 @@ "id": 52, "links": [], "options": { - "displayMode": "basic", - "minVizHeight": 10, - "minVizWidth": 0, + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], "fields": "", "values": false }, - "showUnfilled": false, - "text": {}, + "showUnfilled": true, + "sizing": "auto", "valueMode": "color" }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "disableTextWrap": false, "editorMode": "code", "exemplar": true, - "expr": "max(minio_cluster_objects_size_distribution{job=~\"$scrape_jobs\"})", + "expr": "minio_cluster_objects_size_distribution{job=~\"$scrape_jobs\"}", "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{range}}", "refId": "A", - "step": 300 + "step": 300, + "useBackend": false } ], - "title": "Object size distribution", + "title": "Object Size Distribution", "type": "bargauge" }, { @@ -596,17 +618,42 @@ "description": "", "fieldConfig": { "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } - ], + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -616,11 +663,11 @@ }, { "color": "red", - "value": 2000 + "value": 80 } ] }, - "unit": "short" + "unitScale": true }, "overrides": [] }, @@ -634,42 +681,40 @@ "links": [], "maxDataPoints": 100, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false }, - "text": {}, - "textMode": "auto" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum (minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"})", - "format": "table", + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "", + "legendFormat": "{{server}}", "metric": "process_start_time_seconds", "refId": "A", "step": 60 } ], - "title": "Total Open FDs", - "type": "stat" + "title": "Open FDs ", + "type": "timeseries" }, { "datasource": { @@ -699,7 +744,8 @@ } ] }, - "unit": "bytes" + "unit": "bytes", + "unitScale": true }, "overrides": [] }, @@ -724,10 +770,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -747,7 +795,7 @@ "step": 60 } ], - "title": "Total S3 Traffic Outbound", + "title": "Total S3 Egress", "type": "stat" }, { @@ -756,6 +804,266 @@ "uid": "${DS_PROMETHEUS}" }, "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 3 + }, + "id": 62, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_node_go_routine_total{job=~\"$scrape_jobs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_on_off", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 94, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_health_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Pool: {{pool}} Set: {{set}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cluster Health Status", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 78, + "links": [], + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_online_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".", + "metric": "process_start_time_seconds", + "range": false, + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_offline_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": ".", + "range": false, + "refId": "B" + } + ], + "title": "Total Online/Offline Drives", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "mappings": [ @@ -777,40 +1085,148 @@ "value": null }, { - "color": "red", - "value": 2000 + "color": "dark-yellow", + "value": 75000000 + }, + { + "color": "dark-red", + "value": 100000000 } ] }, - "unit": "short" + "unit": "short", + "unitScale": true }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, - "x": 21, - "y": 3 + "x": 6, + "y": 6 }, - "id": 62, + "id": 66, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", - "orientation": "auto", + "orientation": "horizontal", "reduceOptions": { "calcs": [ - "last" + "lastNotNull" ], "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_bucket_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Buckets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 6 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -818,20 +1234,113 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum without (server,instance) (minio_node_go_routine_total{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 + "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Received [{{server}}]", + "refId": "A" } ], - "title": "Total Goroutines", - "type": "stat" + "title": "S3 API Ingress Rate ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Sent [{{server}}]", + "refId": "A" + } + ], + "title": "S3 API Egress Rate ", + "type": "timeseries" }, { "datasource": { @@ -865,7 +1374,8 @@ } ] }, - "unit": "short" + "unit": "short", + "unitScale": true }, "overrides": [] }, @@ -873,7 +1383,7 @@ "h": 2, "w": 3, "x": 0, - "y": 6 + "y": 8 }, "id": 53, "links": [], @@ -890,10 +1400,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -916,89 +1428,6 @@ "title": "Total Online Servers", "type": "stat" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 3, - "y": 6 - }, - "id": 9, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "10.0.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "max(minio_cluster_drive_online_total{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Total online drives in MinIO Cluster", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Total Online Drives", - "type": "stat" - }, { "datasource": { "type": "prometheus", @@ -1034,441 +1463,8 @@ } ] }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 6 - }, - "id": 66, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "10.0.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(minio_cluster_bucket_total{job=~\"$scrape_jobs\"})", - "format": "time_series", - "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Number of Buckets", - "type": "stat" - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 7, - "x": 9, - "y": 6 - }, - "hiddenSeries": false, - "id": 63, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "Data Received [{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Data Received Rate ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 6 - }, - "hiddenSeries": false, - "id": 70, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "Data Sent [{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Data Sent Rate ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 8 - }, - "id": 69, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "10.0.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "max(minio_cluster_nodes_offline_total{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Total Offline Servers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 3, - "y": 8 - }, - "id": 78, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "10.0.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "max(minio_cluster_drive_offline_total{job=~\"$scrape_jobs\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 - } - ], - "title": "Total Offline Drives", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "dark-yellow", - "value": 75000000 - }, - { - "color": "dark-red", - "value": 100000000 - } - ] - }, - "unit": "short" + "unit": "short", + "unitScale": true }, "overrides": [] }, @@ -1493,10 +1489,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -1538,7 +1536,8 @@ } ] }, - "unit": "ns" + "unit": "ns", + "unitScale": true }, "overrides": [] }, @@ -1563,10 +1562,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -1585,7 +1586,7 @@ "step": 60 } ], - "title": "Time Since Last Heal Activity", + "title": "Time Since Last Heal", "type": "stat" }, { @@ -1609,7 +1610,8 @@ } ] }, - "unit": "ns" + "unit": "ns", + "unitScale": true }, "overrides": [] }, @@ -1634,10 +1636,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -1656,7 +1660,7 @@ "step": 60 } ], - "title": "Time Since Last Scan Activity", + "title": "Time Since Last Scan", "type": "stat" }, { @@ -1671,11 +1675,17 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 10, - "w": 10, + "h": 6, + "w": 9, "x": 0, "y": 12 }, @@ -1697,7 +1707,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -1763,12 +1773,18 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 10, + "h": 6, "w": 7, - "x": 10, + "x": 9, "y": 12 }, "hiddenSeries": false, @@ -1789,7 +1805,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -1813,7 +1829,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "S3 API Request 4xx Error Rate", + "title": "S3 API Request Error Rate (4xx)", "tooltip": { "shared": true, "sort": 0, @@ -1855,12 +1871,18 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 10, - "w": 7, - "x": 17, + "h": 6, + "w": 8, + "x": 16, "y": 12 }, "hiddenSeries": false, @@ -1881,7 +1903,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -1905,7 +1927,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "S3 API Request 5xx Error Rate", + "title": "S3 API Request Error Rate (5xx)", "tooltip": { "shared": true, "sort": 0, @@ -1936,11 +1958,222 @@ } }, { - "aliasColors": { - "10.13.1.25:9000 DELETE": "red", - "10.13.1.25:9000 GET": "green", - "10.13.1.25:9000 POST": "blue" + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 99, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_online_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Online Drives", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_read_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Read Quorum", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_write_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Write Quorum", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_healing_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Healing Drives", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Status", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Health Breakdown", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 76, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "minio_node_process_resident_memory_bytes{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage ", + "type": "bargauge" + }, + { + "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, @@ -1948,23 +2181,320 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Total number of bytes received and sent among all MinIO server instances", "fieldConfig": { "defaults": { - "links": [] + "unitScale": true }, "overrides": [] }, - "fill": 10, - "fillGradient": 1, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, + "h": 7, "w": 12, "x": 0, - "y": 22 + "y": 26 }, "hiddenSeries": false, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_io_rchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Node RChar [{{server}}]", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_io_wchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Node WChar [{{server}}]", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Read, Write I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:381", + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:382", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 77, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_node_process_cpu_total_seconds{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of bytes received and sent on MinIO cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, "id": 17, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Internode Bytes Received [{{server}}]", + "metric": "minio_http_requests_duration_seconds_count", + "range": true, + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Internode Bytes Sent [{{server}}]", + "refId": "B" + } + ], + "title": "Internode Traffic", + "type": "timeseries" + }, + { + "aliasColors": { + "available 10.13.1.25:9000": "green", + "used 10.13.1.25:9000": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [], + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 8, "legend": { "avg": false, "current": false, @@ -1983,8 +2513,8 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 5, + "pluginVersion": "10.3.1", + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -1998,30 +2528,15 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "rate(minio_inter_node_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "format": "time_series", + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", "interval": "", - "intervalFactor": 2, - "legendFormat": "Internode Bytes Received [{{server}}]", - "metric": "minio_http_requests_duration_seconds_count", - "refId": "A", - "step": 4 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "Internode Bytes Sent [{{server}}]", + "legendFormat": "Open FDs [{{server}}]", "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Internode Data Transfer", + "title": "File Descriptors", "tooltip": { "shared": true, "sort": 0, @@ -2034,484 +2549,18 @@ "values": [] }, "yaxes": [ - { - "$$hashKey": "object:211", - "format": "bytes", - "logBase": 1, - "show": true - }, { "$$hashKey": "object:212", - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 22 - }, - "hiddenSeries": false, - "id": 84, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (instance) (minio_heal_objects_heal_total{job=~\"$scrape_jobs\"})", - "interval": "", - "legendFormat": "Objects healed in current self heal run", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (instance) (minio_heal_objects_error_total{job=~\"$scrape_jobs\"})", - "hide": false, - "interval": "", - "legendFormat": "Heal errors in current self heal run", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (instance) (minio_heal_objects_total{job=~\"$scrape_jobs\"}) ", - "hide": false, - "interval": "", - "legendFormat": "Objects scanned in current self heal run", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Healing", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:846", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:847", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 32 - }, - "hiddenSeries": false, - "id": 77, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_process_cpu_total_seconds{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "CPU Usage Rate [{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Node CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1043", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:1044", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 32 - }, - "hiddenSeries": false, - "id": 76, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_process_resident_memory_bytes{job=~\"$scrape_jobs\"}", - "interval": "", - "legendFormat": "Memory Used [{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Node Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1043", "format": "bytes", "logBase": 1, + "min": "0", "show": true }, { - "$$hashKey": "object:1044", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "hiddenSeries": false, - "id": 74, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_used_bytes{job=~\"$scrape_jobs\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Used Capacity [{{server}}:{{drive}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drive Used Capacity", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:381", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:382", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 41 - }, - "hiddenSeries": false, - "id": 82, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_free_inodes{job=~\"$scrape_jobs\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Free Inodes [{{server}}:{{drive}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drives Free Inodes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:381", + "$$hashKey": "object:213", "format": "none", "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:382", - "format": "short", - "logBase": 1, + "min": "0", "show": true } ], @@ -2534,17 +2583,18 @@ "description": "Number of online drives per MinIO Server", "fieldConfig": { "defaults": { - "links": [] + "links": [], + "unitScale": true }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 9, + "h": 7, "w": 12, "x": 0, - "y": 49 + "y": 40 }, "hiddenSeries": false, "id": 11, @@ -2566,7 +2616,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -2604,7 +2654,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Node Syscalls", + "title": "Syscalls", "tooltip": { "shared": true, "sort": 0, @@ -2637,10 +2687,7 @@ } }, { - "aliasColors": { - "available 10.13.1.25:9000": "green", - "used 10.13.1.25:9000": "blue" - }, + "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, @@ -2648,215 +2695,22 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { - "links": [] + "unitScale": true }, "overrides": [] }, "fill": 1, "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", - "interval": "", - "legendFormat": "Open FDs [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Node File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 58 - }, - "hiddenSeries": false, - "id": 73, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_io_rchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Node RChar [{{server}}]", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "rate(minio_node_io_wchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "Node WChar [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Node IO", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:381", - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:382", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 58 + "y": 40 }, "hiddenSeries": false, - "id": 74, + "id": 95, "legend": { "avg": false, "current": false, @@ -2873,7 +2727,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -2929,6 +2783,7 @@ } }, { + "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, @@ -2936,13 +2791,19 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 67 + "y": 47 }, "hiddenSeries": false, "id": 75, @@ -2962,7 +2823,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -3018,6 +2879,7 @@ } }, { + "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, @@ -3025,16 +2887,22 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 67 + "y": 47 }, "hiddenSeries": false, - "id": 76, + "id": 96, "legend": { "avg": false, "current": false, @@ -3051,7 +2919,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -3107,6 +2975,90 @@ } }, { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "dtdurations", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 54 + }, + "id": 89, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_cluster_kms_uptime{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "minio_cluster_kms_uptime", + "refId": "A", + "step": 60 + } + ], + "title": "KMS Uptime", + "type": "stat" + }, + { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, "bars": false, "dashLength": 10, "dashes": false, @@ -3114,16 +3066,22 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 74 + "h": 4, + "w": 4, + "x": 4, + "y": 54 }, "hiddenSeries": false, - "id": 77, + "id": 91, "legend": { "avg": false, "current": false, @@ -3140,7 +3098,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -3155,15 +3113,111 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "rate(minio_node_scanner_bucket_scans_started{job=~\"$scrape_jobs\"}[$__rate_interval])", + "expr": "sum by (server) (increase(minio_cluster_kms_request_error{job=~\"$scrape_jobs\"}[$__rate_interval]))", "interval": "1m", - "legendFormat": "[{{server}}]", + "intervalFactor": 2, + "legendFormat": "{{server}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Bucket Scans Started", + "title": "KMS Request 4xx Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:331", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:332", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "bool_on_off", + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 54 + }, + "hiddenSeries": false, + "id": 90, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_cluster_kms_online{job=~\"$scrape_jobs\"})", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "KMS Online(1)/Offline(0)", "tooltip": { "shared": true, "sort": 0, @@ -3178,7 +3232,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "none", + "format": "bool_on_off", "logBase": 1, "min": "0", "show": true @@ -3196,6 +3250,7 @@ } }, { + "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, @@ -3203,16 +3258,22 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 74 + "y": 54 }, "hiddenSeries": false, - "id": 78, + "id": 98, "legend": { "avg": false, "current": false, @@ -3229,7 +3290,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -3285,6 +3346,10 @@ } }, { + "aliasColors": { + "S3 Errors": "light-red", + "S3 Requests": "light-green" + }, "bars": false, "dashLength": 10, "dashes": false, @@ -3292,1166 +3357,19 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 81 - }, - "hiddenSeries": false, - "id": 79, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_free_bytes{job=~\"$scrape_jobs\"}", - "interval": "1m", - "legendFormat": "[{{server}}:{{drive}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drive Free Capacity", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 81 - }, - "hiddenSeries": false, - "id": 80, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_total_bytes{job=~\"$scrape_jobs\"}", - "interval": "1m", - "legendFormat": "[{{server}}:{{drive}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drive Total Capacity", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 88 - }, - "hiddenSeries": false, - "id": 81, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_used_bytes{job=~\"$scrape_jobs\"}", - "interval": "1m", - "legendFormat": "[{{server}}:{{drive}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drive Used Capacity", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 88 - }, - "hiddenSeries": false, - "id": 82, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_free_inodes{job=~\"$scrape_jobs\"}", - "interval": "1m", - "legendFormat": "[{{server}}:{{drive}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drive Free Inodes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 95 - }, - "hiddenSeries": false, - "id": 83, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server, drive, api) (minio_node_drive_latency_us{job=~\"$scrape_jobs\"})", - "interval": "1m", - "legendFormat": "[{{server}}:{{drive}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drive Latency (Micro Sec)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 95 - }, - "hiddenSeries": false, - "id": 84, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_total{job=~\"$scrape_jobs\"}", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drives Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 4, - "x": 16, - "y": 95 - }, - "hiddenSeries": false, - "id": 85, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prom${DS_PROMETHEUS}etheus" - }, - "exemplar": true, - "expr": "minio_node_drive_online_total{job=~\"$scrape_jobs\"}", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Online Drives", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 95 - }, - "hiddenSeries": false, - "id": 86, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_offline_total{job=~\"$scrape_jobs\"}", - "interval": "1m", - "legendFormat": "[{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Offline Drives", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 102 - }, - "hiddenSeries": false, - "id": 87, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_errors_availability{job=~\"$scrape_jobs\"}", - "interval": "1m", - "legendFormat": "[{{server}}:{{drive}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drive Availability Errors (since start)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 102 - }, - "hiddenSeries": false, - "id": 88, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_node_drive_errors_timeout{job=~\"$scrape_jobs\"}", - "interval": "1m", - "legendFormat": "[{{server}}:{{drive}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drive Timeout Errors (since start)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", "fieldConfig": { "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "dtdurations" + "unitScale": true }, "overrides": [] }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 109 - }, - "id": 89, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "10.0.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "minio_cluster_kms_uptime{job=~\"$scrape_jobs\"}", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "metric": "minio_cluster_kms_uptime", - "refId": "A", - "step": 60 - } - ], - "title": "KMS Uptime", - "type": "stat" - }, - { - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 109 - }, - "hiddenSeries": false, - "id": 90, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum by (server) (minio_cluster_kms_online{job=~\"$scrape_jobs\"})", - "interval": "1m", - "legendFormat": "{{server}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Online(1)/Offline(0)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 109 - }, - "hiddenSeries": false, - "id": 91, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum by (server) (increase(minio_cluster_kms_request_error{job=~\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "KMS Request 4xx Error Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 5, "w": 6, - "x": 12, - "y": 109 + "x": 0, + "y": 58 }, "hiddenSeries": false, "id": 92, @@ -4471,7 +3389,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -4537,13 +3455,19 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 5, "w": 6, - "x": 18, - "y": 109 + "x": 6, + "y": 58 }, "hiddenSeries": false, "id": 93, @@ -4563,7 +3487,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -4626,16 +3550,22 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 9, "w": 12, - "x": 0, - "y": 115 + "x": 12, + "y": 61 }, "hiddenSeries": false, - "id": 94, + "id": 97, "legend": { "avg": false, "current": false, @@ -4652,7 +3582,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.2", + "pluginVersion": "10.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -4667,15 +3597,15 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "minio_cluster_health_erasure_set_tolerance{job=~\"$scrape_jobs\"}", - "interval": "", - "legendFormat": "Pool: {{pool}} Set: {{set}}", + "expr": "rate(minio_node_scanner_bucket_scans_started{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Erasure Set Tolerance", + "title": "Bucket Scans Started", "tooltip": { "shared": true, "sort": 0, @@ -4689,15 +3619,17 @@ }, "yaxes": [ { - "$$hashKey": "object:1043", + "$$hashKey": "object:212", "format": "none", "logBase": 1, + "min": "0", "show": true }, { - "$$hashKey": "object:1044", - "format": "short", + "$$hashKey": "object:213", + "format": "none", "logBase": 1, + "min": "0", "show": true } ], @@ -4707,8 +3639,7 @@ } ], "refresh": "", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [ "minio" ], @@ -4739,7 +3670,7 @@ ] }, "time": { - "from": "now-1h", + "from": "now-6h", "to": "now" }, "timepicker": { @@ -4769,6 +3700,6 @@ "timezone": "", "title": "MinIO Dashboard", "uid": "TgmJnqnnk", - "version": 1, + "version": 54, "weekStart": "" -} +} \ No newline at end of file diff --git a/docs/metrics/prometheus/list.md b/docs/metrics/prometheus/list.md index 0591996ca..2fc7f6429 100644 --- a/docs/metrics/prometheus/list.md +++ b/docs/metrics/prometheus/list.md @@ -35,7 +35,7 @@ For deployments behind a load balancer, use the load balancer hostname instead o | `minio_cluster_usage_version_total` | Total number of versions (includes delete marker) in a cluster | | `minio_cluster_usage_deletemarker_total` | Total number of delete markers in a cluster | | `minio_cluster_usage_total_bytes` | Total cluster usage in bytes | -| `minio_cluster_buckets_total` | Total number of buckets in the cluster | +| `minio_cluster_bucket_total` | Total number of buckets in the cluster | ## Cluster Drive Metrics