MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: add missing minio_s3_requests_total (#15070)

Browse Source 
PR #15052 caused a regression, add the missing metrics back.

Bonus:

- internode information should be only for distributed setups 
- update the dashboard to include 4xx and 5xx error panels.
This commit is contained in:
Harshavardhana 2022-06-11 00:50:31 -07:00 committed by GitHub
parent d76c508566
commit 7413045f0e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 223 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/generic-handlers.go
View File

@ -282,7 +282,9 @@ func setHTTPStatsHandler(h http.Handler) http.Handler {
r.Body = meteredRequest
h.ServeHTTP(meteredResponse, r)
if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
globalConnStats.incInputBytes(meteredRequest.BytesRead())
globalConnStats.incOutputBytes(meteredResponse.BytesWritten())
} else {

2
cmd/http-stats.go
View File

@ -199,6 +199,8 @@ func (st *HTTPStats) updateStats(api string, r *http.Request, w *logger.Response
return
}
st.totalS3Requests.Inc(api)
// Increment the prometheus http request response histogram with appropriate label
httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())

26
cmd/metrics-v2.go
View File

@ -1587,19 +1587,21 @@ func getNetworkMetrics() *MetricsGroup {
mg := &MetricsGroup{}
mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
metrics = make([]Metric, 0, 10)
metrics = append(metrics, Metric{
Description: getInternodeFailedRequests(),
Value: float64(loadAndResetRPCNetworkErrsCounter()),
})
connStats := globalConnStats.toServerConnStats()
metrics = append(metrics, Metric{
Description: getInterNodeSentBytesMD(),
Value: float64(connStats.TotalOutputBytes),
})
metrics = append(metrics, Metric{
Description: getInterNodeReceivedBytesMD(),
Value: float64(connStats.TotalInputBytes),
})
if globalIsDistErasure {
metrics = append(metrics, Metric{
Description: getInternodeFailedRequests(),
Value: float64(loadAndResetRPCNetworkErrsCounter()),
})
metrics = append(metrics, Metric{
Description: getInterNodeSentBytesMD(),
Value: float64(connStats.TotalOutputBytes),
})
metrics = append(metrics, Metric{
Description: getInterNodeReceivedBytesMD(),
Value: float64(connStats.TotalInputBytes),
})
}
metrics = append(metrics, Metric{
Description: getS3SentBytesMD(),
Value: float64(connStats.S3OutputBytes),

216
docs/metrics/prometheus/grafana/minio-dashboard.json
View File

@ -72,7 +72,7 @@
"gnetId": 13502,
"graphTooltip": 0,
"id": null,
"iteration": 1654578559812,
"iteration": 1654921222878,
"links": [
{
"icon": "external link",
@ -1711,6 +1711,198 @@
"alignLevel": null
}
},
{
"aliasColors": {
"S3 Errors": "light-red",
"S3 Requests": "light-green"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_PROMETHEUS}",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 22
},
"hiddenSeries": false,
"id": 86,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.2.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))",
"interval": "1m",
"intervalFactor": 2,
"legendFormat": "{{server,api}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "S3 API Request Error Rate (5xx)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:331",
"format": "none",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:332",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"S3 Errors": "light-red",
"S3 Requests": "light-green"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_PROMETHEUS}",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 22
},
"hiddenSeries": false,
"id": 88,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.2.1",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))",
"interval": "1m",
"intervalFactor": 2,
"legendFormat": "{{server,api}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "S3 API Request Error Rate (4xx)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:331",
"format": "none",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:332",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"10.13.1.25:9000 DELETE": "red",
@ -1734,7 +1926,7 @@
"h": 9,
"w": 12,
"x": 0,
"y": 22
"y": 32
},
"hiddenSeries": false,
"id": 17,
@ -1839,7 +2031,7 @@
"h": 9,
"w": 12,
"x": 12,
"y": 22
"y": 32
},
"hiddenSeries": false,
"id": 84,
@ -1947,7 +2139,7 @@
"h": 9,
"w": 12,
"x": 0,
"y": 31
"y": 41
},
"hiddenSeries": false,
"id": 77,
@ -2039,7 +2231,7 @@
"h": 9,
"w": 12,
"x": 12,
"y": 31
"y": 41
},
"hiddenSeries": false,
"id": 76,
@ -2131,7 +2323,7 @@
"h": 8,
"w": 12,
"x": 0,
"y": 40
"y": 50
},
"hiddenSeries": false,
"id": 74,
@ -2225,7 +2417,7 @@
"h": 8,
"w": 12,
"x": 12,
"y": 40
"y": 50
},
"hiddenSeries": false,
"id": 82,
@ -2330,7 +2522,7 @@
"h": 9,
"w": 12,
"x": 0,
"y": 48
"y": 58
},
"hiddenSeries": false,
"id": 11,
@ -2447,7 +2639,7 @@
"h": 9,
"w": 12,
"x": 12,
"y": 48
"y": 58
},
"hiddenSeries": false,
"id": 8,
@ -2542,7 +2734,7 @@
"h": 7,
"w": 24,
"x": 0,
"y": 57
"y": 67
},
"hiddenSeries": false,
"id": 73,
@ -2666,7 +2858,7 @@
]
},
"time": {
"from": "now-3h",
"from": "now-1h",
"to": "now"
},
"timepicker": {
@ -2696,5 +2888,5 @@
"timezone": "",
"title": "MinIO Dashboard",
"uid": "TgmJnqnnk",
"version": 8
"version": 10
}