mirror of
https://github.com/minio/minio.git
synced 2024-12-26 07:05:55 -05:00
4c8562bcec
Fix a regression in #19733 where TTFB metrics for all APIs except GetObject were removed in v2 and v3 metrics. This causes breakage for existing v2 metrics users. Instead we continue to send TTFB for all APIs in V2 but only send for GetObject in V3.
225 lines
9.4 KiB
Go
225 lines
9.4 KiB
Go
// Copyright (c) 2015-2024 MinIO, Inc.
|
|
//
|
|
// This file is part of MinIO Object Storage stack
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/minio/minio-go/v7/pkg/set"
|
|
)
|
|
|
|
const (
|
|
apiRejectedAuthTotal MetricName = "rejected_auth_total"
|
|
apiRejectedHeaderTotal MetricName = "rejected_header_total"
|
|
apiRejectedTimestampTotal MetricName = "rejected_timestamp_total"
|
|
apiRejectedInvalidTotal MetricName = "rejected_invalid_total"
|
|
|
|
apiRequestsWaitingTotal MetricName = "waiting_total"
|
|
apiRequestsIncomingTotal MetricName = "incoming_total"
|
|
|
|
apiRequestsInFlightTotal MetricName = "inflight_total"
|
|
apiRequestsTotal MetricName = "total"
|
|
apiRequestsErrorsTotal MetricName = "errors_total"
|
|
apiRequests5xxErrorsTotal MetricName = "5xx_errors_total"
|
|
apiRequests4xxErrorsTotal MetricName = "4xx_errors_total"
|
|
apiRequestsCanceledTotal MetricName = "canceled_total"
|
|
|
|
apiRequestsTTFBSecondsDistribution MetricName = "ttfb_seconds_distribution"
|
|
|
|
apiTrafficSentBytes MetricName = "traffic_sent_bytes"
|
|
apiTrafficRecvBytes MetricName = "traffic_received_bytes"
|
|
)
|
|
|
|
var (
|
|
apiRejectedAuthTotalMD = NewCounterMD(apiRejectedAuthTotal,
|
|
"Total number of requests rejected for auth failure", "type")
|
|
apiRejectedHeaderTotalMD = NewCounterMD(apiRejectedHeaderTotal,
|
|
"Total number of requests rejected for invalid header", "type")
|
|
apiRejectedTimestampTotalMD = NewCounterMD(apiRejectedTimestampTotal,
|
|
"Total number of requests rejected for invalid timestamp", "type")
|
|
apiRejectedInvalidTotalMD = NewCounterMD(apiRejectedInvalidTotal,
|
|
"Total number of invalid requests", "type")
|
|
|
|
apiRequestsWaitingTotalMD = NewGaugeMD(apiRequestsWaitingTotal,
|
|
"Total number of requests in the waiting queue", "type")
|
|
apiRequestsIncomingTotalMD = NewGaugeMD(apiRequestsIncomingTotal,
|
|
"Total number of incoming requests", "type")
|
|
|
|
apiRequestsInFlightTotalMD = NewGaugeMD(apiRequestsInFlightTotal,
|
|
"Total number of requests currently in flight", "name", "type")
|
|
apiRequestsTotalMD = NewCounterMD(apiRequestsTotal,
|
|
"Total number of requests", "name", "type")
|
|
apiRequestsErrorsTotalMD = NewCounterMD(apiRequestsErrorsTotal,
|
|
"Total number of requests with (4xx and 5xx) errors", "name", "type")
|
|
apiRequests5xxErrorsTotalMD = NewCounterMD(apiRequests5xxErrorsTotal,
|
|
"Total number of requests with 5xx errors", "name", "type")
|
|
apiRequests4xxErrorsTotalMD = NewCounterMD(apiRequests4xxErrorsTotal,
|
|
"Total number of requests with 4xx errors", "name", "type")
|
|
apiRequestsCanceledTotalMD = NewCounterMD(apiRequestsCanceledTotal,
|
|
"Total number of requests canceled by the client", "name", "type")
|
|
|
|
apiRequestsTTFBSecondsDistributionMD = NewCounterMD(apiRequestsTTFBSecondsDistribution,
|
|
"Distribution of time to first byte across API calls", "name", "type", "le")
|
|
|
|
apiTrafficSentBytesMD = NewCounterMD(apiTrafficSentBytes,
|
|
"Total number of bytes sent", "type")
|
|
apiTrafficRecvBytesMD = NewCounterMD(apiTrafficRecvBytes,
|
|
"Total number of bytes received", "type")
|
|
)
|
|
|
|
// loadAPIRequestsHTTPMetrics - reads S3 HTTP metrics.
|
|
//
|
|
// This is a `MetricsLoaderFn`.
|
|
//
|
|
// This includes node level S3 HTTP metrics.
|
|
//
|
|
// This function currently ignores `opts`.
|
|
func loadAPIRequestsHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
|
|
// Collect node level S3 HTTP metrics.
|
|
httpStats := globalHTTPStats.toServerHTTPStats(false)
|
|
|
|
// Currently we only collect S3 API related stats, so we set the "type"
|
|
// label to "s3".
|
|
|
|
m.Set(apiRejectedAuthTotal, float64(httpStats.TotalS3RejectedAuth), "type", "s3")
|
|
m.Set(apiRejectedTimestampTotal, float64(httpStats.TotalS3RejectedTime), "type", "s3")
|
|
m.Set(apiRejectedHeaderTotal, float64(httpStats.TotalS3RejectedHeader), "type", "s3")
|
|
m.Set(apiRejectedInvalidTotal, float64(httpStats.TotalS3RejectedInvalid), "type", "s3")
|
|
m.Set(apiRequestsWaitingTotal, float64(httpStats.S3RequestsInQueue), "type", "s3")
|
|
m.Set(apiRequestsIncomingTotal, float64(httpStats.S3RequestsIncoming), "type", "s3")
|
|
|
|
for name, value := range httpStats.CurrentS3Requests.APIStats {
|
|
m.Set(apiRequestsInFlightTotal, float64(value), "name", name, "type", "s3")
|
|
}
|
|
for name, value := range httpStats.TotalS3Requests.APIStats {
|
|
m.Set(apiRequestsTotal, float64(value), "name", name, "type", "s3")
|
|
}
|
|
for name, value := range httpStats.TotalS3Errors.APIStats {
|
|
m.Set(apiRequestsErrorsTotal, float64(value), "name", name, "type", "s3")
|
|
}
|
|
for name, value := range httpStats.TotalS35xxErrors.APIStats {
|
|
m.Set(apiRequests5xxErrorsTotal, float64(value), "name", name, "type", "s3")
|
|
}
|
|
for name, value := range httpStats.TotalS34xxErrors.APIStats {
|
|
m.Set(apiRequests4xxErrorsTotal, float64(value), "name", name, "type", "s3")
|
|
}
|
|
for name, value := range httpStats.TotalS3Canceled.APIStats {
|
|
m.Set(apiRequestsCanceledTotal, float64(value), "name", name, "type", "s3")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// loadAPIRequestsTTFBMetrics - loads S3 TTFB metrics.
|
|
//
|
|
// This is a `MetricsLoaderFn`.
|
|
func loadAPIRequestsTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
|
|
renameLabels := map[string]string{"api": "name"}
|
|
labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
|
|
m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, labelsFilter, renameLabels, nil,
|
|
"type", "s3")
|
|
return nil
|
|
}
|
|
|
|
// loadAPIRequestsNetworkMetrics - loads S3 network metrics.
|
|
//
|
|
// This is a `MetricsLoaderFn`.
|
|
func loadAPIRequestsNetworkMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
|
|
connStats := globalConnStats.toServerConnStats()
|
|
m.Set(apiTrafficSentBytes, float64(connStats.s3OutputBytes), "type", "s3")
|
|
m.Set(apiTrafficRecvBytes, float64(connStats.s3InputBytes), "type", "s3")
|
|
return nil
|
|
}
|
|
|
|
// Metric Descriptions for bucket level S3 metrics.
|
|
var (
|
|
bucketAPITrafficSentBytesMD = NewCounterMD(apiTrafficSentBytes,
|
|
"Total number of bytes received for a bucket", "bucket", "type")
|
|
bucketAPITrafficRecvBytesMD = NewCounterMD(apiTrafficRecvBytes,
|
|
"Total number of bytes sent for a bucket", "bucket", "type")
|
|
|
|
bucketAPIRequestsInFlightMD = NewGaugeMD(apiRequestsInFlightTotal,
|
|
"Total number of requests currently in flight for a bucket", "bucket", "name", "type")
|
|
bucketAPIRequestsTotalMD = NewCounterMD(apiRequestsTotal,
|
|
"Total number of requests for a bucket", "bucket", "name", "type")
|
|
bucketAPIRequestsCanceledMD = NewCounterMD(apiRequestsCanceledTotal,
|
|
"Total number of requests canceled by the client for a bucket", "bucket", "name", "type")
|
|
bucketAPIRequests4xxErrorsMD = NewCounterMD(apiRequests4xxErrorsTotal,
|
|
"Total number of requests with 4xx errors for a bucket", "bucket", "name", "type")
|
|
bucketAPIRequests5xxErrorsMD = NewCounterMD(apiRequests5xxErrorsTotal,
|
|
"Total number of requests with 5xx errors for a bucket", "bucket", "name", "type")
|
|
|
|
bucketAPIRequestsTTFBSecondsDistributionMD = NewCounterMD(apiRequestsTTFBSecondsDistribution,
|
|
"Distribution of time to first byte across API calls for a bucket",
|
|
"bucket", "name", "le", "type")
|
|
)
|
|
|
|
// loadBucketAPIHTTPMetrics - loads bucket level S3 HTTP metrics.
|
|
//
|
|
// This is a `MetricsLoaderFn`.
|
|
//
|
|
// This includes bucket level S3 HTTP metrics and S3 network in/out metrics.
|
|
func loadBucketAPIHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
|
|
if len(buckets) == 0 {
|
|
return nil
|
|
}
|
|
for bucket, inOut := range globalBucketConnStats.getBucketS3InOutBytes(buckets) {
|
|
recvBytes := inOut.In
|
|
if recvBytes > 0 {
|
|
m.Set(apiTrafficSentBytes, float64(recvBytes), "bucket", bucket, "type", "s3")
|
|
}
|
|
sentBytes := inOut.Out
|
|
if sentBytes > 0 {
|
|
m.Set(apiTrafficRecvBytes, float64(sentBytes), "bucket", bucket, "type", "s3")
|
|
}
|
|
|
|
httpStats := globalBucketHTTPStats.load(bucket)
|
|
for k, v := range httpStats.currentS3Requests.Load(false) {
|
|
m.Set(apiRequestsInFlightTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
|
}
|
|
|
|
for k, v := range httpStats.totalS3Requests.Load(false) {
|
|
m.Set(apiRequestsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
|
}
|
|
|
|
for k, v := range httpStats.totalS3Canceled.Load(false) {
|
|
m.Set(apiRequestsCanceledTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
|
}
|
|
|
|
for k, v := range httpStats.totalS34xxErrors.Load(false) {
|
|
m.Set(apiRequests4xxErrorsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
|
}
|
|
|
|
for k, v := range httpStats.totalS35xxErrors.Load(false) {
|
|
m.Set(apiRequests5xxErrorsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// loadBucketAPITTFBMetrics - loads bucket S3 TTFB metrics.
|
|
//
|
|
// This is a `MetricsLoaderFn`.
|
|
func loadBucketAPITTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
|
|
renameLabels := map[string]string{"api": "name"}
|
|
labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
|
|
m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, labelsFilter, renameLabels,
|
|
buckets, "type", "s3")
|
|
return nil
|
|
}
|