prometheus: Add incoming requests metrics since last scrape (#14261)

Some users running MinIO claim that their system became slow. One 
way to investigate is to look at this Prometheus history of the number of
the requests reaching the server. The existing current S3 requests metric
is not enough because it can increase of the system really becomes slow, 
due to disk issues for example.
This commit is contained in:
Anis Elleuch 2022-02-08 01:30:14 +01:00 committed by GitHub
parent 362e14fa1a
commit 2ee337ead5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 26 additions and 0 deletions

View File

@ -284,6 +284,7 @@ type ServerHTTPAPIStats struct {
// including their average execution time.
type ServerHTTPStats struct {
S3RequestsInQueue int32 `json:"s3RequestsInQueue"`
S3RequestsIncoming uint64 `json:"s3RequestsIncoming"`
CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"`
TotalS3Requests ServerHTTPAPIStats `json:"totalS3Requests"`
TotalS3Errors ServerHTTPAPIStats `json:"totalS3Errors"`

View File

@ -235,6 +235,8 @@ func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
// maxClients throttles the S3 API calls
func maxClients(f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
globalHTTPStats.incS3RequestsIncoming()
if val := globalServiceFreeze.Load(); val != nil {
if unlock, ok := val.(chan struct{}); ok && unlock != nil {
// Wait until unfrozen.

View File

@ -143,6 +143,7 @@ type HTTPStats struct {
rejectedRequestsHeader uint64
rejectedRequestsInvalid uint64
s3RequestsInQueue int32
s3RequestsIncoming uint64
currentS3Requests HTTPAPIStats
totalS3Requests HTTPAPIStats
totalS3Errors HTTPAPIStats
@ -153,9 +154,15 @@ func (st *HTTPStats) addRequestsInQueue(i int32) {
atomic.AddInt32(&st.s3RequestsInQueue, i)
}
func (st *HTTPStats) incS3RequestsIncoming() {
// Golang automatically resets to zero if this overflows
atomic.AddUint64(&st.s3RequestsIncoming, 1)
}
// Converts http stats into struct to be sent back to the client.
func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
serverStats := ServerHTTPStats{}
serverStats.S3RequestsIncoming = atomic.SwapUint64(&st.s3RequestsIncoming, 0)
serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue)
serverStats.TotalS3RejectedAuth = atomic.LoadUint64(&st.rejectedRequestsAuth)
serverStats.TotalS3RejectedTime = atomic.LoadUint64(&st.rejectedRequestsTime)

View File

@ -137,6 +137,7 @@ const (
limitTotal MetricName = "limit_total"
missedTotal MetricName = "missed_total"
waitingTotal MetricName = "waiting_total"
incomingTotal MetricName = "incoming_total"
objectTotal MetricName = "object_total"
offlineTotal MetricName = "offline_total"
onlineTotal MetricName = "online_total"
@ -570,6 +571,16 @@ func getS3RequestsInQueueMD() MetricDescription {
}
}
func getIncomingS3RequestsMD() MetricDescription {
return MetricDescription{
Namespace: s3MetricNamespace,
Subsystem: requestsSubsystem,
Name: incomingTotal,
Help: "Volatile number of total incoming S3 requests",
Type: gaugeMetric,
}
}
func getS3RequestsTotalMD() MetricDescription {
return MetricDescription{
Namespace: s3MetricNamespace,
@ -1435,6 +1446,11 @@ func getHTTPMetrics() *MetricsGroup {
Description: getS3RequestsInQueueMD(),
Value: float64(httpStats.S3RequestsInQueue),
})
metrics = append(metrics, Metric{
Description: getIncomingS3RequestsMD(),
Value: float64(httpStats.S3RequestsIncoming),
})
for api, value := range httpStats.CurrentS3Requests.APIStats {
metrics = append(metrics, Metric{
Description: getS3RequestsInFlightMD(),