The prometheus metrics refractoring (#8003)

The measures are consolidated to the following metrics

- `disk_storage_used` : Disk space used by the disk.
- `disk_storage_available`: Available disk space left on the disk.
- `disk_storage_total`: Total disk space on the disk.
- `disks_offline`: Total number of offline disks in current MinIO instance.
- `disks_total`: Total number of disks in current MinIO instance.
- `s3_requests_total`: Total number of s3 requests in current MinIO instance.
- `s3_errors_total`: Total number of errors in s3 requests in current MinIO instance.
- `s3_requests_current`: Total number of active s3 requests in current MinIO instance.
- `internode_rx_bytes_total`: Total number of internode bytes received by current MinIO server instance.
- `internode_tx_bytes_total`: Total number of bytes sent to the other nodes by current MinIO server instance.
- `s3_rx_bytes_total`: Total number of s3 bytes received by current MinIO server instance.
- `s3_tx_bytes_total`: Total number of s3 bytes sent by current MinIO server instance.
- `minio_version_info`: Current MinIO version with commit-id.
- `s3_ttfb_seconds_bucket`: Histogram that holds the latency information of the requests.

And this PR also modifies the current StorageInfo queries

- Decouples StorageInfo from ServerInfo .
- StorageInfo is enhanced to give endpoint information.

NOTE: ADMIN API VERSION IS BUMPED UP IN THIS PR

Fixes #7873

cmd/admin-handlers.go

@@ -229,37 +229,30 @@ type ServerConnStats struct {
 	TotalInputBytes  uint64 `json:"transferred"`
 	TotalOutputBytes uint64 `json:"received"`
 	Throughput       uint64 `json:"throughput,omitempty"`
+	S3InputBytes     uint64 `json:"transferredS3"`
+	S3OutputBytes    uint64 `json:"receivedS3"`
 }
 
-// ServerHTTPMethodStats holds total number of HTTP operations from/to the server,
+// ServerHTTPAPIStats holds total number of HTTP operations from/to the server,
 // including the average duration the call was spent.
-type ServerHTTPMethodStats struct {
-	Count       uint64 `json:"count"`
-	AvgDuration string `json:"avgDuration"`
+type ServerHTTPAPIStats struct {
+	APIStats map[string]int `json:"apiStats"`
 }
 
 // ServerHTTPStats holds all type of http operations performed to/from the server
 // including their average execution time.
 type ServerHTTPStats struct {
-	TotalHEADStats     ServerHTTPMethodStats `json:"totalHEADs"`
-	SuccessHEADStats   ServerHTTPMethodStats `json:"successHEADs"`
-	TotalGETStats      ServerHTTPMethodStats `json:"totalGETs"`
-	SuccessGETStats    ServerHTTPMethodStats `json:"successGETs"`
-	TotalPUTStats      ServerHTTPMethodStats `json:"totalPUTs"`
-	SuccessPUTStats    ServerHTTPMethodStats `json:"successPUTs"`
-	TotalPOSTStats     ServerHTTPMethodStats `json:"totalPOSTs"`
-	SuccessPOSTStats   ServerHTTPMethodStats `json:"successPOSTs"`
-	TotalDELETEStats   ServerHTTPMethodStats `json:"totalDELETEs"`
-	SuccessDELETEStats ServerHTTPMethodStats `json:"successDELETEs"`
+	CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"`
+	TotalS3Requests   ServerHTTPAPIStats `json:"totalS3Requests"`
+	TotalS3Errors     ServerHTTPAPIStats `json:"totalS3Errors"`
 }
 
 // ServerInfoData holds storage, connections and other
 // information of a given server.
 type ServerInfoData struct {
-	StorageInfo StorageInfo      `json:"storage"`
-	ConnStats   ServerConnStats  `json:"network"`
-	HTTPStats   ServerHTTPStats  `json:"http"`
-	Properties  ServerProperties `json:"server"`
+	ConnStats  ServerConnStats  `json:"network"`
+	HTTPStats  ServerHTTPStats  `json:"http"`
+	Properties ServerProperties `json:"server"`
 }
 
 // ServerInfo holds server information result of one node
@@ -274,7 +267,6 @@ type ServerInfo struct {
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
-
 	objectAPI := validateAdminReq(ctx, w, r)
 	if objectAPI == nil {
 		return
@@ -286,9 +278,8 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	serverInfo = append(serverInfo, ServerInfo{
 		Addr: getHostName(r),
 		Data: &ServerInfoData{
-			StorageInfo: objectAPI.StorageInfo(ctx),
-			ConnStats:   globalConnStats.toServerConnStats(),
-			HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+			ConnStats: globalConnStats.toServerConnStats(),
+			HTTPStats: globalHTTPStats.toServerHTTPStats(),
 			Properties: ServerProperties{
 				Uptime:       UTCNow().Sub(globalBootTime),
 				Version:      Version,
@@ -312,6 +303,31 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
+// ServerInfoHandler - GET /minio/admin/v1/storageinfo
+// ----------
+// Get server information
+func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "StorageInfo")
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	storageInfo := objectAPI.StorageInfo(ctx)
+
+	// Marshal API response
+	jsonBytes, err := json.Marshal(storageInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Reply with storage information (across nodes in a
+	// distributed setup) as json.
+	writeSuccessResponseJSON(w, jsonBytes)
+
+}
+
 // ServerCPULoadInfo holds informantion about cpu utilization
 // of one minio node. It also reports any errors if encountered
 // while trying to reach this server.
@@ -814,7 +830,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 
 	// find number of disks in the setup
 	info := objectAPI.StorageInfo(ctx)
-	numDisks := info.Backend.OfflineDisks + info.Backend.OnlineDisks
+	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
 
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
 	if hip.clientToken == "" && !hip.forceStart && !hip.forceStop {

cmd/admin-handlers_test.go

@@ -396,7 +396,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) {
 	queryVal := url.Values{}
 	queryVal.Set("action", string(cmd.toServiceAction()))
-	resource := "/minio/admin/v1/service?" + queryVal.Encode()
+	resource := adminAPIPathPrefix + "/service?" + queryVal.Encode()
 	req, err := newTestRequest(http.MethodPost, resource, 0, nil)
 	if err != nil {
 		return nil, err
@@ -465,7 +465,7 @@ func buildAdminRequest(queryVal url.Values, method, path string,
 	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
 
 	req, err := newTestRequest(method,
-		"/minio/admin/v1"+path+"?"+queryVal.Encode(),
+		adminAPIPathPrefix+path+"?"+queryVal.Encode(),
 		contentLength, bodySeeker)
 	if err != nil {
 		return nil, err

cmd/admin-router.go

@@ -20,10 +20,11 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 const (
-	adminAPIPathPrefix = "/minio/admin"
+	adminAPIPathPrefix = minioReservedBucketPath + "/admin/" + madmin.AdminAPIVersion
 )
 
 // adminAPIHandlers provides HTTP handlers for MinIO admin API.
@@ -37,113 +38,113 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 	// Admin router
 	adminRouter := router.PathPrefix(adminAPIPathPrefix).Subrouter()
 
-	// Version handler
-	adminV1Router := adminRouter.PathPrefix("/v1").Subrouter()
-
 	/// Service operations
 
 	// Restart and stop MinIO service.
-	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
+	adminRouter.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
 	// Update MinIO servers.
-	adminV1Router.Methods(http.MethodPost).Path("/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
+	adminRouter.Methods(http.MethodPost).Path("/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
 
 	// Info operations
-	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
+	adminRouter.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 	// Harware Info operations
-	adminV1Router.Methods(http.MethodGet).Path("/hardware").HandlerFunc(httpTraceAll(adminAPI.ServerHardwareInfoHandler)).Queries("hwType", "{hwType:.*}")
+	adminRouter.Methods(http.MethodGet).Path("/hardware").HandlerFunc(httpTraceAll(adminAPI.ServerHardwareInfoHandler)).Queries("hwType", "{hwType:.*}")
+
+	// StorageInfo operations
+	adminRouter.Methods(http.MethodGet).Path("/storageinfo").HandlerFunc(httpTraceAll(adminAPI.StorageInfoHandler))
 
 	if globalIsDistXL || globalIsXL {
 		/// Heal operations
 
 		// Heal processing endpoint.
-		adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminRouter.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminRouter.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminRouter.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
-		adminV1Router.Methods(http.MethodPost).Path("/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
+		adminRouter.Methods(http.MethodPost).Path("/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
 
 		/// Health operations
 
 	}
 	// Performance command - return performance details based on input type
-	adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
+	adminRouter.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
 
 	// Profiling operations
-	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
+	adminRouter.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
 		Queries("profilerType", "{profilerType:.*}")
-	adminV1Router.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
+	adminRouter.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
 
 	/// Config operations
 	if enableConfigOps {
 		// Get config
-		adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
+		adminRouter.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 		// Set config
-		adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
+		adminRouter.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
 	}
 
 	if enableIAMOps {
 		// -- IAM APIs --
 
 		// Add policy IAM
-		adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name",
+		adminRouter.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name",
 			"{name:.*}")
 
 		// Add user IAM
-		adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
-		adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
+		adminRouter.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
 			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
 		// Info policy IAM
-		adminV1Router.Methods(http.MethodGet).Path("/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodGet).Path("/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
 
 		// Remove policy IAM
-		adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
 
 		// Set user or group policy
-		adminV1Router.Methods(http.MethodPut).Path("/set-user-or-group-policy").
+		adminRouter.Methods(http.MethodPut).Path("/set-user-or-group-policy").
 			HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
 			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
 
 		// Remove user IAM
-		adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
 
 		// List users
-		adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+		adminRouter.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
 
 		// User info
-		adminV1Router.Methods(http.MethodGet).Path("/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path("/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
 
 		// Add/Remove members from group
-		adminV1Router.Methods(http.MethodPut).Path("/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
+		adminRouter.Methods(http.MethodPut).Path("/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
 
 		// Get Group
-		adminV1Router.Methods(http.MethodGet).Path("/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
+		adminRouter.Methods(http.MethodGet).Path("/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
 
 		// List Groups
-		adminV1Router.Methods(http.MethodGet).Path("/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
+		adminRouter.Methods(http.MethodGet).Path("/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
 
 		// Set Group Status
-		adminV1Router.Methods(http.MethodPut).Path("/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
+		adminRouter.Methods(http.MethodPut).Path("/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
 
 		// List policies
-		adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+		adminRouter.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
 	}
 
 	// -- Top APIs --
 	// Top locks
-	adminV1Router.Methods(http.MethodGet).Path("/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+	adminRouter.Methods(http.MethodGet).Path("/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
 
 	// HTTP Trace
-	adminV1Router.Methods(http.MethodGet).Path("/trace").HandlerFunc(adminAPI.TraceHandler)
+	adminRouter.Methods(http.MethodGet).Path("/trace").HandlerFunc(adminAPI.TraceHandler)
 
 	// Console Logs
-	adminV1Router.Methods(http.MethodGet).Path("/log").HandlerFunc(httpTraceAll(adminAPI.ConsoleLogHandler))
+	adminRouter.Methods(http.MethodGet).Path("/log").HandlerFunc(httpTraceAll(adminAPI.ConsoleLogHandler))
 
 	// -- KMS APIs --
 	//
-	adminV1Router.Methods(http.MethodGet).Path("/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
+	adminRouter.Methods(http.MethodGet).Path("/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
 
 	// If none of the routes match, return error.
-	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandler))
-	adminV1Router.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(versionMismatchHandler))
+	adminRouter.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandler))
+	adminRouter.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(versionMismatchHandler))
 }

cmd/api-router.go

@@ -59,108 +59,108 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	for _, bucket := range routers {
 		// Object operations
 		// HeadObject
-		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
+		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler)))
 		// CopyObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("listobjectparts", httpTraceAll(api.ListObjectPartsHandler))).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("completemutipartupload", httpTraceAll(api.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("newmultipartupload", httpTraceAll(api.NewMultipartUploadHandler))).Queries("uploads", "")
 		// AbortMultipartUpload
-		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler))).Queries("acl", "")
 		// GetObjectTagging - this is a dummy call.
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler))).Queries("tagging", "")
 		// SelectObjectContent
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2")
 		// GetObject
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler)))
 		// CopyObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler)))
 		// PutObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler)))
 		// DeleteObject
-		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler)))
 
 		/// Bucket operations
 		// GetBucketLocation
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getobjectlocation", httpTraceAll(api.GetBucketLocationHandler))).Queries("location", "")
 		// GetBucketPolicy
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods("GET").HandlerFunc(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler))).Queries("policy", "")
 		// GetBucketLifecycle
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods("GET").HandlerFunc(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler))).Queries("lifecycle", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler))).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler))).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketwebsite", httpTraceAll(api.GetBucketWebsiteHandler))).Queries("website", "")
 		// GetBucketVersioningHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketversion", httpTraceAll(api.GetBucketVersioningHandler))).Queries("versioning", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketaccelerate", httpTraceAll(api.GetBucketAccelerateHandler))).Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketrequestpayment", httpTraceAll(api.GetBucketRequestPaymentHandler))).Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlogging", httpTraceAll(api.GetBucketLoggingHandler))).Queries("logging", "")
 		// GetBucketLifecycleHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler))).Queries("lifecycle", "")
 		// GetBucketReplicationHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketreplication", httpTraceAll(api.GetBucketReplicationHandler))).Queries("replication", "")
 		// GetBucketTaggingHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler))).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
-		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketwebsite", httpTraceAll(api.DeleteBucketWebsiteHandler))).Queries("website", "")
 		// DeleteBucketTaggingHandler
-		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler))).Queries("tagging", "")
 
 		// GetBucketNotification
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler))).Queries("notification", "")
 		// ListenBucketNotification
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listenbucketnotification", httpTraceAll(api.ListenBucketNotificationHandler))).Queries("events", "{events:.*}")
 		// ListMultipartUploads
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler))).Queries("uploads", "")
 		// ListObjectsV2
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler))).Queries("list-type", "2")
 		// ListBucketVersions
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListBucketObjectVersionsHandler)).Queries("versions", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listbucketversions", httpTraceAll(api.ListBucketObjectVersionsHandler))).Queries("versions", "")
 		// ListObjectsV1 (Legacy)
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		bucket.Methods("GET").HandlerFunc(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler)))
 		// PutBucketLifecycle
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods("PUT").HandlerFunc(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler))).Queries("lifecycle", "")
 		// PutBucketPolicy
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods("PUT").HandlerFunc(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler))).Queries("policy", "")
 
 		// PutBucketNotification
-		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketnotification", httpTraceAll(api.PutBucketNotificationHandler))).Queries("notification", "")
 		// PutBucket
-		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketHandler))
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucket", httpTraceAll(api.PutBucketHandler)))
 		// HeadBucket
-		bucket.Methods(http.MethodHead).HandlerFunc(httpTraceAll(api.HeadBucketHandler))
+		bucket.Methods(http.MethodHead).HandlerFunc(collectAPIStats("headbucket", httpTraceAll(api.HeadBucketHandler)))
 		// PostPolicy
-		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
+		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(collectAPIStats("postpolicybucket", httpTraceHdrs(api.PostPolicyBucketHandler)))
 		// DeleteMultipleObjects
-		bucket.Methods(http.MethodPost).HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
+		bucket.Methods(http.MethodPost).HandlerFunc(collectAPIStats("deletemultipleobjects", httpTraceAll(api.DeleteMultipleObjectsHandler))).Queries("delete", "")
 		// DeleteBucketPolicy
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods("DELETE").HandlerFunc(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler))).Queries("policy", "")
 		// DeleteBucketLifecycle
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods("DELETE").HandlerFunc(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler))).Queries("lifecycle", "")
 		// DeleteBucket
-		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler)))
 	}
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler)))
 
 	// If none of the routes match.
-	apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
+	apiRouter.NotFoundHandler = http.HandlerFunc(collectAPIStats("notfound", httpTraceAll(notFoundHandler)))
 }

cmd/daily-heal-ops.go

@@ -92,7 +92,7 @@ func startDailyHeal() {
 
 	// Find number of disks in the setup
 	info := objAPI.StorageInfo(ctx)
-	numDisks := info.Backend.OnlineDisks + info.Backend.OfflineDisks
+	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
 
 	nh := newBgHealSequence(numDisks)
 	globalSweepHealState.LaunchNewHealSequence(nh)

cmd/fs-v1.go

@@ -252,10 +252,12 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
 	if !fs.diskMount {
 		used = atomic.LoadUint64(&fs.totalUsed)
 	}
+	localPeer := GetLocalPeer(globalEndpoints)
 	storageInfo := StorageInfo{
-		Used:      used,
-		Total:     di.Total,
-		Available: di.Free,
+		Used:       []uint64{used},
+		Total:      []uint64{di.Total},
+		Available:  []uint64{di.Free},
+		MountPaths: []string{localPeer + fs.fsPath},
 	}
 	storageInfo.Backend.Type = BackendFS
 	return storageInfo

cmd/gateway-main.go

@@ -196,8 +196,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	}
 
 	globalHTTPServer = xhttp.NewServer([]string{globalCLIContext.Addr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
-	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
-	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
 	go func() {
 		globalHTTPServerErrorCh <- globalHTTPServer.Start()
 	}()

cmd/gateway/hdfs/gateway-hdfs.go

@@ -226,7 +226,7 @@ func (n *hdfsObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
 		return minio.StorageInfo{}
 	}
 	sinfo := minio.StorageInfo{}
-	sinfo.Used = fsInfo.Used
+	sinfo.Used = []uint64{fsInfo.Used}
 	sinfo.Backend.Type = minio.Unknown
 	return sinfo
 }

cmd/generic-handlers.go

@@ -17,9 +17,7 @@
 package cmd
 
 import (
-	"bufio"
 	"context"
-	"net"
 	"net/http"
 	"strings"
 	"time"
@@ -515,34 +513,6 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.handler.ServeHTTP(w, r)
 }
 
-// httpResponseRecorder wraps http.ResponseWriter
-// to record some useful http response data.
-type httpResponseRecorder struct {
-	http.ResponseWriter
-	respStatusCode int
-}
-
-// Wraps ResponseWriter's Write()
-func (rww *httpResponseRecorder) Write(b []byte) (int, error) {
-	return rww.ResponseWriter.Write(b)
-}
-
-// Wraps ResponseWriter's Flush()
-func (rww *httpResponseRecorder) Flush() {
-	rww.ResponseWriter.(http.Flusher).Flush()
-}
-
-// Wraps ResponseWriter's WriteHeader() and record
-// the response status code
-func (rww *httpResponseRecorder) WriteHeader(httpCode int) {
-	rww.respStatusCode = httpCode
-	rww.ResponseWriter.WriteHeader(httpCode)
-}
-
-func (rww *httpResponseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) {
-	return rww.ResponseWriter.(http.Hijacker).Hijack()
-}
-
 // httpStatsHandler definition: gather HTTP statistics
 type httpStatsHandler struct {
 	handler http.Handler
@@ -554,26 +524,13 @@ func setHTTPStatsHandler(h http.Handler) http.Handler {
 }
 
 func (h httpStatsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// Wraps w to record http response information
-	ww := &httpResponseRecorder{ResponseWriter: w}
-
-	// Time start before the call is about to start.
-	tBefore := UTCNow()
-
+	isS3Request := !strings.HasPrefix(r.URL.Path, minioReservedBucketPath)
+	// record s3 connection stats.
+	recordRequest := &recordTrafficRequest{ReadCloser: r.Body, isS3Request: isS3Request}
+	r.Body = recordRequest
+	recordResponse := &recordTrafficResponse{w, isS3Request}
 	// Execute the request
-	h.handler.ServeHTTP(ww, r)
-
-	// Time after call has completed.
-	tAfter := UTCNow()
-
-	// Time duration in secs since the call started.
-	//
-	// We don't need to do nanosecond precision in this
-	// simply for the fact that it is not human readable.
-	durationSecs := tAfter.Sub(tBefore).Seconds()
-
-	// Update http statistics
-	globalHTTPStats.updateStats(r, ww, durationSecs)
+	h.handler.ServeHTTP(recordResponse, r)
 }
 
 // requestValidityHandler validates all the incoming paths for

cmd/handler-utils.go

@@ -348,6 +348,39 @@ func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
+func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		isS3Request := !strings.HasPrefix(r.URL.Path, minioReservedBucketPath)
+		apiStatsWriter := &recordAPIStats{w, UTCNow(), false, 0, isS3Request}
+
+		// Time start before the call is about to start.
+		tBefore := UTCNow()
+
+		if isS3Request {
+			globalHTTPStats.currentS3Requests.Inc(api)
+		}
+		// Execute the request
+		f.ServeHTTP(apiStatsWriter, r)
+
+		if isS3Request {
+			globalHTTPStats.currentS3Requests.Dec(api)
+		}
+
+		// Firstbyte read.
+		tAfter := apiStatsWriter.TTFB
+
+		// Time duration in secs since the call started.
+		//
+		// We don't need to do nanosecond precision in this
+		// simply for the fact that it is not human readable.
+		durationSecs := tAfter.Sub(tBefore).Seconds()
+
+		// Update http statistics
+		globalHTTPStats.updateStats(api, r, apiStatsWriter, durationSecs)
+	}
+}
+
 // Returns "/bucketName/objectName" for path-style or virtual-host-style requests.
 func getResource(path string, host string, domains []string) (string, error) {
 	if len(domains) == 0 {

cmd/http-stats.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"fmt"
 	"net/http"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -31,6 +33,8 @@ import (
 type ConnStats struct {
 	totalInputBytes  atomic.Uint64
 	totalOutputBytes atomic.Uint64
+	s3InputBytes     atomic.Uint64
+	s3OutputBytes    atomic.Uint64
 }
 
 // Increase total input bytes
@@ -53,11 +57,33 @@ func (s *ConnStats) getTotalOutputBytes() uint64 {
 	return s.totalOutputBytes.Load()
 }
 
-// Return connection stats (total input/output bytes)
+// Increase outbound input bytes
+func (s *ConnStats) incS3InputBytes(n int) {
+	s.s3InputBytes.Add(uint64(n))
+}
+
+// Increase outbound output bytes
+func (s *ConnStats) incS3OutputBytes(n int) {
+	s.s3OutputBytes.Add(uint64(n))
+}
+
+// Return outbound input bytes
+func (s *ConnStats) getS3InputBytes() uint64 {
+	return s.s3InputBytes.Load()
+}
+
+// Return outbound output bytes
+func (s *ConnStats) getS3OutputBytes() uint64 {
+	return s.s3OutputBytes.Load()
+}
+
+// Return connection stats (total input/output bytes and total s3 input/output bytes)
 func (s *ConnStats) toServerConnStats() ServerConnStats {
 	return ServerConnStats{
 		TotalInputBytes:  s.getTotalInputBytes(),
 		TotalOutputBytes: s.getTotalOutputBytes(),
+		S3InputBytes:     s.getS3InputBytes(),
+		S3OutputBytes:    s.getS3OutputBytes(),
 	}
 }
 
@@ -66,35 +92,55 @@ func newConnStats() *ConnStats {
 	return &ConnStats{}
 }
 
-// HTTPMethodStats holds statistics information about
-// a given HTTP method made by all clients
-type HTTPMethodStats struct {
-	Counter  atomic.Uint64
-	Duration atomic.Float64
+// HTTPAPIStats holds statistics information about
+// a given API in the requests.
+type HTTPAPIStats struct {
+	APIStats map[string]int
+	sync.RWMutex
+}
+
+// Inc increments the api stats counter.
+func (stats *HTTPAPIStats) Inc(api string) {
+	stats.Lock()
+	defer stats.Unlock()
+	if stats == nil {
+		return
+	}
+	if stats.APIStats == nil {
+		stats.APIStats = make(map[string]int)
+	}
+	if _, ok := stats.APIStats[api]; ok {
+		stats.APIStats[api]++
+		return
+	}
+	stats.APIStats[api] = 1
+}
+
+// Dec increments the api stats counter.
+func (stats *HTTPAPIStats) Dec(api string) {
+	stats.Lock()
+	defer stats.Unlock()
+	if stats == nil {
+		return
+	}
+	if val, ok := stats.APIStats[api]; ok && val > 0 {
+		stats.APIStats[api]--
+	}
+}
+
+// Load returns the recorded stats.
+func (stats *HTTPAPIStats) Load() map[string]int {
+	stats.Lock()
+	defer stats.Unlock()
+	return stats.APIStats
 }
 
 // HTTPStats holds statistics information about
 // HTTP requests made by all clients
 type HTTPStats struct {
-	// HEAD request stats.
-	totalHEADs   HTTPMethodStats
-	successHEADs HTTPMethodStats
-
-	// GET request stats.
-	totalGETs   HTTPMethodStats
-	successGETs HTTPMethodStats
-
-	// PUT request stats.
-	totalPUTs   HTTPMethodStats
-	successPUTs HTTPMethodStats
-
-	// POST request stats.
-	totalPOSTs   HTTPMethodStats
-	successPOSTs HTTPMethodStats
-
-	// DELETE request stats.
-	totalDELETEs   HTTPMethodStats
-	successDELETEs HTTPMethodStats
+	currentS3Requests HTTPAPIStats
+	totalS3Requests   HTTPAPIStats
+	totalS3Errors     HTTPAPIStats
 }
 
 func durationStr(totalDuration, totalCount float64) string {
@@ -102,95 +148,39 @@ func durationStr(totalDuration, totalCount float64) string {
 }
 
 // Converts http stats into struct to be sent back to the client.
-func (st HTTPStats) toServerHTTPStats() ServerHTTPStats {
+func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 	serverStats := ServerHTTPStats{}
-	serverStats.TotalHEADStats = ServerHTTPMethodStats{
-		Count:       st.totalHEADs.Counter.Load(),
-		AvgDuration: durationStr(st.totalHEADs.Duration.Load(), float64(st.totalHEADs.Counter.Load())),
+
+	serverStats.CurrentS3Requests = ServerHTTPAPIStats{
+		APIStats: st.currentS3Requests.Load(),
 	}
-	serverStats.SuccessHEADStats = ServerHTTPMethodStats{
-		Count:       st.successHEADs.Counter.Load(),
-		AvgDuration: durationStr(st.successHEADs.Duration.Load(), float64(st.successHEADs.Counter.Load())),
+
+	serverStats.TotalS3Requests = ServerHTTPAPIStats{
+		APIStats: st.totalS3Requests.Load(),
 	}
-	serverStats.TotalGETStats = ServerHTTPMethodStats{
-		Count:       st.totalGETs.Counter.Load(),
-		AvgDuration: durationStr(st.totalGETs.Duration.Load(), float64(st.totalGETs.Counter.Load())),
-	}
-	serverStats.SuccessGETStats = ServerHTTPMethodStats{
-		Count:       st.successGETs.Counter.Load(),
-		AvgDuration: durationStr(st.successGETs.Duration.Load(), float64(st.successGETs.Counter.Load())),
-	}
-	serverStats.TotalPUTStats = ServerHTTPMethodStats{
-		Count:       st.totalPUTs.Counter.Load(),
-		AvgDuration: durationStr(st.totalPUTs.Duration.Load(), float64(st.totalPUTs.Counter.Load())),
-	}
-	serverStats.SuccessPUTStats = ServerHTTPMethodStats{
-		Count:       st.successPUTs.Counter.Load(),
-		AvgDuration: durationStr(st.successPUTs.Duration.Load(), float64(st.successPUTs.Counter.Load())),
-	}
-	serverStats.TotalPOSTStats = ServerHTTPMethodStats{
-		Count:       st.totalPOSTs.Counter.Load(),
-		AvgDuration: durationStr(st.totalPOSTs.Duration.Load(), float64(st.totalPOSTs.Counter.Load())),
-	}
-	serverStats.SuccessPOSTStats = ServerHTTPMethodStats{
-		Count:       st.successPOSTs.Counter.Load(),
-		AvgDuration: durationStr(st.successPOSTs.Duration.Load(), float64(st.successPOSTs.Counter.Load())),
-	}
-	serverStats.TotalDELETEStats = ServerHTTPMethodStats{
-		Count:       st.totalDELETEs.Counter.Load(),
-		AvgDuration: durationStr(st.totalDELETEs.Duration.Load(), float64(st.totalDELETEs.Counter.Load())),
-	}
-	serverStats.SuccessDELETEStats = ServerHTTPMethodStats{
-		Count:       st.successDELETEs.Counter.Load(),
-		AvgDuration: durationStr(st.successDELETEs.Duration.Load(), float64(st.successDELETEs.Counter.Load())),
+
+	serverStats.TotalS3Errors = ServerHTTPAPIStats{
+		APIStats: st.totalS3Errors.Load(),
 	}
 	return serverStats
 }
 
 // Update statistics from http request and response data
-func (st *HTTPStats) updateStats(r *http.Request, w *httpResponseRecorder, durationSecs float64) {
+func (st *HTTPStats) updateStats(api string, r *http.Request, w *recordAPIStats, durationSecs float64) {
 	// A successful request has a 2xx response code
 	successReq := (w.respStatusCode >= 200 && w.respStatusCode < 300)
-	// Update stats according to method verb
-	switch r.Method {
-	case "HEAD":
-		st.totalHEADs.Counter.Inc()
-		st.totalHEADs.Duration.Add(durationSecs)
-		if successReq {
-			st.successHEADs.Counter.Inc()
-			st.successHEADs.Duration.Add(durationSecs)
-		}
-	case "GET":
-		st.totalGETs.Counter.Inc()
-		st.totalGETs.Duration.Add(durationSecs)
-		if successReq {
-			st.successGETs.Counter.Inc()
-			st.successGETs.Duration.Add(durationSecs)
-		}
-	case "PUT":
-		st.totalPUTs.Counter.Inc()
-		st.totalPUTs.Duration.Add(durationSecs)
-		if successReq {
-			st.successPUTs.Counter.Inc()
-			st.totalPUTs.Duration.Add(durationSecs)
-		}
-	case "POST":
-		st.totalPOSTs.Counter.Inc()
-		st.totalPOSTs.Duration.Add(durationSecs)
-		if successReq {
-			st.successPOSTs.Counter.Inc()
-			st.totalPOSTs.Duration.Add(durationSecs)
-		}
-	case "DELETE":
-		st.totalDELETEs.Counter.Inc()
-		st.totalDELETEs.Duration.Add(durationSecs)
-		if successReq {
-			st.successDELETEs.Counter.Inc()
-			st.successDELETEs.Duration.Add(durationSecs)
+
+	if w.isS3Request && !strings.HasSuffix(r.URL.Path, prometheusMetricsPath) {
+		st.totalS3Requests.Inc(api)
+		if !successReq && w.respStatusCode != 0 {
+			st.totalS3Errors.Inc(api)
 		}
 	}
-	// Increment the prometheus http request response histogram with appropriate label
-	httpRequestsDuration.With(prometheus.Labels{"request_type": r.Method}).Observe(durationSecs)
+
+	if w.isS3Request && r.Method == "GET" {
+		// Increment the prometheus http request response histogram with appropriate label
+		httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(durationSecs)
+	}
 }
 
 // Prepare new HTTPStats structure

cmd/http-traffic-recorder.go

@@ -0,0 +1,107 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"io"
+	"net/http"
+	"time"
+)
+
+// records the incoming bytes from the underlying request.Body.
+type recordTrafficRequest struct {
+	io.ReadCloser
+	isS3Request bool
+}
+
+// Records the bytes read.
+func (r *recordTrafficRequest) Read(p []byte) (n int, err error) {
+	n, err = r.ReadCloser.Read(p)
+	globalConnStats.incInputBytes(n)
+	if r.isS3Request {
+		globalConnStats.incS3InputBytes(n)
+	}
+	return n, err
+}
+
+// Records the outgoing bytes through the responseWriter.
+type recordTrafficResponse struct {
+	// wrapper for underlying http.ResponseWriter.
+	writer      http.ResponseWriter
+	isS3Request bool
+}
+
+// Calls the underlying WriteHeader.
+func (r *recordTrafficResponse) WriteHeader(i int) {
+	r.writer.WriteHeader(i)
+}
+
+// Calls the underlying Header.
+func (r *recordTrafficResponse) Header() http.Header {
+	return r.writer.Header()
+}
+
+// Records the output bytes
+func (r *recordTrafficResponse) Write(p []byte) (n int, err error) {
+	n, err = r.writer.Write(p)
+	globalConnStats.incOutputBytes(n)
+	// Check if it is s3 request
+	if r.isS3Request {
+		globalConnStats.incS3OutputBytes(n)
+	}
+	return n, err
+}
+
+// Calls the underlying Flush.
+func (r *recordTrafficResponse) Flush() {
+	r.writer.(http.Flusher).Flush()
+}
+
+// Records the outgoing bytes through the responseWriter.
+type recordAPIStats struct {
+	// wrapper for underlying http.ResponseWriter.
+	writer         http.ResponseWriter
+	TTFB           time.Time // TimeToFirstByte.
+	firstByteRead  bool
+	respStatusCode int
+	isS3Request    bool
+}
+
+// Calls the underlying WriteHeader.
+func (r *recordAPIStats) WriteHeader(i int) {
+	r.respStatusCode = i
+	r.writer.WriteHeader(i)
+}
+
+// Calls the underlying Header.
+func (r *recordAPIStats) Header() http.Header {
+	return r.writer.Header()
+}
+
+// Records the TTFB on the first byte write.
+func (r *recordAPIStats) Write(p []byte) (n int, err error) {
+	if !r.firstByteRead {
+		r.TTFB = UTCNow()
+		r.firstByteRead = true
+	}
+	return r.writer.Write(p)
+}
+
+// Calls the underlying Flush.
+func (r *recordAPIStats) Flush() {
+	r.writer.(http.Flusher).Flush()
+}

cmd/http/accountingconn.go

@@ -1,57 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package http
-
-import (
-	"net"
-)
-
-// AccountingConn - is a generic stream-oriented network connection supporting buffered reader and read/write timeout.
-type AccountingConn struct {
-	net.Conn
-	updateBytesReadFunc    func(int) // function to be called to update bytes read.
-	updateBytesWrittenFunc func(int) // function to be called to update bytes written.
-}
-
-// Read - reads data from the connection using wrapped buffered reader.
-func (c *AccountingConn) Read(b []byte) (n int, err error) {
-	n, err = c.Conn.Read(b)
-	if err == nil && c.updateBytesReadFunc != nil {
-		c.updateBytesReadFunc(n)
-	}
-
-	return n, err
-}
-
-// Write - writes data to the connection.
-func (c *AccountingConn) Write(b []byte) (n int, err error) {
-	n, err = c.Conn.Write(b)
-	if err == nil && c.updateBytesWrittenFunc != nil {
-		c.updateBytesWrittenFunc(n)
-	}
-
-	return n, err
-}
-
-// newAccountingConn - creates a new connection object wrapping net.Conn with deadlines.
-func newAccountingConn(c net.Conn, updateBytesReadFunc, updateBytesWrittenFunc func(int)) *AccountingConn {
-	return &AccountingConn{
-		Conn:                   c,
-		updateBytesReadFunc:    updateBytesReadFunc,
-		updateBytesWrittenFunc: updateBytesWrittenFunc,
-	}
-}

cmd/http/listener.go

@@ -33,13 +33,11 @@ type acceptResult struct {
 
 // httpListener - HTTP listener capable of handling multiple server addresses.
 type httpListener struct {
-	mutex                  sync.Mutex         // to guard Close() method.
-	tcpListeners           []*net.TCPListener // underlaying TCP listeners.
-	acceptCh               chan acceptResult  // channel where all TCP listeners write accepted connection.
-	doneCh                 chan struct{}      // done channel for TCP listener goroutines.
-	tcpKeepAliveTimeout    time.Duration
-	updateBytesReadFunc    func(int) // function to be called to update bytes read in Deadlineconn.
-	updateBytesWrittenFunc func(int) // function to be called to update bytes written in Deadlineconn.
+	mutex               sync.Mutex         // to guard Close() method.
+	tcpListeners        []*net.TCPListener // underlaying TCP listeners.
+	acceptCh            chan acceptResult  // channel where all TCP listeners write accepted connection.
+	doneCh              chan struct{}      // done channel for TCP listener goroutines.
+	tcpKeepAliveTimeout time.Duration
 }
 
 // isRoutineNetErr returns true if error is due to a network timeout,
@@ -89,9 +87,7 @@ func (listener *httpListener) start() {
 		tcpConn.SetKeepAlive(true)
 		tcpConn.SetKeepAlivePeriod(listener.tcpKeepAliveTimeout)
 
-		acctConn := newAccountingConn(tcpConn, listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
-
-		send(acceptResult{acctConn, nil}, doneCh)
+		send(acceptResult{tcpConn, nil}, doneCh)
 	}
 
 	// Closure to handle TCPListener until done channel is closed.
@@ -172,9 +168,7 @@ func (listener *httpListener) Addrs() (addrs []net.Addr) {
 // * listen to multiple addresses
 // * controls incoming connections only doing HTTP protocol
 func newHTTPListener(serverAddrs []string,
-	tcpKeepAliveTimeout time.Duration,
-	updateBytesReadFunc func(int),
-	updateBytesWrittenFunc func(int)) (listener *httpListener, err error) {
+	tcpKeepAliveTimeout time.Duration) (listener *httpListener, err error) {
 
 	var tcpListeners []*net.TCPListener
 
@@ -207,10 +201,8 @@ func newHTTPListener(serverAddrs []string,
 	}
 
 	listener = &httpListener{
-		tcpListeners:           tcpListeners,
-		tcpKeepAliveTimeout:    tcpKeepAliveTimeout,
-		updateBytesReadFunc:    updateBytesReadFunc,
-		updateBytesWrittenFunc: updateBytesWrittenFunc,
+		tcpListeners:        tcpListeners,
+		tcpKeepAliveTimeout: tcpKeepAliveTimeout,
 	}
 	listener.start()
 

cmd/http/listener_test.go

@@ -132,30 +132,26 @@ func getNonLoopBackIP(t *testing.T) string {
 
 func TestNewHTTPListener(t *testing.T) {
 	testCases := []struct {
-		serverAddrs            []string
-		tcpKeepAliveTimeout    time.Duration
-		readTimeout            time.Duration
-		writeTimeout           time.Duration
-		updateBytesReadFunc    func(int)
-		updateBytesWrittenFunc func(int)
-		expectedErr            bool
+		serverAddrs         []string
+		tcpKeepAliveTimeout time.Duration
+		readTimeout         time.Duration
+		writeTimeout        time.Duration
+		expectedErr         bool
 	}{
-		{[]string{"93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
-		{[]string{"example.org:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
-		{[]string{"unknown-host"}, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
-		{[]string{"unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
-		{[]string{"localhost:65432", "93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
-		{[]string{"localhost:65432", "unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
-		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, false},
-		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, false},
+		{[]string{"93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
+		{[]string{"example.org:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
+		{[]string{"unknown-host"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
+		{[]string{"unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
+		{[]string{"localhost:65432", "93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
+		{[]string{"localhost:65432", "unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
+		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
+		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
 	}
 
 	for _, testCase := range testCases {
 		listener, err := newHTTPListener(
 			testCase.serverAddrs,
 			testCase.tcpKeepAliveTimeout,
-			testCase.updateBytesReadFunc,
-			testCase.updateBytesWrittenFunc,
 		)
 
 		if !testCase.expectedErr {
@@ -190,7 +186,6 @@ func TestHTTPListenerStartClose(t *testing.T) {
 		listener, err := newHTTPListener(
 			testCase.serverAddrs,
 			time.Duration(0),
-			nil, nil,
 		)
 		if err != nil {
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
@@ -231,7 +226,6 @@ func TestHTTPListenerAddr(t *testing.T) {
 		listener, err := newHTTPListener(
 			testCase.serverAddrs,
 			time.Duration(0),
-			nil, nil,
 		)
 		if err != nil {
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
@@ -269,7 +263,6 @@ func TestHTTPListenerAddrs(t *testing.T) {
 		listener, err := newHTTPListener(
 			testCase.serverAddrs,
 			time.Duration(0),
-			nil, nil,
 		)
 		if err != nil {
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)

cmd/http/server.go

@@ -46,15 +46,13 @@ const (
 // Server - extended http.Server supports multiple addresses to serve and enhanced connection handling.
 type Server struct {
 	http.Server
-	Addrs                  []string      // addresses on which the server listens for new connection.
-	ShutdownTimeout        time.Duration // timeout used for graceful server shutdown.
-	TCPKeepAliveTimeout    time.Duration // timeout used for underneath TCP connection.
-	UpdateBytesReadFunc    func(int)     // function to be called to update bytes read in bufConn.
-	UpdateBytesWrittenFunc func(int)     // function to be called to update bytes written in bufConn.
-	listenerMutex          sync.Mutex    // to guard 'listener' field.
-	listener               *httpListener // HTTP listener for all 'Addrs' field.
-	inShutdown             uint32        // indicates whether the server is in shutdown or not
-	requestCount           int32         // counter holds no. of request in progress.
+	Addrs               []string      // addresses on which the server listens for new connection.
+	ShutdownTimeout     time.Duration // timeout used for graceful server shutdown.
+	TCPKeepAliveTimeout time.Duration // timeout used for underneath TCP connection.
+	listenerMutex       sync.Mutex    // to guard 'listener' field.
+	listener            *httpListener // HTTP listener for all 'Addrs' field.
+	inShutdown          uint32        // indicates whether the server is in shutdown or not
+	requestCount        int32         // counter holds no. of request in progress.
 }
 
 // GetRequestCount - returns number of request in progress.
@@ -79,8 +77,6 @@ func (srv *Server) Start() (err error) {
 	listener, err = newHTTPListener(
 		addrs,
 		tcpKeepAliveTimeout,
-		srv.UpdateBytesReadFunc,
-		srv.UpdateBytesWrittenFunc,
 	)
 	if err != nil {
 		return err

cmd/metrics.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"net/http"
+	"strings"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/prometheus/client_golang/prometheus"
@@ -28,11 +29,11 @@ import (
 var (
 	httpRequestsDuration = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name:    "minio_http_requests_duration_seconds",
+			Name:    "s3_ttfb_seconds",
 			Help:    "Time taken by requests served by current MinIO server instance",
-			Buckets: []float64{.001, .003, .005, .1, .5, 1},
+			Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
 		},
-		[]string{"request_type"},
+		[]string{"api"},
 	)
 	minioVersionInfo = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
@@ -79,51 +80,7 @@ func (c *minioCollector) Describe(ch chan<- *prometheus.Desc) {
 func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 
 	// Expose MinIO's version information
-	minioVersionInfo.WithLabelValues(Version, CommitID).Add(1)
-
-	// Always expose network stats
-
-	// Network Sent/Received Bytes
-	ch <- prometheus.MustNewConstMetric(
-		prometheus.NewDesc(
-			prometheus.BuildFQName("minio", "network", "sent_bytes_total"),
-			"Total number of bytes sent by current MinIO server instance",
-			nil, nil),
-		prometheus.CounterValue,
-		float64(globalConnStats.getTotalOutputBytes()),
-	)
-	ch <- prometheus.MustNewConstMetric(
-		prometheus.NewDesc(
-			prometheus.BuildFQName("minio", "network", "received_bytes_total"),
-			"Total number of bytes received by current MinIO server instance",
-			nil, nil),
-		prometheus.CounterValue,
-		float64(globalConnStats.getTotalInputBytes()),
-	)
-
-	// Expose cache stats only if available
-	cacheObjLayer := newCacheObjectsFn()
-	if cacheObjLayer != nil {
-		cs := cacheObjLayer.StorageInfo(context.Background())
-		ch <- prometheus.MustNewConstMetric(
-			prometheus.NewDesc(
-				prometheus.BuildFQName("minio", "disk", "cache_storage_bytes"),
-				"Total cache capacity on current MinIO server instance",
-				nil, nil),
-			prometheus.GaugeValue,
-			float64(cs.Total),
-		)
-		ch <- prometheus.MustNewConstMetric(
-			prometheus.NewDesc(
-				prometheus.BuildFQName("minio", "disk", "cache_storage_free_bytes"),
-				"Total cache available on current MinIO server instance",
-				nil, nil),
-			prometheus.GaugeValue,
-			float64(cs.Free),
-		)
-	}
-
-	// Expose disk stats only if applicable
+	minioVersionInfo.WithLabelValues(Version, CommitID).Set(float64(1.0))
 
 	// Fetch disk space info
 	objLayer := newObjectLayerFn()
@@ -132,70 +89,158 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 		return
 	}
 
-	s := objLayer.StorageInfo(context.Background())
-
-	// Gateways don't provide disk info
-	if s.Backend.Type == Unknown {
-		return
+	storageAPIs := []StorageAPI{}
+	for _, endpoint := range globalEndpoints {
+		if endpoint.IsLocal {
+			// Construct storageAPIs.
+			sAPI, _ := newStorageAPI(endpoint)
+			storageAPIs = append(storageAPIs, sAPI)
+		}
 	}
 
-	var totalDisks, offlineDisks int
-	// Setting totalDisks to 1 and offlineDisks to 0 in FS mode
-	if s.Backend.Type == BackendFS {
-		totalDisks = 1
-		offlineDisks = 0
-	} else {
-		offlineDisks = s.Backend.OfflineDisks
-		totalDisks = s.Backend.OfflineDisks + s.Backend.OnlineDisks
+	disksInfo, onlineDisks, offlineDisks := getDisksInfo(storageAPIs)
+	totalDisks := offlineDisks.Merge(onlineDisks)
+
+	for _, offDisks := range offlineDisks {
+		// MinIO Offline Disks per node
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("minio", "disks", "offline"),
+				"Total number of offline disks in current MinIO server instance",
+				nil, nil),
+			prometheus.GaugeValue,
+			float64(offDisks),
+		)
 	}
 
-	// Total disk usage by current MinIO server instance
+	for _, totDisks := range totalDisks {
+		// MinIO Total Disks per node
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("minio", "disks", "total"),
+				"Total number of disks for current MinIO server instance",
+				nil, nil),
+			prometheus.GaugeValue,
+			float64(totDisks),
+		)
+	}
+
+	localPeer := GetLocalPeer(globalEndpoints)
+	for _, di := range disksInfo {
+		// Trim the host
+		absPath := strings.TrimPrefix(di.RelativePath, localPeer)
+
+		// Total disk usage by the disk
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("disk", "storage", "used"),
+				"Total disk storage used on the disk",
+				[]string{"disk"}, nil),
+			prometheus.GaugeValue,
+			float64(di.Total-di.Free),
+			absPath,
+		)
+
+		// Total available space in the disk
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("disk", "storage", "available"),
+				"Total available space left on the disk",
+				[]string{"disk"}, nil),
+			prometheus.GaugeValue,
+			float64(di.Free),
+			absPath,
+		)
+
+		// Total storage space of the disk
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("disk", "storage", "total"),
+				"Total space on the disk",
+				[]string{"disk"}, nil),
+			prometheus.GaugeValue,
+			float64(di.Total),
+			absPath,
+		)
+	}
+
+	connStats := globalConnStats.toServerConnStats()
+	httpStats := globalHTTPStats.toServerHTTPStats()
+
+	// Network Sent/Received Bytes (internode)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("minio", "disk", "storage_used_bytes"),
-			"Total disk storage used by current MinIO server instance",
+			prometheus.BuildFQName("internode", "tx", "bytes_total"),
+			"Total number of bytes sent to the other peer nodes by current MinIO server instance",
 			nil, nil),
-		prometheus.GaugeValue,
-		float64(s.Used),
+		prometheus.CounterValue,
+		float64(connStats.TotalOutputBytes),
 	)
 
-	// Total disk available space seen by MinIO server instance
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("minio", "disk", "storage_available_bytes"),
-			"Total disk available space seen by MinIO server instance",
+			prometheus.BuildFQName("internode", "rx", "bytes_total"),
+			"Total number of internode bytes received by current MinIO server instance",
 			nil, nil),
-		prometheus.GaugeValue,
-		float64(s.Available),
+		prometheus.CounterValue,
+		float64(connStats.TotalInputBytes),
 	)
 
-	// Total disk space seen by MinIO server instance
+	// Network Sent/Received Bytes (Outbound)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("minio", "disk", "storage_total_bytes"),
-			"Total disk space seen by MinIO server instance",
+			prometheus.BuildFQName("s3", "tx", "bytes_total"),
+			"Total number of s3 bytes sent by current MinIO server instance",
 			nil, nil),
-		prometheus.GaugeValue,
-		float64(s.Total),
+		prometheus.CounterValue,
+		float64(connStats.S3OutputBytes),
 	)
 
-	// MinIO Total Disk/Offline Disk
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("minio", "total", "disks"),
-			"Total number of disks for current MinIO server instance",
+			prometheus.BuildFQName("s3", "rx", "bytes_total"),
+			"Total number of s3 bytes received by current MinIO server instance",
 			nil, nil),
-		prometheus.GaugeValue,
-		float64(totalDisks),
-	)
-	ch <- prometheus.MustNewConstMetric(
-		prometheus.NewDesc(
-			prometheus.BuildFQName("minio", "offline", "disks"),
-			"Total number of offline disks for current MinIO server instance",
-			nil, nil),
-		prometheus.GaugeValue,
-		float64(offlineDisks),
+		prometheus.CounterValue,
+		float64(connStats.S3InputBytes),
 	)
+
+	for api, value := range httpStats.CurrentS3Requests.APIStats {
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("s3", "requests", "current"),
+				"Total number of running s3 requests in current MinIO server instance",
+				[]string{"api"}, nil),
+			prometheus.CounterValue,
+			float64(value),
+			api,
+		)
+	}
+
+	for api, value := range httpStats.TotalS3Requests.APIStats {
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("s3", "requests", "total"),
+				"Total number of s3 requests in current MinIO server instance",
+				[]string{"api"}, nil),
+			prometheus.CounterValue,
+			float64(value),
+			api,
+		)
+	}
+
+	for api, value := range httpStats.TotalS3Errors.APIStats {
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("s3", "errors", "total"),
+				"Total number of s3 errors in current MinIO server instance",
+				[]string{"api"}, nil),
+			prometheus.CounterValue,
+			float64(value),
+			api,
+		)
+	}
+
 }
 
 func metricsHandler() http.Handler {

cmd/object-api-datatypes.go

@@ -39,11 +39,13 @@ const (
 
 // StorageInfo - represents total capacity of underlying storage.
 type StorageInfo struct {
-	Used uint64 // Used total used per tenant.
+	Used []uint64 // Used total used per disk.
 
-	Total uint64 // Total disk space.
+	Total []uint64 // Total disk space per disk.
 
-	Available uint64 // Total disk space available.
+	Available []uint64 // Total disk space available per disk.
+
+	MountPaths []string // Disk mountpoints
 
 	// Backend type.
 	Backend struct {
@@ -51,12 +53,12 @@ type StorageInfo struct {
 		Type BackendType
 
 		// Following fields are only meaningful if BackendType is Erasure.
-		OnlineDisks      int // Online disks during server startup.
-		OfflineDisks     int // Offline disks during server startup.
-		StandardSCData   int // Data disks for currently configured Standard storage class.
-		StandardSCParity int // Parity disks for currently configured Standard storage class.
-		RRSCData         int // Data disks for currently configured Reduced Redundancy storage class.
-		RRSCParity       int // Parity disks for currently configured Reduced Redundancy storage class.
+		OnlineDisks      madmin.BackendDisks // Online disks during server startup.
+		OfflineDisks     madmin.BackendDisks // Offline disks during server startup.
+		StandardSCData   int                 // Data disks for currently configured Standard storage class.
+		StandardSCParity int                 // Parity disks for currently configured Standard storage class.
+		RRSCData         int                 // Data disks for currently configured Reduced Redundancy storage class.
+		RRSCParity       int                 // Parity disks for currently configured Reduced Redundancy storage class.
 
 		// List of all disk status, this is only meaningful if BackendType is Erasure.
 		Sets [][]madmin.DriveInfo

cmd/peer-rest-server.go

@@ -50,11 +50,11 @@ func getServerInfo() (*ServerInfoData, error) {
 	if objLayer == nil {
 		return nil, errServerNotInitialized
 	}
+
 	// Server info data.
 	return &ServerInfoData{
-		StorageInfo: objLayer.StorageInfo(context.Background()),
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+		ConnStats: globalConnStats.toServerConnStats(),
+		HTTPStats: globalHTTPStats.toServerHTTPStats(),
 		Properties: ServerProperties{
 			Uptime:       UTCNow().Sub(globalBootTime),
 			Version:      Version,

cmd/posix.go

@@ -309,10 +309,11 @@ func (s *posix) IsOnline() bool {
 // DiskInfo is an extended type which returns current
 // disk usage per path.
 type DiskInfo struct {
-	Total    uint64
-	Free     uint64
-	Used     uint64
-	RootDisk bool
+	Total        uint64
+	Free         uint64
+	Used         uint64
+	RootDisk     bool
+	RelativePath string
 }
 
 // DiskInfo provides current information about disk space usage,
@@ -346,12 +347,14 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
 	if err != nil {
 		return info, err
 	}
+	localPeer := GetLocalPeer(globalEndpoints)
 
 	return DiskInfo{
-		Total:    di.Total,
-		Free:     di.Free,
-		Used:     used,
-		RootDisk: rootDisk,
+		Total:        di.Total,
+		Free:         di.Free,
+		Used:         used,
+		RootDisk:     rootDisk,
+		RelativePath: localPeer + s.diskPath,
 	}, nil
 }
 

cmd/server-main.go

@@ -305,8 +305,6 @@ func serverMain(ctx *cli.Context) {
 	}
 
 	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, criticalErrorHandler{handler}, getCert)
-	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
-	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
 	go func() {
 		globalHTTPServerErrorCh <- globalHTTPServer.Start()
 	}()

cmd/server-startup-msg.go

@@ -188,7 +188,7 @@ func printObjectAPIMsg() {
 func getStorageInfoMsg(storageInfo StorageInfo) string {
 	var msg string
 	if storageInfo.Backend.Type == BackendErasure {
-		diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks, storageInfo.Backend.OfflineDisks)
+		diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks.Sum(), storageInfo.Backend.OfflineDisks.Sum())
 		msg += color.Blue("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo)
 	}
 	return msg

cmd/server-startup-msg_test.go

@@ -27,14 +27,15 @@ import (
 	"time"
 
 	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // Tests if we generate storage info.
 func TestStorageInfoMsg(t *testing.T) {
 	infoStorage := StorageInfo{}
 	infoStorage.Backend.Type = BackendErasure
-	infoStorage.Backend.OnlineDisks = 7
-	infoStorage.Backend.OfflineDisks = 1
+	infoStorage.Backend.OnlineDisks = madmin.BackendDisks{"127.0.0.1:9000": 4, "127.0.0.1:9001": 3}
+	infoStorage.Backend.OfflineDisks = madmin.BackendDisks{"127.0.0.1:9000": 0, "127.0.0.1:9001": 1}
 
 	if msg := getStorageInfoMsg(infoStorage); !strings.Contains(msg, "7 Online, 1 Offline") {
 		t.Fatal("Unexpected storage info message, found:", msg)

cmd/xl-sets.go

@@ -322,11 +322,12 @@ func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
 	g.Wait()
 
 	for _, lstorageInfo := range storageInfos {
-		storageInfo.Used += lstorageInfo.Used
-		storageInfo.Total += lstorageInfo.Total
-		storageInfo.Available += lstorageInfo.Available
-		storageInfo.Backend.OnlineDisks += lstorageInfo.Backend.OnlineDisks
-		storageInfo.Backend.OfflineDisks += lstorageInfo.Backend.OfflineDisks
+		storageInfo.Used = append(storageInfo.Used, lstorageInfo.Used...)
+		storageInfo.Total = append(storageInfo.Total, lstorageInfo.Total...)
+		storageInfo.Available = append(storageInfo.Available, lstorageInfo.Available...)
+		storageInfo.MountPaths = append(storageInfo.MountPaths, lstorageInfo.MountPaths...)
+		storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks)
+		storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks)
 	}
 
 	scfg := globalServerConfig.GetStorageClass()

cmd/xl-v1.go

@@ -19,9 +19,12 @@ package cmd
 import (
 	"context"
 	"sort"
+	"strings"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
+	"github.com/minio/minio/pkg/madmin"
+	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
@@ -69,7 +72,7 @@ func (d byDiskTotal) Less(i, j int) bool {
 }
 
 // getDisksInfo - fetch disks info across all other storage API.
-func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks int, offlineDisks int) {
+func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlineDisks madmin.BackendDisks) {
 	disksInfo = make([]DiskInfo, len(disks))
 
 	g := errgroup.WithNErrs(len(disks))
@@ -94,13 +97,33 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks int, of
 		}, index)
 	}
 
-	// Wait for the routines.
-	for _, err := range g.Wait() {
+	getPeerAddress := func(diskPath string) (string, error) {
+		hostPort := strings.Split(diskPath, SlashSeparator)[0]
+		thisAddr, err := xnet.ParseHost(hostPort)
 		if err != nil {
-			offlineDisks++
+			return "", err
+		}
+		return thisAddr.String(), nil
+	}
+
+	onlineDisks = make(madmin.BackendDisks)
+	offlineDisks = make(madmin.BackendDisks)
+	// Wait for the routines.
+	for i, err := range g.Wait() {
+		peerAddr, pErr := getPeerAddress(disksInfo[i].RelativePath)
+		if pErr != nil {
 			continue
 		}
-		onlineDisks++
+		if _, ok := offlineDisks[peerAddr]; !ok {
+			offlineDisks[peerAddr] = 0
+		}
+		if _, ok := onlineDisks[peerAddr]; !ok {
+			onlineDisks[peerAddr] = 0
+		}
+		if err != nil {
+			offlineDisks[peerAddr]++
+		}
+		onlineDisks[peerAddr]++
 	}
 
 	// Success.
@@ -134,17 +157,23 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
 	}
 
 	// Combine all disks to get total usage
-	var used, total, available uint64
-	for _, di := range validDisksInfo {
-		used = used + di.Used
-		total = total + di.Total
-		available = available + di.Free
+	usedList := make([]uint64, len(validDisksInfo))
+	totalList := make([]uint64, len(validDisksInfo))
+	availableList := make([]uint64, len(validDisksInfo))
+	mountPaths := make([]string, len(validDisksInfo))
+
+	for i, di := range validDisksInfo {
+		usedList[i] = di.Used
+		totalList[i] = di.Total
+		availableList[i] = di.Free
+		mountPaths[i] = di.RelativePath
 	}
 
 	storageInfo := StorageInfo{
-		Used:      used,
-		Total:     total,
-		Available: available,
+		Used:       usedList,
+		Total:      totalList,
+		Available:  availableList,
+		MountPaths: mountPaths,
 	}
 
 	storageInfo.Backend.Type = BackendErasure