publish storage API latency through node metrics (#14117)

Publish storage functions latency to help compare the performance 
of different disks in a single deployment.

e.g.:
```
minio_node_disk_latency_us{api="storage.WalkDir",disk="/tmp/xl/1",server="localhost:9001"} 226
minio_node_disk_latency_us{api="storage.WalkDir",disk="/tmp/xl/2",server="localhost:9002"} 1180
minio_node_disk_latency_us{api="storage.WalkDir",disk="/tmp/xl/3",server="localhost:9003"} 1183
minio_node_disk_latency_us{api="storage.WalkDir",disk="/tmp/xl/4",server="localhost:9004"} 1625
```

cmd/metrics-v2.go

@@ -70,6 +70,7 @@ func init() {
 
 	nodeCollector = newMinioCollectorNode([]*MetricsGroup{
 		getNodeHealthMetrics(),
+		getLocalDiskStorageMetrics(),
 		getCacheMetrics(),
 		getHTTPMetrics(),
 		getNetworkMetrics(),
@@ -157,6 +158,8 @@ const (
 	writeBytes      MetricName = "write_bytes"
 	wcharBytes      MetricName = "wchar_bytes"
 
+	apiLatencyMicroSec MetricName = "latency_us"
+
 	usagePercent MetricName = "update_percent"
 
 	commitInfo  MetricName = "commit_info"
@@ -315,6 +318,16 @@ func getClusterCapacityUsageFreeBytesMD() MetricDescription {
 	}
 }
 
+func getNodeDiskAPILatencyMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: diskSubsystem,
+		Name:      apiLatencyMicroSec,
+		Help:      "Average last minute latency in µs for disk API storage operations.",
+		Type:      gaugeMetric,
+	}
+}
+
 func getNodeDiskUsedBytesMD() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
@@ -1583,6 +1596,35 @@ func getLocalStorageMetrics() *MetricsGroup {
 				Value:          float64(disk.FreeInodes),
 				VariableLabels: map[string]string{"disk": disk.DrivePath},
 			})
+
+		}
+		return
+	})
+	return mg
+}
+
+func getLocalDiskStorageMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: 3 * time.Second,
+	}
+	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
+		objLayer := newObjectLayerFn()
+		// Service not initialized yet
+		if objLayer == nil || globalIsGateway {
+			return
+		}
+
+		metrics = make([]Metric, 0, 50)
+		storageInfo, _ := objLayer.LocalStorageInfo(ctx)
+		for _, disk := range storageInfo.Disks {
+			for apiName, latency := range disk.Metrics.APILatencies {
+				val := latency.(uint64)
+				metrics = append(metrics, Metric{
+					Description:    getNodeDiskAPILatencyMD(),
+					Value:          float64(val / 1000),
+					VariableLabels: map[string]string{"disk": disk.DrivePath, "api": "storage." + apiName},
+				})
+			}
 		}
 		return
 	})