add metrics ioerror counter for alerts on I/O errors (#19618)

cmd/metrics-v2.go

@@ -541,6 +541,16 @@ func getNodeDriveTimeoutErrorsMD() MetricDescription {
 	}
 }
 
+func getNodeDriveIOErrorsMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: driveSubsystem,
+		Name:      "errors_ioerror",
+		Help:      "Total number of drive I/O errors since server start",
+		Type:      counterMetric,
+	}
+}
+
 func getNodeDriveAvailabilityErrorsMD() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
@@ -3521,6 +3531,12 @@ func getLocalStorageMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
 					VariableLabels: map[string]string{"drive": disk.DrivePath},
 				})
 
+				metrics = append(metrics, MetricV2{
+					Description:    getNodeDriveIOErrorsMD(),
+					Value:          float64(disk.Metrics.TotalErrorsAvailability - disk.Metrics.TotalErrorsTimeout),
+					VariableLabels: map[string]string{"drive": disk.DrivePath},
+				})
+
 				metrics = append(metrics, MetricV2{
 					Description:    getNodeDriveAvailabilityErrorsMD(),
 					Value:          float64(disk.Metrics.TotalErrorsAvailability),

cmd/metrics-v3-system-drive.go

@@ -47,6 +47,7 @@ const (
 	driveFreeInodes              = "free_inodes"
 	driveTotalInodes             = "total_inodes"
 	driveTimeoutErrorsTotal      = "timeout_errors_total"
+	driveIOErrorsTotal           = "io_errors_total"
 	driveAvailabilityErrorsTotal = "availability_errors_total"
 	driveWaitingIO               = "waiting_io"
 	driveAPILatencyMicros        = "api_latency_micros"
@@ -82,6 +83,8 @@ var (
 		"Total inodes available on a drive", allDriveLabels...)
 	driveTimeoutErrorsMD = NewCounterMD(driveTimeoutErrorsTotal,
 		"Total timeout errors on a drive", allDriveLabels...)
+	driveIOErrorsMD = NewCounterMD(driveIOErrorsTotal,
+		"Total I/O errors on a drive", allDriveLabels...)
 	driveAvailabilityErrorsMD = NewCounterMD(driveAvailabilityErrorsTotal,
 		"Total availability errors (I/O errors, timeouts) on a drive",
 		allDriveLabels...)
@@ -167,6 +170,7 @@ func (m *MetricValues) setDriveAPIMetrics(disk madmin.Disk, labels []string) {
 	}
 
 	m.Set(driveTimeoutErrorsTotal, float64(disk.Metrics.TotalErrorsTimeout), labels...)
+	m.Set(driveIOErrorsTotal, float64(disk.Metrics.TotalErrorsAvailability-disk.Metrics.TotalErrorsTimeout), labels...)
 	m.Set(driveAvailabilityErrorsTotal, float64(disk.Metrics.TotalErrorsAvailability), labels...)
 	m.Set(driveWaitingIO, float64(disk.Metrics.TotalWaiting), labels...)
 

cmd/metrics-v3.go

@@ -153,6 +153,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 			driveFreeInodesMD,
 			driveTotalInodesMD,
 			driveTimeoutErrorsMD,
+			driveIOErrorsMD,
 			driveAvailabilityErrorsMD,
 			driveWaitingIOMD,
 			driveAPILatencyMD,