From 2fa561f22e1dcb05372d9ed8f074ea790ea68f8c Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Tue, 1 Aug 2023 00:55:39 -0700
Subject: [PATCH] do not crash on invalid metric values (#17764)

```
minio[1032735]: panic: label value "\xc0.\xc0." is not valid UTF-8
minio[1032735]: goroutine 1781101 [running]:
minio[1032735]: github.com/prometheus/client_golang/prometheus.MustNewConstMetric(...)
```

log such errors for investigation
---
 cmd/metrics-v2.go | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go
index e07d1a1eb..351e4e478 100644
--- a/cmd/metrics-v2.go
+++ b/cmd/metrics-v2.go
@@ -2953,7 +2953,7 @@ func (c *minioBucketCollector) Collect(out chan<- prometheus.Metric) {
 					continue
 				}
 				for k, v := range metric.Histogram {
-					out <- prometheus.MustNewConstMetric(
+					pmetric, err := prometheus.NewConstMetric(
 						prometheus.NewDesc(
 							prometheus.BuildFQName(string(metric.Description.Namespace),
 								string(metric.Description.Subsystem),
@@ -2965,6 +2965,11 @@ func (c *minioBucketCollector) Collect(out chan<- prometheus.Metric) {
 						prometheus.GaugeValue,
 						float64(v),
 						append(values, k)...)
+					if err != nil {
+						logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram), "bucket-metrics-histogram")
+					} else {
+						out <- pmetric
+					}
 				}
 				continue
 			}
@@ -2972,7 +2977,7 @@ func (c *minioBucketCollector) Collect(out chan<- prometheus.Metric) {
 			if metric.Description.Type == counterMetric {
 				metricType = prometheus.CounterValue
 			}
-			toPost := prometheus.MustNewConstMetric(
+			pmetric, err := prometheus.NewConstMetric(
 				prometheus.NewDesc(
 					prometheus.BuildFQName(string(metric.Description.Namespace),
 						string(metric.Description.Subsystem),
@@ -2984,7 +2989,11 @@ func (c *minioBucketCollector) Collect(out chan<- prometheus.Metric) {
 				metricType,
 				metric.Value,
 				values...)
-			out <- toPost
+			if err != nil {
+				logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "bucket-metrics")
+			} else {
+				out <- pmetric
+			}
 		}
 	}
 
@@ -3024,7 +3033,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 					continue
 				}
 				for k, v := range metric.Histogram {
-					out <- prometheus.MustNewConstMetric(
+					pmetric, err := prometheus.NewConstMetric(
 						prometheus.NewDesc(
 							prometheus.BuildFQName(string(metric.Description.Namespace),
 								string(metric.Description.Subsystem),
@@ -3036,6 +3045,11 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 						prometheus.GaugeValue,
 						float64(v),
 						append(values, k)...)
+					if err != nil {
+						logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v:%v", err, values, metric.Histogram), "cluster-metrics-histogram")
+					} else {
+						out <- pmetric
+					}
 				}
 				continue
 			}
@@ -3043,7 +3057,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 			if metric.Description.Type == counterMetric {
 				metricType = prometheus.CounterValue
 			}
-			toPost := prometheus.MustNewConstMetric(
+			pmetric, err := prometheus.NewConstMetric(
 				prometheus.NewDesc(
 					prometheus.BuildFQName(string(metric.Description.Namespace),
 						string(metric.Description.Subsystem),
@@ -3055,7 +3069,11 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 				metricType,
 				metric.Value,
 				values...)
-			out <- toPost
+			if err != nil {
+				logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "cluster-metrics")
+			} else {
+				out <- pmetric
+			}
 		}
 	}