re-implement prometheus metrics endpoint to be simpler (#13922)

data-structures were repeatedly initialized
this causes GC pressure, instead re-use the
collectors.

Initialize collectors in `init()`, also make
sure to honor the cache semantics for performance
requirements.

Avoid a global map and a global lock for metrics
lookup instead let them all be lock-free unless
the cache is being invalidated.

cmd/peer-rest-server.go

@@ -1113,7 +1113,7 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 
 	enc := gob.NewEncoder(w)
 
-	ch := ReportMetrics(r.Context(), GetGeneratorsForPeer)
+	ch := ReportMetrics(r.Context(), peerMetricsGroups)
 	for m := range ch {
 		if err := enc.Encode(m); err != nil {
 			s.writeErrorResponse(w, errors.New("Encoding metric failed: "+err.Error()))

cmd/utils_test.go

@@ -29,6 +29,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 )
 
 // Tests maximum object size.
@@ -486,3 +487,32 @@ func TestGetMinioMode(t *testing.T) {
 	testMinioMode(globalMinioModeGatewayPrefix + globalGatewayName)
 
 }
+
+func TestTimedValue(t *testing.T) {
+	var cache timedValue
+	t.Parallel()
+	cache.Once.Do(func() {
+		cache.TTL = 2 * time.Second
+		cache.Update = func() (interface{}, error) {
+			return time.Now(), nil
+		}
+	})
+
+	i, _ := cache.Get()
+	t1 := i.(time.Time)
+
+	j, _ := cache.Get()
+	t2 := j.(time.Time)
+
+	if !t1.Equal(t2) {
+		t.Fatalf("expected time to be equal: %s != %s", t1, t2)
+	}
+
+	time.Sleep(3 * time.Second)
+	k, _ := cache.Get()
+	t3 := k.(time.Time)
+
+	if t1.Equal(t3) {
+		t.Fatalf("expected time to be un-equal: %s == %s", t1, t3)
+	}
+}