Add metrics support for Azure & GCS Gateway (#8954)

We added support for caching and S3 related metrics in #8591. As
a continuation, it would be helpful to add support for Azure & GCS
gateway related metrics as well.

cmd/gateway-common.go

@@ -417,3 +417,32 @@ func gatewayHandleEnvVars() {
 		}
 	}
 }
+
+// shouldMeterRequest checks whether incoming request should be added to prometheus gateway metrics
+func shouldMeterRequest(req *http.Request) bool {
+	return !(guessIsBrowserReq(req) || guessIsHealthCheckReq(req) || guessIsMetricsReq(req))
+}
+
+// MetricsTransport is a custom wrapper around Transport to track metrics
+type MetricsTransport struct {
+	Transport *http.Transport
+	Metrics   *Metrics
+}
+
+// RoundTrip implements the RoundTrip method for MetricsTransport
+func (m MetricsTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	metered := shouldMeterRequest(r)
+	if metered && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
+		m.Metrics.IncRequests(r.Method)
+		m.Metrics.IncBytesSent(r.ContentLength)
+	}
+	// Make the request to the server.
+	resp, err := m.Transport.RoundTrip(r)
+	if err != nil {
+		return nil, err
+	}
+	if metered && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
+		m.Metrics.IncBytesReceived(resp.ContentLength)
+	}
+	return resp, nil
+}

cmd/gateway/azure/gateway-azure.go

@@ -146,7 +146,14 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
 		return &azureObjects{}, err
 	}
 
-	httpClient := &http.Client{Transport: minio.NewCustomHTTPTransport()}
+	metrics := minio.NewMetrics()
+
+	t := &minio.MetricsTransport{
+		Transport: minio.NewCustomHTTPTransport(),
+		Metrics:   metrics,
+	}
+
+	httpClient := &http.Client{Transport: t}
 	userAgent := fmt.Sprintf("APN/1.0 MinIO/1.0 MinIO/%s", minio.Version)
 
 	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
@@ -168,6 +175,7 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
 		endpoint:   endpointURL.String(),
 		httpClient: httpClient,
 		client:     client,
+		metrics:    metrics,
 	}, nil
 }
 
@@ -357,6 +365,7 @@ type azureObjects struct {
 	minio.GatewayUnsupported
 	endpoint   string
 	httpClient *http.Client
+	metrics    *minio.Metrics
 	client     azblob.ServiceURL // Azure sdk client
 }
 
@@ -460,6 +469,11 @@ func parseAzurePart(metaPartFileName, prefix string) (partID int, err error) {
 	return
 }
 
+// GetMetrics returns this gateway's metrics
+func (a *azureObjects) GetMetrics(ctx context.Context) (*minio.Metrics, error) {
+	return a.metrics, nil
+}
+
 // Shutdown - save any gateway metadata to disk
 // if necessary and reload upon next restart.
 func (a *azureObjects) Shutdown(ctx context.Context) error {

cmd/gateway/gcs/gateway-gcs.go

@@ -179,6 +179,13 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 		}
 	}
 
+	metrics := minio.NewMetrics()
+
+	t := &minio.MetricsTransport{
+		Transport: minio.NewCustomHTTPTransport(),
+		Metrics:   metrics,
+	}
+
 	// Initialize a GCS client.
 	// Send user-agent in this format for Google to obtain usage insights while participating in the
 	// Google Cloud Technology Partners (https://cloud.google.com/partners/)
@@ -190,8 +197,9 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 	gcs := &gcsGateway{
 		client:    client,
 		projectID: g.projectID,
+		metrics:   metrics,
 		httpClient: &http.Client{
-			Transport: minio.NewCustomHTTPTransport(),
+			Transport: t,
 		},
 	}
 
@@ -333,6 +341,7 @@ type gcsGateway struct {
 	minio.GatewayUnsupported
 	client     *storage.Client
 	httpClient *http.Client
+	metrics    *minio.Metrics
 	projectID  string
 }
 
@@ -349,6 +358,11 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) {
 	return googleCreds[gcsProjectIDKey], err
 }
 
+// GetMetrics returns this gateway's metrics
+func (l *gcsGateway) GetMetrics(ctx context.Context) (*minio.Metrics, error) {
+	return l.metrics, nil
+}
+
 // Cleanup old files in minio.sys.tmp of the given bucket.
 func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket string) {
 	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})

cmd/gateway/s3/gateway-s3.go

@@ -106,11 +106,9 @@ func (g *S3) Name() string {
 
 const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
 const (
-	letterIdxBits           = 6                    // 6 bits to represent a letter index
+	letterIdxBits = 6                    // 6 bits to represent a letter index
-	letterIdxMask           = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
+	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
-	letterIdxMax            = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
+	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
-	minioReservedBucket     = "minio"
-	minioReservedBucketPath = minio.SlashSeparator + minioReservedBucket
 )
 
 // randString generates random names and prepends them with a known prefix.
@@ -158,32 +156,6 @@ var defaultAWSCredProviders = []credentials.Provider{
 	&credentials.EnvMinio{},
 }
 
-type metricsTransport struct {
-	transport *http.Transport
-	metrics   *minio.Metrics
-}
-
-func (s metricsTransport) RoundTrip(r *http.Request) (*http.Response, error) {
-	isS3Request := func() bool {
-		return !(minio.HasPrefix(r.URL.Path, minioReservedBucketPath) ||
-			minio.HasSuffix(r.URL.Path, ".js") || strings.Contains(r.URL.Path, "favicon.ico") ||
-			strings.Contains(r.URL.Path, ".html"))
-	}
-	if isS3Request() && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
-		s.metrics.IncRequests(r.Method)
-		s.metrics.IncBytesSent(r.ContentLength)
-	}
-	// Make the request to the server.
-	resp, err := s.transport.RoundTrip(r)
-	if err != nil {
-		return nil, err
-	}
-	if isS3Request() && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
-		s.metrics.IncBytesReceived(resp.ContentLength)
-	}
-	return resp, nil
-}
-
 // newS3 - Initializes a new client by auto probing S3 server signature.
 func newS3(urlStr string) (*miniogo.Core, error) {
 	if urlStr == "" {
@@ -237,9 +209,9 @@ func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 
 	metrics := minio.NewMetrics()
 
-	t := &metricsTransport{
+	t := &minio.MetricsTransport{
-		transport: minio.NewCustomHTTPTransport(),
+		Transport: minio.NewCustomHTTPTransport(),
-		metrics:   metrics,
+		Metrics:   metrics,
 	}
 
 	// Set custom transport

cmd/metrics.go

@@ -272,12 +272,12 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 		)
 	}
 
-	if globalIsGateway && globalGatewayName == "s3" {
+	if globalIsGateway && (globalGatewayName == "s3" || globalGatewayName == "azure" || globalGatewayName == "gcs") {
 		m, _ := globalObjectAPI.GetMetrics(context.Background())
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
 				prometheus.BuildFQName("gateway", globalGatewayName, "bytes_received"),
-				"Total number of bytes received by current MinIO S3 Gateway from AWS S3",
+				"Total number of bytes received by current MinIO Gateway "+globalGatewayName+" backend",
 				nil, nil),
 			prometheus.CounterValue,
 			float64(m.GetBytesReceived()),
@@ -285,7 +285,7 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
 				prometheus.BuildFQName("gateway", globalGatewayName, "bytes_sent"),
-				"Total number of bytes sent by current MinIO S3 Gateway to AWS S3",
+				"Total number of bytes sent by current MinIO Gateway to "+globalGatewayName+" backend",
 				nil, nil),
 			prometheus.CounterValue,
 			float64(m.GetBytesSent()),
@@ -294,7 +294,7 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
 				prometheus.BuildFQName("gateway", globalGatewayName, "requests"),
-				"Total number of requests made to AWS S3 by current MinIO S3 Gateway",
+				"Total number of requests made to "+globalGatewayName+" by current MinIO Gateway",
 				[]string{"method"}, nil),
 			prometheus.CounterValue,
 			float64(s.Get.Load()),
@@ -303,7 +303,7 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
 				prometheus.BuildFQName("gateway", globalGatewayName, "requests"),
-				"Total number of requests made to AWS S3 by current MinIO S3 Gateway",
+				"Total number of requests made to "+globalGatewayName+" by current MinIO Gateway",
 				[]string{"method"}, nil),
 			prometheus.CounterValue,
 			float64(s.Head.Load()),

docs/metrics/prometheus/README.md

@@ -147,13 +147,15 @@ MinIO Gateway instances enabled with Disk-Caching expose caching related metrics
 - `cache_hits_total`: Total number of cache hits.
 - `cache_misses_total`: Total number of cache misses.
 
-### S3 Gateway & Cache specific metrics
+### Gateway & Cache specific metrics
 
-MinIO S3 Gateway instance exposes metrics related to Gateway communication with AWS S3.
+MinIO Gateway instance exposes metrics related to Gateway communication with the cloud backend (S3, Azure & GCS Gateway).
 
-- `gateway_s3_requests`: Total number of GET & HEAD requests made to AWS S3. This metrics has a label `method` that identifies GET & HEAD Requests.
+- `gateway_<gateway_type>_requests`: Total number of GET & HEAD requests made to cloud backend. This metrics has a label `method` that identifies GET & HEAD Requests.
-- `gateway_s3_bytes_sent`: Total number of bytes sent to AWS S3 (in GET & HEAD Requests).
+- `gateway_<gateway_type>_bytes_sent`: Total number of bytes sent to cloud backend (in GET & HEAD Requests).
-- `gateway_s3_bytes_received`: Total number of bytes received from AWS S3 (in GET & HEAD Requests).
+- `gateway_<gateway_type>_bytes_received`: Total number of bytes received from cloud backend (in GET & HEAD Requests).
+
+Note that this is currently only support for Azure, S3 and GCS Gateway.
 
 ## Migration guide for the new set of metrics