Add metrics support for Azure & GCS Gateway (#8954)

We added support for caching and S3 related metrics in #8591. As
a continuation, it would be helpful to add support for Azure & GCS
gateway related metrics as well.

cmd/gateway/azure/gateway-azure.go

@@ -146,7 +146,14 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
 		return &azureObjects{}, err
 	}
 
-	httpClient := &http.Client{Transport: minio.NewCustomHTTPTransport()}
+	metrics := minio.NewMetrics()
+
+	t := &minio.MetricsTransport{
+		Transport: minio.NewCustomHTTPTransport(),
+		Metrics:   metrics,
+	}
+
+	httpClient := &http.Client{Transport: t}
 	userAgent := fmt.Sprintf("APN/1.0 MinIO/1.0 MinIO/%s", minio.Version)
 
 	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
@@ -168,6 +175,7 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
 		endpoint:   endpointURL.String(),
 		httpClient: httpClient,
 		client:     client,
+		metrics:    metrics,
 	}, nil
 }
 
@@ -357,6 +365,7 @@ type azureObjects struct {
 	minio.GatewayUnsupported
 	endpoint   string
 	httpClient *http.Client
+	metrics    *minio.Metrics
 	client     azblob.ServiceURL // Azure sdk client
 }
 
@@ -460,6 +469,11 @@ func parseAzurePart(metaPartFileName, prefix string) (partID int, err error) {
 	return
 }
 
+// GetMetrics returns this gateway's metrics
+func (a *azureObjects) GetMetrics(ctx context.Context) (*minio.Metrics, error) {
+	return a.metrics, nil
+}
+
 // Shutdown - save any gateway metadata to disk
 // if necessary and reload upon next restart.
 func (a *azureObjects) Shutdown(ctx context.Context) error {

cmd/gateway/gcs/gateway-gcs.go

@@ -179,6 +179,13 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 		}
 	}
 
+	metrics := minio.NewMetrics()
+
+	t := &minio.MetricsTransport{
+		Transport: minio.NewCustomHTTPTransport(),
+		Metrics:   metrics,
+	}
+
 	// Initialize a GCS client.
 	// Send user-agent in this format for Google to obtain usage insights while participating in the
 	// Google Cloud Technology Partners (https://cloud.google.com/partners/)
@@ -190,8 +197,9 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 	gcs := &gcsGateway{
 		client:    client,
 		projectID: g.projectID,
+		metrics:   metrics,
 		httpClient: &http.Client{
-			Transport: minio.NewCustomHTTPTransport(),
+			Transport: t,
 		},
 	}
 
@@ -333,6 +341,7 @@ type gcsGateway struct {
 	minio.GatewayUnsupported
 	client     *storage.Client
 	httpClient *http.Client
+	metrics    *minio.Metrics
 	projectID  string
 }
 
@@ -349,6 +358,11 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) {
 	return googleCreds[gcsProjectIDKey], err
 }
 
+// GetMetrics returns this gateway's metrics
+func (l *gcsGateway) GetMetrics(ctx context.Context) (*minio.Metrics, error) {
+	return l.metrics, nil
+}
+
 // Cleanup old files in minio.sys.tmp of the given bucket.
 func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket string) {
 	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})

cmd/gateway/s3/gateway-s3.go

@@ -106,11 +106,9 @@ func (g *S3) Name() string {
 
 const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
 const (
-	letterIdxBits           = 6                    // 6 bits to represent a letter index
-	letterIdxMask           = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
-	letterIdxMax            = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
-	minioReservedBucket     = "minio"
-	minioReservedBucketPath = minio.SlashSeparator + minioReservedBucket
+	letterIdxBits = 6                    // 6 bits to represent a letter index
+	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
+	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
 )
 
 // randString generates random names and prepends them with a known prefix.
@@ -158,32 +156,6 @@ var defaultAWSCredProviders = []credentials.Provider{
 	&credentials.EnvMinio{},
 }
 
-type metricsTransport struct {
-	transport *http.Transport
-	metrics   *minio.Metrics
-}
-
-func (s metricsTransport) RoundTrip(r *http.Request) (*http.Response, error) {
-	isS3Request := func() bool {
-		return !(minio.HasPrefix(r.URL.Path, minioReservedBucketPath) ||
-			minio.HasSuffix(r.URL.Path, ".js") || strings.Contains(r.URL.Path, "favicon.ico") ||
-			strings.Contains(r.URL.Path, ".html"))
-	}
-	if isS3Request() && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
-		s.metrics.IncRequests(r.Method)
-		s.metrics.IncBytesSent(r.ContentLength)
-	}
-	// Make the request to the server.
-	resp, err := s.transport.RoundTrip(r)
-	if err != nil {
-		return nil, err
-	}
-	if isS3Request() && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
-		s.metrics.IncBytesReceived(resp.ContentLength)
-	}
-	return resp, nil
-}
-
 // newS3 - Initializes a new client by auto probing S3 server signature.
 func newS3(urlStr string) (*miniogo.Core, error) {
 	if urlStr == "" {
@@ -237,9 +209,9 @@ func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 
 	metrics := minio.NewMetrics()
 
-	t := &metricsTransport{
-		transport: minio.NewCustomHTTPTransport(),
-		metrics:   metrics,
+	t := &minio.MetricsTransport{
+		Transport: minio.NewCustomHTTPTransport(),
+		Metrics:   metrics,
 	}
 
 	// Set custom transport