diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 8bc44704a..b6e0fb2e1 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -19,9 +19,6 @@ package cmd import ( "net/http" - "reflect" - "runtime" - "strings" "github.com/klauspost/compress/gzhttp" "github.com/klauspost/compress/gzip" @@ -69,14 +66,6 @@ func (h hFlag) Has(flag hFlag) bool { return h&flag != 0 } -func getHandlerName(f http.HandlerFunc) string { - name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() - name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.adminAPIHandlers.") - name = strings.TrimSuffix(name, "Handler-fm") - name = strings.TrimSuffix(name, "-fm") - return name -} - // adminMiddleware performs some common admin handler functionality for all // handlers: // @@ -86,9 +75,13 @@ func getHandlerName(f http.HandlerFunc) string { // // - sets up call to send AuditLog // -// Note that, while this is a middleware function (i.e. it takes a handler -// function and returns one), due to flags being passed based on required -// conditions, it is done per-"handler function registration" in the router. +// While this is a middleware function (i.e. it takes a handler function and +// returns one), due to flags being passed based on required conditions, it is +// done per-"handler function registration" in the router. +// +// The passed in handler function must be a method of `adminAPIHandlers` for the +// name displayed in logs and trace to be accurate. The name is extracted via +// reflection. // // When no flags are passed, gzip compression, http tracing of headers and // checking of object layer availability are all enabled. Use flags to modify @@ -100,10 +93,8 @@ func adminMiddleware(f http.HandlerFunc, flags ...hFlag) http.HandlerFunc { handlerFlags |= flag } - // Get name of the handler using reflection. NOTE: The passed in handler - // function must be a method of `adminAPIHandlers` for this extraction to - // work as expected. - handlerName := getHandlerName(f) + // Get name of the handler using reflection. + handlerName := getHandlerName(f, "adminAPIHandlers") var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { // Update request context with `logger.ReqInfo`. diff --git a/cmd/api-router.go b/cmd/api-router.go index 085ac151a..6943ed423 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -18,14 +18,11 @@ package cmd import ( - "compress/gzip" "net" "net/http" - "github.com/klauspost/compress/gzhttp" consoleapi "github.com/minio/console/api" xhttp "github.com/minio/minio/internal/http" - "github.com/minio/minio/internal/logger" "github.com/minio/mux" "github.com/minio/pkg/v2/wildcard" "github.com/rs/cors" @@ -171,6 +168,87 @@ var rejectedBucketAPIs = []rejectedAPI{ }, } +// Set of s3 handler options as bit flags. +type s3HFlag uint8 + +const ( + // when provided, disables Gzip compression. + noGZS3HFlag = 1 << iota + + // when provided, enables only tracing of headers. Otherwise, both headers + // and body are traced. + traceHdrsS3HFlag + + // when provided, disables throttling via the `maxClients` middleware. + noThrottleS3HFlag +) + +func (h s3HFlag) has(flag s3HFlag) bool { + // Use bitwise-AND and check if the result is non-zero. + return h&flag != 0 +} + +// s3APIMiddleware - performs some common handler functionality for S3 API +// handlers. +// +// It is set per-"handler function registration" in the router to allow for +// behavior modification via flags. +// +// This middleware always calls `collectAPIStats` to collect API stats. +// +// The passed in handler function must be a method of `objectAPIHandlers` for +// the name displayed in logs and trace to be accurate. The name is extracted +// via reflection. +// +// When **no** flags are passed, the behavior is to trace both headers and body, +// gzip the response and throttle the handler via `maxClients`. Each of these +// can be disabled via the corresponding `s3HFlag`. +// +// CAUTION: for requests involving large req/resp bodies ensure to pass the +// `traceHdrsS3HFlag`, otherwise both headers and body will be traced, causing +// high memory usage! +func s3APIMiddleware(f http.HandlerFunc, flags ...s3HFlag) http.HandlerFunc { + // Collect all flags with bitwise-OR and assign operator + var handlerFlags s3HFlag + for _, flag := range flags { + handlerFlags |= flag + } + + // Get name of the handler using reflection. + handlerName := getHandlerName(f, "objectAPIHandlers") + + var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + // Wrap the actual handler with the appropriate tracing middleware. + var tracedHandler http.HandlerFunc + if handlerFlags.has(traceHdrsS3HFlag) { + tracedHandler = httpTraceHdrs(f) + } else { + tracedHandler = httpTraceAll(f) + } + + // Skip wrapping with the gzip middleware if specified. + var gzippedHandler http.HandlerFunc = tracedHandler + if !handlerFlags.has(noGZS3HFlag) { + gzippedHandler = gzipHandler(gzippedHandler) + } + + // Skip wrapping with throttling middleware if specified. + var throttledHandler http.HandlerFunc = gzippedHandler + if !handlerFlags.has(noThrottleS3HFlag) { + throttledHandler = maxClients(throttledHandler) + } + + // Collect API stats using the API name got from reflection in + // `getHandlerName`. + statsCollectedHandler := collectAPIStats(handlerName, throttledHandler) + + // Call the final handler. + statsCollectedHandler(w, r) + } + + return handler +} + // registerAPIRouter - registers S3 compatible APIs. func registerAPIRouter(router *mux.Router) { // Initialize API. @@ -208,12 +286,6 @@ func registerAPIRouter(router *mux.Router) { } routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) - gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed)) - if err != nil { - // Static params, so this is very unlikely. - logger.Fatal(err, "Unable to initialize server") - } - for _, router := range routers { // Register all rejected object APIs for _, r := range rejectedObjAPIs { @@ -225,240 +297,307 @@ func registerAPIRouter(router *mux.Router) { // Object operations // HeadObject - router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc( - collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler))))) + router.Methods(http.MethodHead).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.HeadObjectHandler)) + // GetObjectAttributes - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectattributes", maxClients(gz(httpTraceHdrs(api.GetObjectAttributesHandler))))).Queries("attributes", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectAttributesHandler, traceHdrsS3HFlag)). + Queries("attributes", "") + // CopyObjectPart router.Methods(http.MethodPut).Path("/{object:.+}"). HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?"). - HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))). + HandlerFunc(s3APIMiddleware(api.CopyObjectPartHandler)). Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") // PutObjectPart - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectPartHandler, traceHdrsS3HFlag)). + Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") // ListObjectParts - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.ListObjectPartsHandler)). + Queries("uploadId", "{uploadId:.*}") // CompleteMultipartUpload - router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - collectAPIStats("completemultipartupload", maxClients(gz(httpTraceAll(api.CompleteMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}") + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.CompleteMultipartUploadHandler)). + Queries("uploadId", "{uploadId:.*}") // NewMultipartUpload - router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - collectAPIStats("newmultipartupload", maxClients(gz(httpTraceAll(api.NewMultipartUploadHandler))))).Queries("uploads", "") + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.NewMultipartUploadHandler)). + Queries("uploads", "") // AbortMultipartUpload - router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - collectAPIStats("abortmultipartupload", maxClients(gz(httpTraceAll(api.AbortMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}") + router.Methods(http.MethodDelete).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.AbortMultipartUploadHandler)). + Queries("uploadId", "{uploadId:.*}") // GetObjectACL - this is a dummy call. - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectacl", maxClients(gz(httpTraceHdrs(api.GetObjectACLHandler))))).Queries("acl", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectACLHandler, traceHdrsS3HFlag)). + Queries("acl", "") // PutObjectACL - this is a dummy call. - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjectacl", maxClients(gz(httpTraceHdrs(api.PutObjectACLHandler))))).Queries("acl", "") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectACLHandler, traceHdrsS3HFlag)). + Queries("acl", "") // GetObjectTagging - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjecttagging", maxClients(gz(httpTraceHdrs(api.GetObjectTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectTaggingHandler, traceHdrsS3HFlag)). + Queries("tagging", "") // PutObjectTagging - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjecttagging", maxClients(gz(httpTraceHdrs(api.PutObjectTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectTaggingHandler, traceHdrsS3HFlag)). + Queries("tagging", "") // DeleteObjectTagging - router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - collectAPIStats("deleteobjecttagging", maxClients(gz(httpTraceHdrs(api.DeleteObjectTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodDelete).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.DeleteObjectTaggingHandler, traceHdrsS3HFlag)). + Queries("tagging", "") // SelectObjectContent - router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - collectAPIStats("selectobjectcontent", maxClients(gz(httpTraceHdrs(api.SelectObjectContentHandler))))).Queries("select", "").Queries("select-type", "2") + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.SelectObjectContentHandler, traceHdrsS3HFlag)). + Queries("select", "").Queries("select-type", "2") // GetObjectRetention - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectretention", maxClients(gz(httpTraceAll(api.GetObjectRetentionHandler))))).Queries("retention", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectRetentionHandler)). + Queries("retention", "") // GetObjectLegalHold - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectLegalHoldHandler)). + Queries("legal-hold", "") // GetObject with lambda ARNs - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectlambda", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectLambdaHandler, traceHdrsS3HFlag)). + Queries("lambdaArn", "{lambdaArn:.*}") // GetObject - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler))))) + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectHandler, traceHdrsS3HFlag)) // CopyObject - router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc( - collectAPIStats("copyobject", maxClients(gz(httpTraceAll(api.CopyObjectHandler))))) + router.Methods(http.MethodPut).Path("/{object:.+}"). + HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?"). + HandlerFunc(s3APIMiddleware(api.CopyObjectHandler)) // PutObjectRetention - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjectretention", maxClients(gz(httpTraceAll(api.PutObjectRetentionHandler))))).Queries("retention", "") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectRetentionHandler)). + Queries("retention", "") // PutObjectLegalHold - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjectlegalhold", maxClients(gz(httpTraceAll(api.PutObjectLegalHoldHandler))))).Queries("legal-hold", "") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectLegalHoldHandler)). + Queries("legal-hold", "") // PutObject with auto-extract support for zip - router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc( - collectAPIStats("putobjectextract", maxClients(gz(httpTraceHdrs(api.PutObjectExtractHandler))))) + router.Methods(http.MethodPut).Path("/{object:.+}"). + HeadersRegexp(xhttp.AmzSnowballExtract, "true"). + HandlerFunc(s3APIMiddleware(api.PutObjectExtractHandler, traceHdrsS3HFlag)) // PutObject - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler))))) + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectHandler, traceHdrsS3HFlag)) // DeleteObject - router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - collectAPIStats("deleteobject", maxClients(gz(httpTraceAll(api.DeleteObjectHandler))))) + router.Methods(http.MethodDelete).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.DeleteObjectHandler)) // PostRestoreObject - router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - collectAPIStats("postrestoreobject", maxClients(gz(httpTraceAll(api.PostRestoreObjectHandler))))).Queries("restore", "") + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PostRestoreObjectHandler)). + Queries("restore", "") // Bucket operations // GetBucketLocation - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketlocation", maxClients(gz(httpTraceAll(api.GetBucketLocationHandler))))).Queries("location", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketLocationHandler)). + Queries("location", "") // GetBucketPolicy - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketpolicy", maxClients(gz(httpTraceAll(api.GetBucketPolicyHandler))))).Queries("policy", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketPolicyHandler)). + Queries("policy", "") // GetBucketLifecycle - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketlifecycle", maxClients(gz(httpTraceAll(api.GetBucketLifecycleHandler))))).Queries("lifecycle", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketLifecycleHandler)). + Queries("lifecycle", "") // GetBucketEncryption - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketencryption", maxClients(gz(httpTraceAll(api.GetBucketEncryptionHandler))))).Queries("encryption", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketEncryptionHandler)). + Queries("encryption", "") // GetBucketObjectLockConfig - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketobjectlockconfig", maxClients(gz(httpTraceAll(api.GetBucketObjectLockConfigHandler))))).Queries("object-lock", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketObjectLockConfigHandler)). + Queries("object-lock", "") // GetBucketReplicationConfig - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketreplicationconfig", maxClients(gz(httpTraceAll(api.GetBucketReplicationConfigHandler))))).Queries("replication", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketReplicationConfigHandler)). + Queries("replication", "") // GetBucketVersioning - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketversioning", maxClients(gz(httpTraceAll(api.GetBucketVersioningHandler))))).Queries("versioning", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketVersioningHandler)). + Queries("versioning", "") // GetBucketNotification - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketnotification", maxClients(gz(httpTraceAll(api.GetBucketNotificationHandler))))).Queries("notification", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketNotificationHandler)). + Queries("notification", "") // ListenNotification - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)). + Queries("events", "{events:.*}") // ResetBucketReplicationStatus - MinIO extension API - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("resetbucketreplicationstatus", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStatusHandler))))).Queries("replication-reset-status", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStatusHandler)). + Queries("replication-reset-status", "") // Dummy Bucket Calls // GetBucketACL -- this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketacl", maxClients(gz(httpTraceAll(api.GetBucketACLHandler))))).Queries("acl", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketACLHandler)). + Queries("acl", "") // PutBucketACL -- this is a dummy call. - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketacl", maxClients(gz(httpTraceAll(api.PutBucketACLHandler))))).Queries("acl", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)). + Queries("acl", "") // GetBucketCors - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketcors", maxClients(gz(httpTraceAll(api.GetBucketCorsHandler))))).Queries("cors", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)). + Queries("cors", "") // GetBucketWebsiteHandler - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketwebsite", maxClients(gz(httpTraceAll(api.GetBucketWebsiteHandler))))).Queries("website", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketWebsiteHandler)). + Queries("website", "") // GetBucketAccelerateHandler - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketaccelerate", maxClients(gz(httpTraceAll(api.GetBucketAccelerateHandler))))).Queries("accelerate", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketAccelerateHandler)). + Queries("accelerate", "") // GetBucketRequestPaymentHandler - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketrequestpayment", maxClients(gz(httpTraceAll(api.GetBucketRequestPaymentHandler))))).Queries("requestPayment", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketRequestPaymentHandler)). + Queries("requestPayment", "") // GetBucketLoggingHandler - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketlogging", maxClients(gz(httpTraceAll(api.GetBucketLoggingHandler))))).Queries("logging", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketLoggingHandler)). + Queries("logging", "") // GetBucketTaggingHandler - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbuckettagging", maxClients(gz(httpTraceAll(api.GetBucketTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketTaggingHandler)). + Queries("tagging", "") // DeleteBucketWebsiteHandler - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketwebsite", maxClients(gz(httpTraceAll(api.DeleteBucketWebsiteHandler))))).Queries("website", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketWebsiteHandler)). + Queries("website", "") // DeleteBucketTaggingHandler - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebuckettagging", maxClients(gz(httpTraceAll(api.DeleteBucketTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketTaggingHandler)). + Queries("tagging", "") // ListMultipartUploads - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listmultipartuploads", maxClients(gz(httpTraceAll(api.ListMultipartUploadsHandler))))).Queries("uploads", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListMultipartUploadsHandler)). + Queries("uploads", "") // ListObjectsV2M - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectsv2M", maxClients(gz(httpTraceAll(api.ListObjectsV2MHandler))))).Queries("list-type", "2", "metadata", "true") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectsV2MHandler)). + Queries("list-type", "2", "metadata", "true") // ListObjectsV2 - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectsV2Handler)). + Queries("list-type", "2") // ListObjectVersions - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectversionsM", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectVersionsMHandler)). + Queries("versions", "", "metadata", "true") // ListObjectVersions - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectVersionsHandler)). + Queries("versions", "") // GetBucketPolicyStatus - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketpolicystatus", maxClients(gz(httpTraceAll(api.GetBucketPolicyStatusHandler))))).Queries("policyStatus", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketPolicyStatusHandler)). + Queries("policyStatus", "") // PutBucketLifecycle - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketlifecycle", maxClients(gz(httpTraceAll(api.PutBucketLifecycleHandler))))).Queries("lifecycle", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketLifecycleHandler)). + Queries("lifecycle", "") // PutBucketReplicationConfig - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketreplicationconfig", maxClients(gz(httpTraceAll(api.PutBucketReplicationConfigHandler))))).Queries("replication", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketReplicationConfigHandler)). + Queries("replication", "") // PutBucketEncryption - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketencryption", maxClients(gz(httpTraceAll(api.PutBucketEncryptionHandler))))).Queries("encryption", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketEncryptionHandler)). + Queries("encryption", "") // PutBucketPolicy - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketpolicy", maxClients(gz(httpTraceAll(api.PutBucketPolicyHandler))))).Queries("policy", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketPolicyHandler)). + Queries("policy", "") // PutBucketObjectLockConfig - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketobjectlockconfig", maxClients(gz(httpTraceAll(api.PutBucketObjectLockConfigHandler))))).Queries("object-lock", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketObjectLockConfigHandler)). + Queries("object-lock", "") // PutBucketTaggingHandler - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbuckettagging", maxClients(gz(httpTraceAll(api.PutBucketTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketTaggingHandler)). + Queries("tagging", "") // PutBucketVersioning - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketversioning", maxClients(gz(httpTraceAll(api.PutBucketVersioningHandler))))).Queries("versioning", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketVersioningHandler)). + Queries("versioning", "") // PutBucketNotification - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketnotification", maxClients(gz(httpTraceAll(api.PutBucketNotificationHandler))))).Queries("notification", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketNotificationHandler)). + Queries("notification", "") // ResetBucketReplicationStart - MinIO extension API - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("resetbucketreplicationstart", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStartHandler))))).Queries("replication-reset", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStartHandler)). + Queries("replication-reset", "") // PutBucket - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucket", maxClients(gz(httpTraceAll(api.PutBucketHandler))))) + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketHandler)) // HeadBucket - router.Methods(http.MethodHead).HandlerFunc( - collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler))))) + router.Methods(http.MethodHead). + HandlerFunc(s3APIMiddleware(api.HeadBucketHandler)) // PostPolicy - router.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool { - return isRequestPostPolicySignatureV4(r) - }).HandlerFunc(collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler))))) + router.Methods(http.MethodPost). + MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool { + return isRequestPostPolicySignatureV4(r) + }). + HandlerFunc(s3APIMiddleware(api.PostPolicyBucketHandler, traceHdrsS3HFlag)) // DeleteMultipleObjects - router.Methods(http.MethodPost).HandlerFunc( - collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "") + router.Methods(http.MethodPost). + HandlerFunc(s3APIMiddleware(api.DeleteMultipleObjectsHandler)). + Queries("delete", "") // DeleteBucketPolicy - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketpolicy", maxClients(gz(httpTraceAll(api.DeleteBucketPolicyHandler))))).Queries("policy", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketPolicyHandler)). + Queries("policy", "") // DeleteBucketReplication - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketreplicationconfig", maxClients(gz(httpTraceAll(api.DeleteBucketReplicationConfigHandler))))).Queries("replication", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketReplicationConfigHandler)). + Queries("replication", "") // DeleteBucketLifecycle - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketlifecycle", maxClients(gz(httpTraceAll(api.DeleteBucketLifecycleHandler))))).Queries("lifecycle", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketLifecycleHandler)). + Queries("lifecycle", "") // DeleteBucketEncryption - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketencryption", maxClients(gz(httpTraceAll(api.DeleteBucketEncryptionHandler))))).Queries("encryption", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketEncryptionHandler)). + Queries("encryption", "") // DeleteBucket - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucket", maxClients(gz(httpTraceAll(api.DeleteBucketHandler))))) + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketHandler)) // MinIO extension API for replication. // - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketreplicationmetricsv2", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsV2Handler))))).Queries("replication-metrics", "2") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsV2Handler)). + Queries("replication-metrics", "2") // deprecated handler - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsHandler)). + Queries("replication-metrics", "") // ValidateBucketReplicationCreds - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("validatebucketreplicationcreds", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ValidateBucketReplicationCredsHandler)). + Queries("replication-check", "") // Register rejected bucket APIs for _, r := range rejectedBucketAPIs { @@ -468,24 +607,25 @@ func registerAPIRouter(router *mux.Router) { } // S3 ListObjectsV1 (Legacy) - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectsv1", maxClients(gz(httpTraceAll(api.ListObjectsV1Handler))))) + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectsV1Handler)) } // Root operation // ListenNotification - apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc( - collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}") + apiRouter.Methods(http.MethodGet).Path(SlashSeparator). + HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)). + Queries("events", "{events:.*}") // ListBuckets - apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc( - collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler))))) + apiRouter.Methods(http.MethodGet).Path(SlashSeparator). + HandlerFunc(s3APIMiddleware(api.ListBucketsHandler)) // S3 browser with signature v4 adds '//' for ListBuckets request, so rather // than failing with UnknownAPIRequest we simply handle it for now. - apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc( - collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler))))) + apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator). + HandlerFunc(s3APIMiddleware(api.ListBucketsHandler)) // If none of the routes match add default error handler routes apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler)) diff --git a/cmd/api-utils.go b/cmd/api-utils.go index 90f16071c..ab191f067 100644 --- a/cmd/api-utils.go +++ b/cmd/api-utils.go @@ -18,6 +18,10 @@ package cmd import ( + "fmt" + "net/http" + "reflect" + "runtime" "strings" ) @@ -100,3 +104,16 @@ func s3EncodeName(name, encodingType string) string { } return name } + +// getHandlerName returns the name of the handler function. It takes the type +// name as a string to clean up the name retrieved via reflection. This function +// only works correctly when the type is present in the cmd package. +func getHandlerName(f http.HandlerFunc, cmdType string) string { + name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() + + packageName := fmt.Sprintf("github.com/minio/minio/cmd.%s.", cmdType) + name = strings.TrimPrefix(name, packageName) + name = strings.TrimSuffix(name, "Handler-fm") + name = strings.TrimSuffix(name, "-fm") + return name +} diff --git a/cmd/http-stats.go b/cmd/http-stats.go index 08fb9ba52..1393636f6 100644 --- a/cmd/http-stats.go +++ b/cmd/http-stats.go @@ -19,6 +19,7 @@ package cmd import ( "net/http" + "strings" "sync" "sync/atomic" @@ -326,7 +327,7 @@ func (stats *HTTPAPIStats) Get(api string) int { } // Load returns the recorded stats. -func (stats *HTTPAPIStats) Load() map[string]int { +func (stats *HTTPAPIStats) Load(toLower bool) map[string]int { if stats == nil { return map[string]int{} } @@ -336,6 +337,9 @@ func (stats *HTTPAPIStats) Load() map[string]int { apiStats := make(map[string]int, len(stats.apiStats)) for k, v := range stats.apiStats { + if toLower { + k = strings.ToLower(k) + } apiStats[k] = v } return apiStats @@ -373,7 +377,7 @@ func (st *HTTPStats) incS3RequestsIncoming() { } // Converts http stats into struct to be sent back to the client. -func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats { +func (st *HTTPStats) toServerHTTPStats(toLowerKeys bool) ServerHTTPStats { serverStats := ServerHTTPStats{} serverStats.S3RequestsIncoming = atomic.SwapUint64(&st.s3RequestsIncoming, 0) serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue) @@ -382,22 +386,22 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats { serverStats.TotalS3RejectedHeader = atomic.LoadUint64(&st.rejectedRequestsHeader) serverStats.TotalS3RejectedInvalid = atomic.LoadUint64(&st.rejectedRequestsInvalid) serverStats.CurrentS3Requests = ServerHTTPAPIStats{ - APIStats: st.currentS3Requests.Load(), + APIStats: st.currentS3Requests.Load(toLowerKeys), } serverStats.TotalS3Requests = ServerHTTPAPIStats{ - APIStats: st.totalS3Requests.Load(), + APIStats: st.totalS3Requests.Load(toLowerKeys), } serverStats.TotalS3Errors = ServerHTTPAPIStats{ - APIStats: st.totalS3Errors.Load(), + APIStats: st.totalS3Errors.Load(toLowerKeys), } serverStats.TotalS34xxErrors = ServerHTTPAPIStats{ - APIStats: st.totalS34xxErrors.Load(), + APIStats: st.totalS34xxErrors.Load(toLowerKeys), } serverStats.TotalS35xxErrors = ServerHTTPAPIStats{ - APIStats: st.totalS35xxErrors.Load(), + APIStats: st.totalS35xxErrors.Load(toLowerKeys), } serverStats.TotalS3Canceled = ServerHTTPAPIStats{ - APIStats: st.totalS3Canceled.Load(), + APIStats: st.totalS3Canceled.Load(toLowerKeys), } return serverStats } diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index c18ef497c..3ae7cc8ad 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -1827,7 +1827,10 @@ func getGoMetrics() *MetricsGroup { // getHistogramMetrics fetches histogram metrics and returns it in a []Metric // Note: Typically used in MetricGroup.RegisterRead -func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription) []Metric { +// +// The last parameter is added for compatibility - if true it lowercases the +// `api` label values. +func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription, toLowerAPILabels bool) []Metric { ch := make(chan prometheus.Metric) go func() { defer xioutil.SafeClose(ch) @@ -1851,7 +1854,11 @@ func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription) for _, b := range h.Bucket { labels := make(map[string]string) for _, lp := range dtoMetric.GetLabel() { - labels[*lp.Name] = *lp.Value + if *lp.Name == "api" && toLowerAPILabels { + labels[*lp.Name] = strings.ToLower(*lp.Value) + } else { + labels[*lp.Name] = *lp.Value + } } labels["le"] = fmt.Sprintf("%.3f", *b.UpperBound) metric := Metric{ @@ -1881,7 +1888,8 @@ func getBucketTTFBMetric() *MetricsGroup { cacheInterval: 10 * time.Second, } mg.RegisterRead(func(ctx context.Context) []Metric { - return getHistogramMetrics(bucketHTTPRequestsDuration, getBucketTTFBDistributionMD()) + return getHistogramMetrics(bucketHTTPRequestsDuration, + getBucketTTFBDistributionMD(), true) }) return mg } @@ -1891,7 +1899,8 @@ func getS3TTFBMetric() *MetricsGroup { cacheInterval: 10 * time.Second, } mg.RegisterRead(func(ctx context.Context) []Metric { - return getHistogramMetrics(httpRequestsDuration, getS3TTFBDistributionMD()) + return getHistogramMetrics(httpRequestsDuration, + getS3TTFBDistributionMD(), true) }) return mg } @@ -2918,7 +2927,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { } mg.RegisterRead(func(ctx context.Context) (metrics []Metric) { if !mg.metricsGroupOpts.bucketOnly { - httpStats := globalHTTPStats.toServerHTTPStats() + httpStats := globalHTTPStats.toServerHTTPStats(true) metrics = make([]Metric, 0, 3+ len(httpStats.CurrentS3Requests.APIStats)+ len(httpStats.TotalS3Requests.APIStats)+ @@ -3014,7 +3023,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { } httpStats := globalBucketHTTPStats.load(bucket) - for k, v := range httpStats.currentS3Requests.Load() { + for k, v := range httpStats.currentS3Requests.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3RequestsInFlightMD(), Value: float64(v), @@ -3022,7 +3031,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { }) } - for k, v := range httpStats.totalS3Requests.Load() { + for k, v := range httpStats.totalS3Requests.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3RequestsTotalMD(), Value: float64(v), @@ -3030,7 +3039,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { }) } - for k, v := range httpStats.totalS3Canceled.Load() { + for k, v := range httpStats.totalS3Canceled.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3RequestsCanceledMD(), Value: float64(v), @@ -3038,7 +3047,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { }) } - for k, v := range httpStats.totalS34xxErrors.Load() { + for k, v := range httpStats.totalS34xxErrors.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3Requests4xxErrorsMD(), Value: float64(v), @@ -3046,7 +3055,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { }) } - for k, v := range httpStats.totalS35xxErrors.Load() { + for k, v := range httpStats.totalS35xxErrors.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3Requests5xxErrorsMD(), Value: float64(v), diff --git a/cmd/metrics-v2_test.go b/cmd/metrics-v2_test.go index c91c209fc..03f01e85f 100644 --- a/cmd/metrics-v2_test.go +++ b/cmd/metrics-v2_test.go @@ -80,7 +80,7 @@ func TestGetHistogramMetrics(t *testing.T) { } } - metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD()) + metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false) // additional labels for +Inf for all histogram metrics if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) { t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics)) diff --git a/cmd/metrics.go b/cmd/metrics.go index d38fd27c7..a198593b4 100644 --- a/cmd/metrics.go +++ b/cmd/metrics.go @@ -190,7 +190,7 @@ func healingMetricsPrometheus(ch chan<- prometheus.Metric) { // collects http metrics for MinIO server in Prometheus specific format // and sends to given channel func httpMetricsPrometheus(ch chan<- prometheus.Metric) { - httpStats := globalHTTPStats.toServerHTTPStats() + httpStats := globalHTTPStats.toServerHTTPStats(true) for api, value := range httpStats.CurrentS3Requests.APIStats { ch <- prometheus.MustNewConstMetric( diff --git a/cmd/tier.go b/cmd/tier.go index 890dc7026..f785ce845 100644 --- a/cmd/tier.go +++ b/cmd/tier.go @@ -159,7 +159,7 @@ var ( ) func (t *tierMetrics) Report() []Metric { - metrics := getHistogramMetrics(t.histogram, tierTTLBMD) + metrics := getHistogramMetrics(t.histogram, tierTTLBMD, true) t.RLock() defer t.RUnlock() for tier, stat := range t.requestsCount { diff --git a/docs/bucket/replication/delete-replication.sh b/docs/bucket/replication/delete-replication.sh index b8c33c7ed..2eadf52f7 100755 --- a/docs/bucket/replication/delete-replication.sh +++ b/docs/bucket/replication/delete-replication.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +echo "Running $0" + if [ -n "$TEST_DEBUG" ]; then set -x fi diff --git a/docs/bucket/replication/setup_2site_existing_replication.sh b/docs/bucket/replication/setup_2site_existing_replication.sh index fe977e7c5..5a5e026c0 100755 --- a/docs/bucket/replication/setup_2site_existing_replication.sh +++ b/docs/bucket/replication/setup_2site_existing_replication.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +echo "Running $0" + set -x trap 'catch $LINENO' ERR diff --git a/docs/bucket/replication/setup_3site_replication.sh b/docs/bucket/replication/setup_3site_replication.sh index 193fc8be1..6aa45f582 100755 --- a/docs/bucket/replication/setup_3site_replication.sh +++ b/docs/bucket/replication/setup_3site_replication.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +echo "Running $0" + if [ -n "$TEST_DEBUG" ]; then set -x fi @@ -46,11 +48,11 @@ unset MINIO_KMS_KES_KEY_NAME go install -v ) -wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && +wget -q -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x mc if [ ! -f mc.RELEASE.2021-03-12T03-36-59Z ]; then - wget -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z && + wget -q -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z && chmod +x mc.RELEASE.2021-03-12T03-36-59Z fi diff --git a/docs/bucket/replication/sio-error.sh b/docs/bucket/replication/sio-error.sh index e7106e93a..f7ebe323e 100755 --- a/docs/bucket/replication/sio-error.sh +++ b/docs/bucket/replication/sio-error.sh @@ -1,11 +1,13 @@ #!/bin/bash +echo "Running $0" + set -e set -x export CI=1 -make || exit -1 +make || exit 255 killall -9 minio || true