From bfb505aa8e852b737437d7395087f8241efdac04 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 19 Nov 2018 14:47:03 -0800 Subject: [PATCH] Refactor logging in more Go idiomatic style (#6816) This refactor brings a change which allows targets to be added in a cleaner way and also audit is now moved out. This PR also simplifies logger dependency for auditing --- cmd/acl-handlers.go | 4 +- cmd/api-datatypes.go | 2 + cmd/bucket-handlers-listobjects.go | 4 +- cmd/bucket-handlers.go | 16 +- cmd/bucket-notification-handlers.go | 6 +- cmd/bucket-policy-handlers.go | 6 +- cmd/common-main.go | 10 +- cmd/format-fs.go | 3 +- cmd/gateway-main.go | 2 +- cmd/generic-handlers.go | 25 +-- cmd/globals.go | 3 + cmd/logger/audit-logger.go | 97 ---------- cmd/logger/audit.go | 68 +++++++ cmd/logger/console.go | 219 +++++++++++++++-------- cmd/logger/logger.go | 255 +++++---------------------- cmd/logger/message/audit/entry.go | 90 ++++++++++ cmd/logger/message/log/entry.go | 50 ++++++ cmd/logger/reqinfo.go | 18 +- cmd/logger/target/console/console.go | 109 ++++++++++++ cmd/logger/{ => target/http}/http.go | 25 +-- cmd/logger/targets.go | 14 +- cmd/logger/utils.go | 20 +-- cmd/object-handlers.go | 24 +-- cmd/prepare-storage.go | 2 +- cmd/routers.go | 4 +- cmd/utils.go | 13 +- cmd/web-handlers.go | 6 +- cmd/xl-v1-healing.go | 4 +- 28 files changed, 618 insertions(+), 481 deletions(-) delete mode 100644 cmd/logger/audit-logger.go create mode 100644 cmd/logger/audit.go create mode 100644 cmd/logger/message/audit/entry.go create mode 100644 cmd/logger/message/log/entry.go create mode 100644 cmd/logger/target/console/console.go rename cmd/logger/{ => target/http}/http.go (78%) diff --git a/cmd/acl-handlers.go b/cmd/acl-handlers.go index 639e82d2d..da67f3676 100644 --- a/cmd/acl-handlers.go +++ b/cmd/acl-handlers.go @@ -57,7 +57,7 @@ type accessControlPolicy struct { func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetBucketACL") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "GetBucketACL") vars := mux.Vars(r) bucket := vars["bucket"] @@ -106,7 +106,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http. func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetObjectACL") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "GetObjectACL") vars := mux.Vars(r) bucket := vars["bucket"] diff --git a/cmd/api-datatypes.go b/cmd/api-datatypes.go index f74c64440..03102178f 100644 --- a/cmd/api-datatypes.go +++ b/cmd/api-datatypes.go @@ -23,6 +23,8 @@ import ( const ( // Response request id. responseRequestIDKey = "x-amz-request-id" + // Deployment id. + responseDeploymentIDKey = "x-minio-deployment-id" ) // ObjectIdentifier carries key name for the object to delete. diff --git a/cmd/bucket-handlers-listobjects.go b/cmd/bucket-handlers-listobjects.go index e4f1c571f..2edad2f58 100644 --- a/cmd/bucket-handlers-listobjects.go +++ b/cmd/bucket-handlers-listobjects.go @@ -59,7 +59,7 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListObjectsV2") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "ListObjectsV2") vars := mux.Vars(r) bucket := vars["bucket"] @@ -141,7 +141,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListObjectsV1") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "ListObjectsV1") vars := mux.Vars(r) bucket := vars["bucket"] diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 752a9af1e..1abc0e58b 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -89,7 +89,7 @@ func initFederatorBackend(objLayer ObjectLayer) { func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetBucketLocation") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "GetBucketLocation") vars := mux.Vars(r) bucket := vars["bucket"] @@ -139,7 +139,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r * func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListMultipartUploads") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "ListMultipartUploads") vars := mux.Vars(r) bucket := vars["bucket"] @@ -192,7 +192,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListBuckets") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "ListBuckets") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -250,7 +250,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "DeleteMultipleObjects") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "DeleteMultipleObjects") vars := mux.Vars(r) bucket := vars["bucket"] @@ -397,7 +397,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutBucket") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "PutBucket") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -475,7 +475,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PostPolicyBucket") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "PostPolicyBucket") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -703,7 +703,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "HeadBucket") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "HeadBucket") vars := mux.Vars(r) bucket := vars["bucket"] @@ -735,7 +735,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "DeleteBucket") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "DeleteBucket") vars := mux.Vars(r) bucket := vars["bucket"] diff --git a/cmd/bucket-notification-handlers.go b/cmd/bucket-notification-handlers.go index 4fbec0312..eefd804ee 100644 --- a/cmd/bucket-notification-handlers.go +++ b/cmd/bucket-notification-handlers.go @@ -44,7 +44,7 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetBucketNotification") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "GetBucketNotification") vars := mux.Vars(r) bucketName := vars["bucket"] @@ -102,7 +102,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutBucketNotification") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "PutBucketNotification") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -164,7 +164,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListenBucketNotification") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "ListenBucketNotification") // Validate if bucket exists. objAPI := api.ObjectAPI() diff --git a/cmd/bucket-policy-handlers.go b/cmd/bucket-policy-handlers.go index 06fc83f6a..7bf1c605b 100644 --- a/cmd/bucket-policy-handlers.go +++ b/cmd/bucket-policy-handlers.go @@ -40,7 +40,7 @@ const ( func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutBucketPolicy") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "PutBucketPolicy") objAPI := api.ObjectAPI() if objAPI == nil { @@ -103,7 +103,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "DeleteBucketPolicy") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "DeleteBucketPolicy") objAPI := api.ObjectAPI() if objAPI == nil { @@ -141,7 +141,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetBucketPolicy") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "GetBucketPolicy") objAPI := api.ObjectAPI() if objAPI == nil { diff --git a/cmd/common-main.go b/cmd/common-main.go index 3780240fe..d0decbef4 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -32,6 +32,8 @@ import ( "github.com/minio/minio-go/pkg/set" "github.com/minio/minio/cmd/crypto" "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/cmd/logger/target/console" + "github.com/minio/minio/cmd/logger/target/http" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/dns" xnet "github.com/minio/minio/pkg/net" @@ -54,25 +56,25 @@ func loadLoggers() { auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT") if ok { // Enable audit HTTP logging through ENV. - logger.AddAuditTarget(logger.NewHTTP(auditEndpoint, NewCustomHTTPTransport())) + logger.AddAuditTarget(http.New(auditEndpoint, NewCustomHTTPTransport())) } loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT") if ok { // Enable HTTP logging through ENV. - logger.AddTarget(logger.NewHTTP(loggerEndpoint, NewCustomHTTPTransport())) + logger.AddTarget(http.New(loggerEndpoint, NewCustomHTTPTransport())) } else { for _, l := range globalServerConfig.Logger.HTTP { if l.Enabled { // Enable http logging - logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport())) + logger.AddTarget(http.New(l.Endpoint, NewCustomHTTPTransport())) } } } if globalServerConfig.Logger.Console.Enabled { // Enable console logging - logger.AddTarget(logger.NewConsole()) + logger.AddTarget(console.New()) } } diff --git a/cmd/format-fs.go b/cmd/format-fs.go index 6cc7f2297..e07e14b35 100644 --- a/cmd/format-fs.go +++ b/cmd/format-fs.go @@ -24,7 +24,6 @@ import ( "path" "time" - "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/lock" ) @@ -273,7 +272,7 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er rlk.Close() return nil, err } - logger.SetDeploymentID(id) + globalDeploymentID = id return rlk, nil } } diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 8fb0076bc..a136495ab 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -239,7 +239,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) { loadLoggers() // This is only to uniquely identify each gateway deployments. - logger.SetDeploymentID(os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID")) + globalDeploymentID = os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID") var cacheConfig = globalServerConfig.GetCacheConfig() if len(cacheConfig.Drives) > 0 { diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 650dc4ac3..2a6b0789a 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -723,24 +723,25 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) { l.handler.ServeHTTP(w, r) } -// requestIDHeaderHandler sets x-amz-request-id header. -// Previously, this value was set right before a response -// was sent to the client.So, logger and Error response XML -// were not using this value. -// This is set here so that this header can be logged as -// part of the log entry and Error response XML. -type requestIDHeaderHandler struct { +// customHeaderHandler sets x-amz-request-id, x-minio-deployment-id header. +// Previously, this value was set right before a response was sent to +// the client. So, logger and Error response XML were not using this +// value. This is set here so that this header can be logged as +// part of the log entry, Error response XML and auditing. +type customHeaderHandler struct { handler http.Handler } -func addrequestIDHeader(h http.Handler) http.Handler { - return requestIDHeaderHandler{handler: h} +func addCustomHeaders(h http.Handler) http.Handler { + return customHeaderHandler{handler: h} } -func (s requestIDHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Set unique request ID for each response. +func (s customHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Set custom headers such as x-amz-request-id and x-minio-deployment-id + // for each request. w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow())) - s.handler.ServeHTTP(w, r) + w.Header().Set(responseDeploymentIDKey, globalDeploymentID) + s.handler.ServeHTTP(logger.NewResponseWriter(w), r) } type securityHeaderHandler struct { diff --git a/cmd/globals.go b/cmd/globals.go index d9c35c581..aa7d2330d 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -255,6 +255,9 @@ var ( // OPA policy system. globalPolicyOPA *iampolicy.Opa + // Deployment ID - unique per deployment + globalDeploymentID string + // Add new variable global values here. ) diff --git a/cmd/logger/audit-logger.go b/cmd/logger/audit-logger.go deleted file mode 100644 index f09e0b409..000000000 --- a/cmd/logger/audit-logger.go +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2018 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package logger - -import ( - "context" - "net/http" - "strings" - "time" -) - -// Represents the current version of audit log structure. -const auditLogVersion = "1" - -// AuditEntry - audit entry logs. -type AuditEntry struct { - Version string `json:"version"` - DeploymentID string `json:"deploymentid,omitempty"` - Time string `json:"time"` - API *api `json:"api,omitempty"` - RemoteHost string `json:"remotehost,omitempty"` - RequestID string `json:"requestID,omitempty"` - UserAgent string `json:"userAgent,omitempty"` - ReqQuery map[string]string `json:"requestQuery,omitempty"` - ReqHeader map[string]string `json:"requestHeader,omitempty"` - RespHeader map[string]string `json:"responseHeader,omitempty"` -} - -// AuditTargets is the list of enabled audit loggers -var AuditTargets = []LoggingTarget{} - -// AddAuditTarget adds a new audit logger target to the -// list of enabled loggers -func AddAuditTarget(t LoggingTarget) { - AuditTargets = append(AuditTargets, t) -} - -// AuditLog - logs audit logs to all targets. -func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request) { - if Disable { - return - } - - req := GetReqInfo(ctx) - if req == nil { - req = &ReqInfo{API: "SYSTEM"} - } - - reqQuery := make(map[string]string) - for k, v := range r.URL.Query() { - reqQuery[k] = strings.Join(v, ",") - } - reqHeader := make(map[string]string) - for k, v := range r.Header { - reqHeader[k] = strings.Join(v, ",") - } - respHeader := make(map[string]string) - for k, v := range w.Header() { - respHeader[k] = strings.Join(v, ",") - } - - // Send audit logs only to http targets. - for _, t := range AuditTargets { - t.send(AuditEntry{ - Version: auditLogVersion, - DeploymentID: deploymentID, - RemoteHost: req.RemoteHost, - RequestID: req.RequestID, - UserAgent: req.UserAgent, - Time: time.Now().UTC().Format(time.RFC3339Nano), - API: &api{ - Name: req.API, - Args: &args{ - Bucket: req.BucketName, - Object: req.ObjectName, - }, - }, - ReqQuery: reqQuery, - ReqHeader: reqHeader, - RespHeader: respHeader, - }) - } -} diff --git a/cmd/logger/audit.go b/cmd/logger/audit.go new file mode 100644 index 000000000..b74ee8c85 --- /dev/null +++ b/cmd/logger/audit.go @@ -0,0 +1,68 @@ +/* + * Minio Cloud Storage, (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logger + +import ( + "net/http" + + "github.com/minio/minio/cmd/logger/message/audit" +) + +// ResponseWriter - is a wrapper to trap the http response status code. +type ResponseWriter struct { + http.ResponseWriter + statusCode int +} + +// NewResponseWriter - returns a wrapped response writer to trap +// http status codes for auditiing purposes. +func NewResponseWriter(w http.ResponseWriter) *ResponseWriter { + return &ResponseWriter{w, http.StatusOK} +} + +// WriteHeader - writes http status code +func (lrw *ResponseWriter) WriteHeader(code int) { + lrw.statusCode = code + lrw.ResponseWriter.WriteHeader(code) +} + +// Flush - Calls the underlying Flush. +func (lrw *ResponseWriter) Flush() { + lrw.ResponseWriter.(http.Flusher).Flush() +} + +// AuditTargets is the list of enabled audit loggers +var AuditTargets = []Target{} + +// AddAuditTarget adds a new audit logger target to the +// list of enabled loggers +func AddAuditTarget(t Target) { + AuditTargets = append(AuditTargets, t) +} + +// AuditLog - logs audit logs to all audit targets. +func AuditLog(w http.ResponseWriter, r *http.Request, api string) { + var statusCode int + lrw, ok := w.(*ResponseWriter) + if ok { + statusCode = lrw.statusCode + } + // Send audit logs only to http targets. + for _, t := range AuditTargets { + t.Send(audit.ToEntry(w, r, api, statusCode)) + } +} diff --git a/cmd/logger/console.go b/cmd/logger/console.go index 507e0cdd7..329130dfc 100644 --- a/cmd/logger/console.go +++ b/cmd/logger/console.go @@ -19,87 +19,168 @@ package logger import ( "encoding/json" "fmt" + "os" "strings" "time" + + c "github.com/minio/mc/pkg/console" + "github.com/minio/minio/cmd/logger/message/log" ) -// ConsoleTarget implements loggerTarget to send log -// in plain or json format to the standard output. -type ConsoleTarget struct{} +// Console interface describes the methods that need to be implemented to satisfy the interface requirements. +type Console interface { + json(msg string, args ...interface{}) + quiet(msg string, args ...interface{}) + pretty(msg string, args ...interface{}) +} -func (c *ConsoleTarget) send(e interface{}) error { - entry, ok := e.(logEntry) - if !ok { - return fmt.Errorf("Uexpected log entry structure %#v", e) +func consoleLog(console Console, msg string, args ...interface{}) { + switch { + case jsonFlag: + // Strip escape control characters from json message + msg = ansiRE.ReplaceAllLiteralString(msg, "") + console.json(msg, args...) + case quietFlag: + console.quiet(msg, args...) + default: + console.pretty(msg, args...) } - if jsonFlag { - logJSON, err := json.Marshal(&entry) - if err != nil { - return err +} + +// Fatal prints only fatal error message with no stack trace +// it will be called for input validation failures +func Fatal(err error, msg string, data ...interface{}) { + fatal(err, msg, data...) +} + +func fatal(err error, msg string, data ...interface{}) { + var errMsg string + if msg != "" { + errMsg = errorFmtFunc(fmt.Sprintf(msg, data...), err, jsonFlag) + } else { + errMsg = err.Error() + } + consoleLog(fatalMessage, errMsg) +} + +var fatalMessage fatalMsg + +type fatalMsg struct { +} + +func (f fatalMsg) json(msg string, args ...interface{}) { + logJSON, err := json.Marshal(&log.Entry{ + Level: FatalLvl.String(), + Time: time.Now().UTC().Format(time.RFC3339Nano), + Trace: &log.Trace{Message: fmt.Sprintf(msg, args...), Source: []string{getSource(6)}}, + }) + if err != nil { + panic(err) + } + fmt.Println(string(logJSON)) + + os.Exit(1) + +} + +func (f fatalMsg) quiet(msg string, args ...interface{}) { + f.pretty(msg, args...) +} + +var ( + logTag = "ERROR" + logBanner = ColorBgRed(ColorFgWhite(ColorBold(logTag))) + " " + emptyBanner = ColorBgRed(strings.Repeat(" ", len(logTag))) + " " + bannerWidth = len(logTag) + 1 +) + +func (f fatalMsg) pretty(msg string, args ...interface{}) { + // Build the passed error message + errMsg := fmt.Sprintf(msg, args...) + + tagPrinted := false + + // Print the error message: the following code takes care + // of splitting error text and always pretty printing the + // red banner along with the error message. Since the error + // message itself contains some colored text, we needed + // to use some ANSI control escapes to cursor color state + // and freely move in the screen. + for _, line := range strings.Split(errMsg, "\n") { + if len(line) == 0 { + // No more text to print, just quit. + break } - fmt.Println(string(logJSON)) - return nil - } - trace := make([]string, len(entry.Trace.Source)) - - // Add a sequence number and formatting for each stack trace - // No formatting is required for the first entry - for i, element := range entry.Trace.Source { - trace[i] = fmt.Sprintf("%8v: %s", i+1, element) - } - - tagString := "" - for key, value := range entry.Trace.Variables { - if value != "" { - if tagString != "" { - tagString += ", " + for { + // Save the attributes of the current cursor helps + // us save the text color of the passed error message + ansiSaveAttributes() + // Print banner with or without the log tag + if !tagPrinted { + fmt.Print(logBanner) + tagPrinted = true + } else { + fmt.Print(emptyBanner) } - tagString += key + "=" + value + // Restore the text color of the error message + ansiRestoreAttributes() + ansiMoveRight(bannerWidth) + // Continue error message printing + fmt.Println(line) + break } } - apiString := "API: " + entry.API.Name + "(" - if entry.API.Args != nil && entry.API.Args.Bucket != "" { - apiString = apiString + "bucket=" + entry.API.Args.Bucket - } - if entry.API.Args != nil && entry.API.Args.Object != "" { - apiString = apiString + ", object=" + entry.API.Args.Object - } - apiString += ")" - timeString := "Time: " + time.Now().Format(loggerTimeFormat) - - var requestID string - if entry.RequestID != "" { - requestID = "\nRequestID: " + entry.RequestID - } - - var remoteHost string - if entry.RemoteHost != "" { - remoteHost = "\nRemoteHost: " + entry.RemoteHost - } - - var userAgent string - if entry.UserAgent != "" { - userAgent = "\nUserAgent: " + entry.UserAgent - } - - if len(entry.Trace.Variables) > 0 { - tagString = "\n " + tagString - } - - var msg = colorFgRed(colorBold(entry.Trace.Message)) - var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s", - apiString, timeString, requestID, remoteHost, userAgent, - msg, tagString, strings.Join(trace, "\n")) - - fmt.Println(output) - return nil + // Exit because this is a fatal error message + os.Exit(1) } -// NewConsole initializes a new logger target -// which prints log directly in the standard -// output. -func NewConsole() LoggingTarget { - return &ConsoleTarget{} +type infoMsg struct{} + +var info infoMsg + +func (i infoMsg) json(msg string, args ...interface{}) { + logJSON, err := json.Marshal(&log.Entry{ + Level: InformationLvl.String(), + Message: fmt.Sprintf(msg, args...), + Time: time.Now().UTC().Format(time.RFC3339Nano), + }) + if err != nil { + panic(err) + } + fmt.Println(string(logJSON)) +} + +func (i infoMsg) quiet(msg string, args ...interface{}) { + i.pretty(msg, args...) +} + +func (i infoMsg) pretty(msg string, args ...interface{}) { + c.Printf(msg, args...) +} + +// Info : +func Info(msg string, data ...interface{}) { + consoleLog(info, msg+"\n", data...) +} + +var startupMessage startUpMsg + +type startUpMsg struct { +} + +func (s startUpMsg) json(msg string, args ...interface{}) { +} + +func (s startUpMsg) quiet(msg string, args ...interface{}) { +} + +func (s startUpMsg) pretty(msg string, args ...interface{}) { + c.Printf(msg, args...) +} + +// StartupMessage : +func StartupMessage(msg string, data ...interface{}) { + consoleLog(startupMessage, msg+"\n", data...) } diff --git a/cmd/logger/logger.go b/cmd/logger/logger.go index 92169a190..aee3c3835 100644 --- a/cmd/logger/logger.go +++ b/cmd/logger/logger.go @@ -18,23 +18,20 @@ package logger import ( "context" - "encoding/json" "fmt" "go/build" - "os" "path/filepath" "runtime" "strings" "time" - c "github.com/minio/mc/pkg/console" + "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio/cmd/logger/message/log" ) // Disable disables all logging, false by default. (used for "go test") var Disable = false -var trimStrings []string - // Level type type Level int8 @@ -45,7 +42,10 @@ const ( FatalLvl ) -const loggerTimeFormat string = "15:04:05 MST 01/02/2006" +var trimStrings []string + +// TimeFormat - logging time format. +const TimeFormat string = "15:04:05 MST 01/02/2006" // List of error strings to be ignored by LogIf const ( @@ -91,79 +91,33 @@ func (level Level) String() string { return lvlStr } -// Console interface describes the methods that needs to be implemented to satisfy the interface requirements. -type Console interface { - json(msg string, args ...interface{}) - quiet(msg string, args ...interface{}) - pretty(msg string, args ...interface{}) -} - -func consoleLog(console Console, msg string, args ...interface{}) { - switch { - case jsonFlag: - // Strip escape control characters from json message - msg = ansiRE.ReplaceAllLiteralString(msg, "") - console.json(msg, args...) - case quiet: - console.quiet(msg, args...) - default: - console.pretty(msg, args...) - } -} - -type traceEntry struct { - Message string `json:"message,omitempty"` - Source []string `json:"source,omitempty"` - Variables map[string]string `json:"variables,omitempty"` -} - -type args struct { - Bucket string `json:"bucket,omitempty"` - Object string `json:"object,omitempty"` - Metadata map[string]string `json:"metadata,omitempty"` -} - -type api struct { - Name string `json:"name,omitempty"` - Args *args `json:"args,omitempty"` -} - -type logEntry struct { - DeploymentID string `json:"deploymentid,omitempty"` - Level string `json:"level"` - Time string `json:"time"` - API *api `json:"api,omitempty"` - RemoteHost string `json:"remotehost,omitempty"` - RequestID string `json:"requestID,omitempty"` - UserAgent string `json:"userAgent,omitempty"` - Message string `json:"message,omitempty"` - Trace *traceEntry `json:"error,omitempty"` -} - -// quiet: Hide startup messages if enabled +// quietFlag: Hide startup messages if enabled // jsonFlag: Display in JSON format, if enabled var ( - quiet, jsonFlag bool + quietFlag, jsonFlag bool // Custom function to format error errorFmtFunc func(string, error, bool) string - - deploymentID string ) -// SetDeploymentID - Used to set the deployment ID, in XL and FS mode -func SetDeploymentID(id string) { - deploymentID = id -} - // EnableQuiet - turns quiet option on. func EnableQuiet() { - quiet = true + quietFlag = true } // EnableJSON - outputs logs in json format. func EnableJSON() { jsonFlag = true - quiet = true + quietFlag = true +} + +// IsJSON - returns true if jsonFlag is true +func IsJSON() bool { + return jsonFlag +} + +// IsQuiet - returns true if quietFlag is true +func IsQuiet() bool { + return quietFlag } // RegisterUIError registers the specified rendering function. This latter @@ -172,6 +126,17 @@ func RegisterUIError(f func(string, error, bool) string) { errorFmtFunc = f } +// Remove any duplicates and return unique entries. +func uniqueEntries(paths []string) []string { + m := make(set.StringSet) + for _, p := range paths { + if !m.Contains(p) { + m.Add(p) + } + } + return m.ToSlice() +} + // Init sets the trimStrings to possible GOPATHs // and GOROOT directories. Also append github.com/minio/minio // This is done to clean up the filename, when stack trace is @@ -326,20 +291,30 @@ func logIf(ctx context.Context, err error) { // Get the cause for the Error message := err.Error() - entry := logEntry{ - DeploymentID: deploymentID, + entry := log.Entry{ + DeploymentID: req.DeploymentID, Level: ErrorLvl.String(), RemoteHost: req.RemoteHost, RequestID: req.RequestID, UserAgent: req.UserAgent, Time: time.Now().UTC().Format(time.RFC3339Nano), - API: &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}}, - Trace: &traceEntry{Message: message, Source: trace, Variables: tags}, + API: &log.API{ + Name: API, + Args: &log.Args{ + Bucket: req.BucketName, + Object: req.ObjectName, + }, + }, + Trace: &log.Trace{ + Message: message, + Source: trace, + Variables: tags, + }, } // Iterate over all logger targets to send the log entry for _, t := range Targets { - t.send(entry) + t.Send(entry) } } @@ -362,141 +337,3 @@ func FatalIf(err error, msg string, data ...interface{}) { } fatal(err, msg, data...) } - -// Fatal prints only fatal error message without no stack trace -// it will be called for input validation failures -func Fatal(err error, msg string, data ...interface{}) { - fatal(err, msg, data...) -} - -func fatal(err error, msg string, data ...interface{}) { - var errMsg string - if msg != "" { - errMsg = errorFmtFunc(fmt.Sprintf(msg, data...), err, jsonFlag) - } else { - errMsg = err.Error() - } - consoleLog(fatalMessage, errMsg) -} - -var fatalMessage fatalMsg - -type fatalMsg struct { -} - -func (f fatalMsg) json(msg string, args ...interface{}) { - logJSON, err := json.Marshal(&logEntry{ - Level: FatalLvl.String(), - Time: time.Now().UTC().Format(time.RFC3339Nano), - Trace: &traceEntry{Message: fmt.Sprintf(msg, args...), Source: []string{getSource(6)}}, - }) - if err != nil { - panic(err) - } - fmt.Println(string(logJSON)) - - os.Exit(1) - -} - -func (f fatalMsg) quiet(msg string, args ...interface{}) { - f.pretty(msg, args...) -} - -var ( - logTag = "ERROR" - logBanner = colorBgRed(colorFgWhite(colorBold(logTag))) + " " - emptyBanner = colorBgRed(strings.Repeat(" ", len(logTag))) + " " - bannerWidth = len(logTag) + 1 -) - -func (f fatalMsg) pretty(msg string, args ...interface{}) { - // Build the passed error message - errMsg := fmt.Sprintf(msg, args...) - - tagPrinted := false - - // Print the error message: the following code takes care - // of splitting error text and always pretty printing the - // red banner along with the error message. Since the error - // message itself contains some colored text, we needed - // to use some ANSI control escapes to cursor color state - // and freely move in the screen. - for _, line := range strings.Split(errMsg, "\n") { - if len(line) == 0 { - // No more text to print, just quit. - break - } - - for { - // Save the attributes of the current cursor helps - // us save the text color of the passed error message - ansiSaveAttributes() - // Print banner with or without the log tag - if !tagPrinted { - fmt.Print(logBanner) - tagPrinted = true - } else { - fmt.Print(emptyBanner) - } - // Restore the text color of the error message - ansiRestoreAttributes() - ansiMoveRight(bannerWidth) - // Continue error message printing - fmt.Println(line) - break - } - } - - // Exit because this is a fatal error message - os.Exit(1) -} - -type infoMsg struct{} - -var info infoMsg - -func (i infoMsg) json(msg string, args ...interface{}) { - logJSON, err := json.Marshal(&logEntry{ - Level: InformationLvl.String(), - Message: fmt.Sprintf(msg, args...), - Time: time.Now().UTC().Format(time.RFC3339Nano), - }) - if err != nil { - panic(err) - } - fmt.Println(string(logJSON)) -} - -func (i infoMsg) quiet(msg string, args ...interface{}) { - i.pretty(msg, args...) -} - -func (i infoMsg) pretty(msg string, args ...interface{}) { - c.Printf(msg, args...) -} - -// Info : -func Info(msg string, data ...interface{}) { - consoleLog(info, msg+"\n", data...) -} - -var startupMessage startUpMsg - -type startUpMsg struct { -} - -func (s startUpMsg) json(msg string, args ...interface{}) { -} - -func (s startUpMsg) quiet(msg string, args ...interface{}) { -} - -func (s startUpMsg) pretty(msg string, args ...interface{}) { - c.Printf(msg, args...) -} - -// StartupMessage : -func StartupMessage(msg string, data ...interface{}) { - consoleLog(startupMessage, msg+"\n", data...) -} diff --git a/cmd/logger/message/audit/entry.go b/cmd/logger/message/audit/entry.go new file mode 100644 index 000000000..d3d57e80c --- /dev/null +++ b/cmd/logger/message/audit/entry.go @@ -0,0 +1,90 @@ +/* + * Minio Cloud Storage, (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package audit + +import ( + "net/http" + "strings" + "time" + + "github.com/gorilla/mux" + "github.com/minio/minio/pkg/handlers" +) + +// Version - represents the current version of audit log structure. +const Version = "1" + +// Entry - audit entry logs. +type Entry struct { + Version string `json:"version"` + DeploymentID string `json:"deploymentid,omitempty"` + Time string `json:"time"` + API struct { + Name string `json:"name,omitempty"` + Bucket string `json:"bucket,omitempty"` + Object string `json:"object,omitempty"` + Status string `json:"status,omitempty"` + StatusCode int `json:"statusCode,omitempty"` + } `json:"api"` + RemoteHost string `json:"remotehost,omitempty"` + RequestID string `json:"requestID,omitempty"` + UserAgent string `json:"userAgent,omitempty"` + ReqQuery map[string]string `json:"requestQuery,omitempty"` + ReqHeader map[string]string `json:"requestHeader,omitempty"` + RespHeader map[string]string `json:"responseHeader,omitempty"` +} + +// ToEntry - constructs an audit entry object. +func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int) Entry { + vars := mux.Vars(r) + bucket := vars["bucket"] + object := vars["object"] + + reqQuery := make(map[string]string) + for k, v := range r.URL.Query() { + reqQuery[k] = strings.Join(v, ",") + } + reqHeader := make(map[string]string) + for k, v := range r.Header { + reqHeader[k] = strings.Join(v, ",") + } + respHeader := make(map[string]string) + for k, v := range w.Header() { + respHeader[k] = strings.Join(v, ",") + } + respHeader["Etag"] = strings.Trim(respHeader["Etag"], `"`) + + entry := Entry{ + Version: Version, + DeploymentID: w.Header().Get("x-minio-deployment-id"), + RemoteHost: handlers.GetSourceIP(r), + RequestID: w.Header().Get("x-amz-request-id"), + UserAgent: r.UserAgent(), + Time: time.Now().UTC().Format(time.RFC3339Nano), + ReqQuery: reqQuery, + ReqHeader: reqHeader, + RespHeader: respHeader, + } + + entry.API.Name = api + entry.API.Bucket = bucket + entry.API.Object = object + entry.API.Status = http.StatusText(statusCode) + entry.API.StatusCode = statusCode + + return entry +} diff --git a/cmd/logger/message/log/entry.go b/cmd/logger/message/log/entry.go new file mode 100644 index 000000000..c0ef70947 --- /dev/null +++ b/cmd/logger/message/log/entry.go @@ -0,0 +1,50 @@ +/* + * Minio Cloud Storage, (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package log + +// Args - defines the arguments for the API. +type Args struct { + Bucket string `json:"bucket,omitempty"` + Object string `json:"object,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// Trace - defines the trace. +type Trace struct { + Message string `json:"message,omitempty"` + Source []string `json:"source,omitempty"` + Variables map[string]string `json:"variables,omitempty"` +} + +// API - defines the api type and its args. +type API struct { + Name string `json:"name,omitempty"` + Args *Args `json:"args,omitempty"` +} + +// Entry - defines fields and values of each log entry. +type Entry struct { + DeploymentID string `json:"deploymentid,omitempty"` + Level string `json:"level"` + Time string `json:"time"` + API *API `json:"api,omitempty"` + RemoteHost string `json:"remotehost,omitempty"` + RequestID string `json:"requestID,omitempty"` + UserAgent string `json:"userAgent,omitempty"` + Message string `json:"message,omitempty"` + Trace *Trace `json:"error,omitempty"` +} diff --git a/cmd/logger/reqinfo.go b/cmd/logger/reqinfo.go index ae932969e..0b1de81d7 100644 --- a/cmd/logger/reqinfo.go +++ b/cmd/logger/reqinfo.go @@ -35,22 +35,24 @@ type KeyVal struct { // ReqInfo stores the request info. type ReqInfo struct { - RemoteHost string // Client Host/IP - UserAgent string // User Agent - RequestID string // x-amz-request-id - API string // API name - GetObject PutObject NewMultipartUpload etc. - BucketName string // Bucket name - ObjectName string // Object name - tags []KeyVal // Any additional info not accommodated by above fields + RemoteHost string // Client Host/IP + UserAgent string // User Agent + DeploymentID string // x-minio-deployment-id + RequestID string // x-amz-request-id + API string // API name - GetObject PutObject NewMultipartUpload etc. + BucketName string // Bucket name + ObjectName string // Object name + tags []KeyVal // Any additional info not accommodated by above fields sync.RWMutex } // NewReqInfo : -func NewReqInfo(remoteHost, userAgent, requestID, api, bucket, object string) *ReqInfo { +func NewReqInfo(remoteHost, userAgent, deploymentID, requestID, api, bucket, object string) *ReqInfo { req := ReqInfo{} req.RemoteHost = remoteHost req.UserAgent = userAgent req.API = api + req.DeploymentID = deploymentID req.RequestID = requestID req.BucketName = bucket req.ObjectName = object diff --git a/cmd/logger/target/console/console.go b/cmd/logger/target/console/console.go new file mode 100644 index 000000000..a90c68f9a --- /dev/null +++ b/cmd/logger/target/console/console.go @@ -0,0 +1,109 @@ +/* + * Minio Cloud Storage, (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package console + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/cmd/logger/message/log" +) + +// Target implements loggerTarget to send log +// in plain or json format to the standard output. +type Target struct{} + +// Send log message 'e' to console +func (c *Target) Send(e interface{}) error { + entry, ok := e.(log.Entry) + if !ok { + return fmt.Errorf("Uexpected log entry structure %#v", e) + } + if logger.IsJSON() { + logJSON, err := json.Marshal(&entry) + if err != nil { + return err + } + fmt.Println(string(logJSON)) + return nil + } + + trace := make([]string, len(entry.Trace.Source)) + + // Add a sequence number and formatting for each stack trace + // No formatting is required for the first entry + for i, element := range entry.Trace.Source { + trace[i] = fmt.Sprintf("%8v: %s", i+1, element) + } + + tagString := "" + for key, value := range entry.Trace.Variables { + if value != "" { + if tagString != "" { + tagString += ", " + } + tagString += key + "=" + value + } + } + + apiString := "API: " + entry.API.Name + "(" + if entry.API.Args != nil && entry.API.Args.Bucket != "" { + apiString = apiString + "bucket=" + entry.API.Args.Bucket + } + if entry.API.Args != nil && entry.API.Args.Object != "" { + apiString = apiString + ", object=" + entry.API.Args.Object + } + apiString += ")" + timeString := "Time: " + time.Now().Format(logger.TimeFormat) + + var requestID string + if entry.RequestID != "" { + requestID = "\nRequestID: " + entry.RequestID + } + + var remoteHost string + if entry.RemoteHost != "" { + remoteHost = "\nRemoteHost: " + entry.RemoteHost + } + + var userAgent string + if entry.UserAgent != "" { + userAgent = "\nUserAgent: " + entry.UserAgent + } + + if len(entry.Trace.Variables) > 0 { + tagString = "\n " + tagString + } + + var msg = logger.ColorFgRed(logger.ColorBold(entry.Trace.Message)) + var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s", + apiString, timeString, requestID, remoteHost, userAgent, + msg, tagString, strings.Join(trace, "\n")) + + fmt.Println(output) + return nil +} + +// New initializes a new logger target +// which prints log directly in the standard +// output. +func New() *Target { + return &Target{} +} diff --git a/cmd/logger/http.go b/cmd/logger/target/http/http.go similarity index 78% rename from cmd/logger/http.go rename to cmd/logger/target/http/http.go index 549281008..8b914674f 100644 --- a/cmd/logger/http.go +++ b/cmd/logger/target/http/http.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package logger +package http import ( "bytes" @@ -22,24 +22,24 @@ import ( "errors" "io" "io/ioutil" - "net/http" + gohttp "net/http" ) -// HTTPTarget implements loggerTarget and sends the json +// Target implements logger.Target and sends the json // format of a log entry to the configured http endpoint. // An internal buffer of logs is maintained but when the // buffer is full, new logs are just ignored and an error // is returned to the caller. -type HTTPTarget struct { +type Target struct { // Channel of log entries logCh chan interface{} // HTTP(s) endpoint endpoint string - client http.Client + client gohttp.Client } -func (h *HTTPTarget) startHTTPLogger() { +func (h *Target) startHTTPLogger() { // Create a routine which sends json logs received // from an internal channel. go func() { @@ -49,7 +49,7 @@ func (h *HTTPTarget) startHTTPLogger() { continue } - req, err := http.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON)) + req, err := gohttp.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON)) req.Header.Set("Content-Type", "application/json") resp, err := h.client.Do(req) @@ -65,12 +65,12 @@ func (h *HTTPTarget) startHTTPLogger() { }() } -// NewHTTP initializes a new logger target which +// New initializes a new logger target which // sends log over http to the specified endpoint -func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget { - h := HTTPTarget{ +func New(endpoint string, transport *gohttp.Transport) *Target { + h := Target{ endpoint: endpoint, - client: http.Client{ + client: gohttp.Client{ Transport: transport, }, logCh: make(chan interface{}, 10000), @@ -80,7 +80,8 @@ func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget { return &h } -func (h *HTTPTarget) send(entry interface{}) error { +// Send log message 'e' to http target. +func (h *Target) Send(entry interface{}) error { select { case h.logCh <- entry: default: diff --git a/cmd/logger/targets.go b/cmd/logger/targets.go index 4a9a0a62c..962dae148 100644 --- a/cmd/logger/targets.go +++ b/cmd/logger/targets.go @@ -16,18 +16,18 @@ package logger -// LoggingTarget is the entity that we will receive -// a single log entry and send it to the log target -// e.g. send the log to a http server -type LoggingTarget interface { - send(entry interface{}) error +// Target is the entity that we will receive +// a single log entry and Send it to the log target +// e.g. Send the log to a http server +type Target interface { + Send(entry interface{}) error } // Targets is the set of enabled loggers -var Targets = []LoggingTarget{} +var Targets = []Target{} // AddTarget adds a new logger target to the // list of enabled loggers -func AddTarget(t LoggingTarget) { +func AddTarget(t Target) { Targets = append(Targets, t) } diff --git a/cmd/logger/utils.go b/cmd/logger/utils.go index 5bcac3a5b..1c1c0622d 100644 --- a/cmd/logger/utils.go +++ b/cmd/logger/utils.go @@ -33,25 +33,25 @@ var ( return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd()) } - colorBold = func() func(a ...interface{}) string { + ColorBold = func() func(a ...interface{}) string { if isTerminal() { return color.New(color.Bold).SprintFunc() } return fmt.Sprint }() - colorFgRed = func() func(format string, a ...interface{}) string { + ColorFgRed = func() func(format string, a ...interface{}) string { if isTerminal() { return color.New(color.FgRed).SprintfFunc() } return fmt.Sprintf }() - colorBgRed = func() func(format string, a ...interface{}) string { + ColorBgRed = func() func(format string, a ...interface{}) string { if isTerminal() { return color.New(color.BgRed).SprintfFunc() } return fmt.Sprintf }() - colorFgWhite = func() func(format string, a ...interface{}) string { + ColorFgWhite = func() func(format string, a ...interface{}) string { if isTerminal() { return color.New(color.FgWhite).SprintfFunc() } @@ -83,17 +83,5 @@ func ansiRestoreAttributes() { if isTerminal() { ansiEscape("8") } -} -func uniqueEntries(paths []string) []string { - found := map[string]bool{} - unqiue := []string{} - - for v := range paths { - if _, ok := found[paths[v]]; !ok { - found[paths[v]] = true - unqiue = append(unqiue, paths[v]) - } - } - return unqiue } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 22a825228..eba163d2c 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -79,7 +79,7 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "SelectObject") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "SelectObject") // Fetch object stat info. objectAPI := api.ObjectAPI() @@ -305,7 +305,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetObject") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "GetObject") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -473,7 +473,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "HeadObject") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "HeadObject") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -656,7 +656,7 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "CopyObject") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "CopyObject") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -1028,7 +1028,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutObject") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "PutObject") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -1291,7 +1291,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "NewMultipartUpload") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "NewMultipartUpload") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -1388,7 +1388,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "CopyObjectPart") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "CopyObjectPart") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -1632,7 +1632,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutObjectPart") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "PutObjectPart") objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -1883,7 +1883,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "AbortMultipartUpload") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "AbortMultipartUpload") vars := mux.Vars(r) bucket := vars["bucket"] @@ -1929,7 +1929,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListObjectParts") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "ListObjectParts") vars := mux.Vars(r) bucket := vars["bucket"] @@ -2008,7 +2008,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "CompleteMultipartUpload") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "CompleteMultipartUpload") vars := mux.Vars(r) bucket := vars["bucket"] @@ -2187,7 +2187,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "DeleteObject") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "DeleteObject") vars := mux.Vars(r) bucket := vars["bucket"] diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index 41fc8bffc..d07dd11d0 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -210,7 +210,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointLi } } - logger.SetDeploymentID(format.ID) + globalDeploymentID = format.ID if err = formatXLFixLocalDeploymentID(context.Background(), storageDisks, format); err != nil { return nil, err diff --git a/cmd/routers.go b/cmd/routers.go index 5b59e5ee9..761b8c02b 100644 --- a/cmd/routers.go +++ b/cmd/routers.go @@ -47,8 +47,8 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) { // List of some generic handlers which are applied for all incoming requests. var globalHandlers = []HandlerFunc{ - // set x-amz-request-id header. - addrequestIDHeader, + // set x-amz-request-id, x-minio-deployment-id header. + addCustomHeaders, // set HTTP security headers such as Content-Security-Policy. addSecurityHeaders, // Forward path style requests to actual host in a bucket federated setup. diff --git a/cmd/utils.go b/cmd/utils.go index 81117a356..956822a80 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -397,12 +397,13 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont object = prefix } reqInfo := &logger.ReqInfo{ - RequestID: w.Header().Get(responseRequestIDKey), - RemoteHost: handlers.GetSourceIP(r), - UserAgent: r.UserAgent(), - API: api, - BucketName: bucket, - ObjectName: object, + DeploymentID: w.Header().Get(responseDeploymentIDKey), + RequestID: w.Header().Get(responseRequestIDKey), + RemoteHost: handlers.GetSourceIP(r), + UserAgent: r.UserAgent(), + API: api, + BucketName: bucket, + ObjectName: object, } return logger.SetReqInfo(context.Background(), reqInfo) } diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index dc6bdb926..8b0fc5381 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -707,7 +707,7 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs, func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "WebUpload") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "WebUpload") objectAPI := web.ObjectAPI() if objectAPI == nil { @@ -845,7 +845,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "WebDownload") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "WebDownload") var wg sync.WaitGroup objectAPI := web.ObjectAPI() @@ -1027,7 +1027,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) { } ctx := newContext(r, w, "WebDownloadZip") - defer logger.AuditLog(ctx, w, r) + defer logger.AuditLog(w, r, "WebDownloadZip") var wg sync.WaitGroup objectAPI := web.ObjectAPI() diff --git a/cmd/xl-v1-healing.go b/cmd/xl-v1-healing.go index 245672417..3c077ecf0 100644 --- a/cmd/xl-v1-healing.go +++ b/cmd/xl-v1-healing.go @@ -636,9 +636,9 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu reqInfo := logger.GetReqInfo(ctx) var newReqInfo *logger.ReqInfo if reqInfo != nil { - newReqInfo = logger.NewReqInfo(reqInfo.RemoteHost, reqInfo.UserAgent, reqInfo.RequestID, reqInfo.API, bucket, object) + newReqInfo = logger.NewReqInfo(reqInfo.RemoteHost, reqInfo.UserAgent, reqInfo.DeploymentID, reqInfo.RequestID, reqInfo.API, bucket, object) } else { - newReqInfo = logger.NewReqInfo("", "", "", "Heal", bucket, object) + newReqInfo = logger.NewReqInfo("", "", globalDeploymentID, "", "Heal", bucket, object) } healCtx := logger.SetReqInfo(context.Background(), newReqInfo)