From b0c9ae74902cfbf2e658ba422a3d7df843e3f1e2 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 12 Oct 2018 12:25:59 -0700 Subject: [PATCH] Add audit logging for S3 and Web handlers (#6571) This PR brings an additional logger implementation called AuditLog which logs to http targets The intention is to use AuditLog to log all incoming requests, this is used as a mechanism by external log collection entities for processing Minio requests. --- cmd/acl-handlers.go | 5 +++ cmd/bucket-handlers-listobjects.go | 5 +++ cmd/bucket-handlers.go | 20 ++++++++++ cmd/bucket-notification-handlers.go | 6 +++ cmd/bucket-policy-handlers.go | 6 +++ cmd/common-main.go | 7 ++++ cmd/gateway-common.go | 11 +++++- cmd/gateway-main.go | 6 +++ cmd/gateway/s3/gateway-s3.go | 14 ++++++- cmd/logger/console.go | 6 ++- cmd/logger/http.go | 7 ++-- cmd/logger/logger.go | 55 ++++++++++++++++++++++++-- cmd/logger/targets.go | 2 +- cmd/notification.go | 1 + cmd/object-handlers.go | 60 +++++++++++++++++++++++++++++ cmd/utils.go | 2 +- cmd/web-handlers.go | 36 ++++++++++++++--- 17 files changed, 232 insertions(+), 17 deletions(-) diff --git a/cmd/acl-handlers.go b/cmd/acl-handlers.go index 718531c1b..cea36659d 100644 --- a/cmd/acl-handlers.go +++ b/cmd/acl-handlers.go @@ -21,6 +21,7 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/policy" ) @@ -56,6 +57,8 @@ type accessControlPolicy struct { func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetBucketACL") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] @@ -103,6 +106,8 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http. func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetObjectACL") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] diff --git a/cmd/bucket-handlers-listobjects.go b/cmd/bucket-handlers-listobjects.go index e827d54d6..ccda15713 100644 --- a/cmd/bucket-handlers-listobjects.go +++ b/cmd/bucket-handlers-listobjects.go @@ -21,6 +21,7 @@ import ( "github.com/gorilla/mux" "github.com/minio/minio/cmd/crypto" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/policy" ) @@ -58,6 +59,8 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListObjectsV2") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] @@ -137,6 +140,8 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListObjectsV1") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index b55caf456..3c4bfd930 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -89,6 +89,8 @@ func initFederatorBackend(objLayer ObjectLayer) { func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetBucketLocation") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] @@ -137,6 +139,8 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r * func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListMultipartUploads") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] @@ -184,6 +188,8 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListBuckets") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -384,6 +390,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutBucket") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -460,6 +468,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PostPolicyBucket") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -671,6 +681,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h default: writeSuccessNoContent(w) } + + for k, v := range objInfo.UserDefined { + logger.GetReqInfo(ctx).SetTags(k, v) + } + + logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag) } // HeadBucketHandler - HEAD Bucket @@ -682,6 +698,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "HeadBucket") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] @@ -712,6 +730,8 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "DeleteBucket") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] diff --git a/cmd/bucket-notification-handlers.go b/cmd/bucket-notification-handlers.go index 7f176f597..6195d268a 100644 --- a/cmd/bucket-notification-handlers.go +++ b/cmd/bucket-notification-handlers.go @@ -44,6 +44,8 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetBucketNotification") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucketName := vars["bucket"] @@ -96,6 +98,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutBucketNotification") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -156,6 +160,8 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListenBucketNotification") + defer logger.AuditLog(ctx, r) + // Validate if bucket exists. objAPI := api.ObjectAPI() if objAPI == nil { diff --git a/cmd/bucket-policy-handlers.go b/cmd/bucket-policy-handlers.go index c61a4b249..92c0aa586 100644 --- a/cmd/bucket-policy-handlers.go +++ b/cmd/bucket-policy-handlers.go @@ -40,6 +40,8 @@ const ( func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutBucketPolicy") + defer logger.AuditLog(ctx, r) + objAPI := api.ObjectAPI() if objAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -101,6 +103,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "DeleteBucketPolicy") + defer logger.AuditLog(ctx, r) + objAPI := api.ObjectAPI() if objAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -137,6 +141,8 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetBucketPolicy") + defer logger.AuditLog(ctx, r) + objAPI := api.ObjectAPI() if objAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) diff --git a/cmd/common-main.go b/cmd/common-main.go index 91d558453..c4e5e19aa 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -49,10 +49,17 @@ func checkUpdate(mode string) { // Load logger targets based on user's configuration func loadLoggers() { + if endpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT"); ok { + // Enable http logging through ENV, this is specifically added gateway audit logging. + logger.AddTarget(logger.NewHTTP(endpoint, NewCustomHTTPTransport())) + return + } + if globalServerConfig.Logger.Console.Enabled { // Enable console logging logger.AddTarget(logger.NewConsole()) } + for _, l := range globalServerConfig.Logger.HTTP { if l.Enabled { // Enable http logging diff --git a/cmd/gateway-common.go b/cmd/gateway-common.go index abdcf025b..c37b2ea67 100644 --- a/cmd/gateway-common.go +++ b/cmd/gateway-common.go @@ -224,7 +224,16 @@ func FromMinioClientListBucketResultToV2Info(bucket string, result minio.ListBuc } } -// ToMinioClientMetadata converts metadata to map[string][]string +// ToMinioClientObjectInfoMetadata convertes metadata to map[string][]string +func ToMinioClientObjectInfoMetadata(metadata map[string]string) map[string][]string { + mm := make(map[string][]string, len(metadata)) + for k, v := range metadata { + mm[http.CanonicalHeaderKey(k)] = []string{v} + } + return mm +} + +// ToMinioClientMetadata converts metadata to map[string]string func ToMinioClientMetadata(metadata map[string]string) map[string]string { mm := make(map[string]string) for k, v := range metadata { diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 8ae650abf..372a74d4a 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -228,6 +228,12 @@ func StartGateway(ctx *cli.Context, gw Gateway) { globalServerConfig = srvCfg globalServerConfigMu.Unlock() + // Load logger subsystem + loadLoggers() + + // This is only to uniquely identify each gateway deployments. + logger.SetDeploymentID(os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID")) + var cacheConfig = globalServerConfig.GetCacheConfig() if len(cacheConfig.Drives) > 0 { var err error diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index a47d482d7..c0e120439 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -71,6 +71,9 @@ ENVIRONMENT VARIABLES: MINIO_CACHE_EXPIRY: Cache expiry duration in days. MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100). + LOGGER: + MINIO_LOGGER_HTTP_ENDPOINT: HTTP endpoint URL to log all incoming requests. + EXAMPLES: 1. Start minio gateway server for AWS S3 backend. $ export MINIO_ACCESS_KEY=accesskey @@ -82,7 +85,13 @@ EXAMPLES: $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG $ {{.HelpName}} https://play.minio.io:9000 - 3. Start minio gateway server for AWS S3 backend with edge caching enabled. + 3. Start minio gateway server for AWS S3 backend logging all requests to http endpoint. + $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F + $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG + $ export MINIO_LOGGER_HTTP_ENDPOINT="http://localhost:8000/" + $ {{.HelpName}} https://play.minio.io:9000 + + 4. Start minio gateway server for AWS S3 backend with edge caching enabled. $ export MINIO_ACCESS_KEY=accesskey $ export MINIO_SECRET_KEY=secretkey $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4" @@ -401,6 +410,9 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, if err != nil { return objInfo, minio.ErrorRespToObjectError(err, bucket, object) } + // On success, populate the key & metadata so they are present in the notification + oi.Key = object + oi.Metadata = minio.ToMinioClientObjectInfoMetadata(metadata) return minio.FromMinioClientObjectInfo(bucket, oi), nil } diff --git a/cmd/logger/console.go b/cmd/logger/console.go index c9fefde8c..507e0cdd7 100644 --- a/cmd/logger/console.go +++ b/cmd/logger/console.go @@ -27,7 +27,11 @@ import ( // in plain or json format to the standard output. type ConsoleTarget struct{} -func (c *ConsoleTarget) send(entry logEntry) error { +func (c *ConsoleTarget) send(e interface{}) error { + entry, ok := e.(logEntry) + if !ok { + return fmt.Errorf("Uexpected log entry structure %#v", e) + } if jsonFlag { logJSON, err := json.Marshal(&entry) if err != nil { diff --git a/cmd/logger/http.go b/cmd/logger/http.go index 5c5b9b72c..549281008 100644 --- a/cmd/logger/http.go +++ b/cmd/logger/http.go @@ -32,7 +32,8 @@ import ( // is returned to the caller. type HTTPTarget struct { // Channel of log entries - logCh chan logEntry + logCh chan interface{} + // HTTP(s) endpoint endpoint string client http.Client @@ -72,14 +73,14 @@ func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget { client: http.Client{ Transport: transport, }, - logCh: make(chan logEntry, 10000), + logCh: make(chan interface{}, 10000), } h.startHTTPLogger() return &h } -func (h *HTTPTarget) send(entry logEntry) error { +func (h *HTTPTarget) send(entry interface{}) error { select { case h.logCh <- entry: default: diff --git a/cmd/logger/logger.go b/cmd/logger/logger.go index 513dd6fa7..f5fd52d21 100644 --- a/cmd/logger/logger.go +++ b/cmd/logger/logger.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "go/build" + "net/http" "os" "path/filepath" "runtime" @@ -341,6 +342,55 @@ func logIf(ctx context.Context, err error) { } } +type auditEntry struct { + DeploymentID string `json:"deploymentid,omitempty"` + Time string `json:"time"` + API *api `json:"api,omitempty"` + RemoteHost string `json:"remotehost,omitempty"` + RequestID string `json:"requestID,omitempty"` + UserAgent string `json:"userAgent,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// AuditLog - logs audit logs to all targets. +func AuditLog(ctx context.Context, r *http.Request) { + if Disable { + return + } + + req := GetReqInfo(ctx) + if req == nil { + req = &ReqInfo{API: "SYSTEM"} + } + + API := "SYSTEM" + if req.API != "" { + API = req.API + } + + tags := make(map[string]string) + for _, entry := range req.GetTags() { + tags[entry.Key] = entry.Val + } + + entry := auditEntry{ + DeploymentID: deploymentID, + RemoteHost: req.RemoteHost, + RequestID: req.RequestID, + UserAgent: req.UserAgent, + Time: time.Now().UTC().Format(time.RFC3339Nano), + API: &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}}, + Metadata: tags, + } + + // Send audit logs only to http targets. + for _, t := range Targets { + if _, ok := t.(*HTTPTarget); ok { + t.send(entry) + } + } +} + // ErrCritical is the value panic'd whenever CriticalIf is called. var ErrCritical struct{} @@ -450,10 +500,9 @@ func (f fatalMsg) pretty(msg string, args ...interface{}) { os.Exit(1) } -var info infoMsg +type infoMsg struct{} -type infoMsg struct { -} +var info infoMsg func (i infoMsg) json(msg string, args ...interface{}) { logJSON, err := json.Marshal(&logEntry{ diff --git a/cmd/logger/targets.go b/cmd/logger/targets.go index efb149abd..4a9a0a62c 100644 --- a/cmd/logger/targets.go +++ b/cmd/logger/targets.go @@ -20,7 +20,7 @@ package logger // a single log entry and send it to the log target // e.g. send the log to a http server type LoggingTarget interface { - send(entry logEntry) error + send(entry interface{}) error } // Targets is the set of enabled loggers diff --git a/cmd/notification.go b/cmd/notification.go index dc9df91f4..d2b44b09d 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -448,6 +448,7 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr { sys.RLock() targetIDSet := sys.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name) sys.RUnlock() + if len(targetIDSet) == 0 { return nil } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 461108425..af06697b1 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -78,6 +78,8 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "SelectObject") + defer logger.AuditLog(ctx, r) + // Fetch object stat info. objectAPI := api.ObjectAPI() if objectAPI == nil { @@ -273,6 +275,12 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r logger.LogIf(ctx, err) } } + + for k, v := range objInfo.UserDefined { + logger.GetReqInfo(ctx).SetTags(k, v) + } + + logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag) } // GetObjectHandler - GET Object @@ -282,6 +290,8 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "GetObject") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -437,6 +447,12 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req Host: host, Port: port, }) + + for k, v := range objInfo.UserDefined { + logger.GetReqInfo(ctx).SetTags(k, v) + } + + logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag) } // HeadObjectHandler - HEAD Object @@ -445,6 +461,8 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "HeadObject") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponseHeadersOnly(w, ErrServerNotInitialized) @@ -578,6 +596,12 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re Host: host, Port: port, }) + + for k, v := range objInfo.UserDefined { + logger.GetReqInfo(ctx).SetTags(k, v) + } + + logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag) } // Extract metadata relevant for an CopyObject operation based on conditional @@ -618,6 +642,8 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "CopyObject") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -951,6 +977,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re Host: host, Port: port, }) + + for k, v := range objInfo.UserDefined { + logger.GetReqInfo(ctx).SetTags(k, v) + } + + logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag) } // PutObjectHandler - PUT Object @@ -964,6 +996,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutObject") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -1203,6 +1237,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req Host: host, Port: port, }) + + for k, v := range objInfo.UserDefined { + logger.GetReqInfo(ctx).SetTags(k, v) + } + + logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag) } /// Multipart objectAPIHandlers @@ -1216,6 +1256,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "NewMultipartUpload") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -1308,6 +1350,8 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "CopyObjectPart") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -1530,6 +1574,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "PutObjectPart") + defer logger.AuditLog(ctx, r) + objectAPI := api.ObjectAPI() if objectAPI == nil { writeErrorResponse(w, ErrServerNotInitialized, r.URL) @@ -1770,6 +1816,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "AbortMultipartUpload") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -1809,6 +1857,8 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListObjectParts") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -1849,6 +1899,8 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "CompleteMultipartUpload") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -1955,6 +2007,12 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite Host: host, Port: port, }) + + for k, v := range objInfo.UserDefined { + logger.GetReqInfo(ctx).SetTags(k, v) + } + + logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag) } /// Delete objectAPIHandlers @@ -1963,6 +2021,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "DeleteObject") + defer logger.AuditLog(ctx, r) + vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] diff --git a/cmd/utils.go b/cmd/utils.go index 33f38bb53..e064dd0b3 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -391,7 +391,7 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont reqInfo := &logger.ReqInfo{ RequestID: w.Header().Get(responseRequestIDKey), RemoteHost: handlers.GetSourceIP(r), - UserAgent: r.Header.Get("user-agent"), + UserAgent: r.UserAgent(), API: api, BucketName: bucket, ObjectName: object, diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 2f1433697..0851b9f7e 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -640,6 +640,10 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs, // Upload - file upload handler. func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "WebUpload") + + defer logger.AuditLog(ctx, r) + objectAPI := web.ObjectAPI() if objectAPI == nil { writeWebErrorResponse(w, errServerNotInitialized) @@ -741,13 +745,13 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { opts := ObjectOptions{} // Deny if WORM is enabled if globalWORMEnabled { - if _, err = objectAPI.GetObjectInfo(context.Background(), bucket, object, opts); err == nil { + if _, err = objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil { writeWebErrorResponse(w, errMethodNotAllowed) return } } - objInfo, err := putObject(context.Background(), bucket, object, hashReader, metadata, opts) + objInfo, err := putObject(ctx, bucket, object, hashReader, metadata, opts) if err != nil { writeWebErrorResponse(w, err) return @@ -769,10 +773,20 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { Host: host, Port: port, }) + + for k, v := range objInfo.UserDefined { + logger.GetReqInfo(ctx).SetTags(k, v) + } + + logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag) } // Download - file download handler. func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "WebDownload") + + defer logger.AuditLog(ctx, r) + var wg sync.WaitGroup objectAPI := web.ObjectAPI() if objectAPI == nil { @@ -827,7 +841,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { getObjectInfo = web.CacheAPI().GetObjectInfo getObject = web.CacheAPI().GetObject } - objInfo, err := getObjectInfo(context.Background(), bucket, object, opts) + objInfo, err := getObjectInfo(ctx, bucket, object, opts) if err != nil { writeWebErrorResponse(w, err) return @@ -897,7 +911,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { // Add content disposition. w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(object))) - if err = getObject(context.Background(), bucket, object, 0, -1, httpWriter, "", opts); err != nil { + if err = getObject(ctx, bucket, object, 0, -1, httpWriter, "", opts); err != nil { httpWriter.Close() if objInfo.IsCompressed() { wg.Wait() @@ -933,6 +947,12 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { Host: host, Port: port, }) + + for k, v := range objInfo.UserDefined { + logger.GetReqInfo(ctx).SetTags(k, v) + } + + logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag) } // DownloadZipArgs - Argument for downloading a bunch of files as a zip file. @@ -952,6 +972,10 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) { host, port = "", "" } + ctx := newContext(r, w, "WebDownloadZip") + + defer logger.AuditLog(ctx, r) + var wg sync.WaitGroup objectAPI := web.ObjectAPI() if objectAPI == nil { @@ -1030,7 +1054,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) { for _, object := range args.Objects { // Writes compressed object file to the response. zipit := func(objectName string) error { - info, err := getObjectInfo(context.Background(), args.BucketName, objectName, opts) + info, err := getObjectInfo(ctx, args.BucketName, objectName, opts) if err != nil { return err } @@ -1103,7 +1127,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) { } } httpWriter := ioutil.WriteOnClose(writer) - if err = getObject(context.Background(), args.BucketName, objectName, 0, length, httpWriter, "", opts); err != nil { + if err = getObject(ctx, args.BucketName, objectName, 0, length, httpWriter, "", opts); err != nil { httpWriter.Close() if info.IsCompressed() { // Wait for decompression go-routine to retire.