fix: audit log to support object names in multipleObjectNames() handler (#14017)

This commit is contained in:
Harshavardhana 2022-01-03 01:28:52 -08:00 committed by GitHub
parent 42ba0da6b0
commit a60ac7ca17
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 171 additions and 77 deletions

View File

@ -37,7 +37,7 @@ lint: ## runs golangci-lint suite of linters
@echo "Running $@ check" @echo "Running $@ check"
@${GOPATH}/bin/golangci-lint cache clean @${GOPATH}/bin/golangci-lint cache clean
@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml @${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
@${GOPATH}/bin/gofumpt -s -l . @${GOPATH}/bin/gofumpt -l .
check: test check: test
test: verifiers build ## builds minio, runs linters, tests test: verifiers build ## builds minio, runs linters, tests

View File

@ -48,10 +48,15 @@ func (t DeleteMarkerMTime) MarshalXML(e *xml.Encoder, startElement xml.StartElem
return e.EncodeElement(t.Time.Format(time.RFC3339), startElement) return e.EncodeElement(t.Time.Format(time.RFC3339), startElement)
} }
// ObjectToDelete carries key name for the object to delete. // ObjectV object version key/versionId
type ObjectToDelete struct { type ObjectV struct {
ObjectName string `xml:"Key"` ObjectName string `xml:"Key"`
VersionID string `xml:"VersionId"` VersionID string `xml:"VersionId"`
}
// ObjectToDelete carries key name for the object to delete.
type ObjectToDelete struct {
ObjectV
// Replication status of DeleteMarker // Replication status of DeleteMarker
DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus"` DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus"`
// Status of versioned delete (of object or DeleteMarker) // Status of versioned delete (of object or DeleteMarker)

View File

@ -422,11 +422,16 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
return return
} }
objects := make([]ObjectV, len(deleteObjectsReq.Objects))
// Convert object name delete objects if it has `/` in the beginning. // Convert object name delete objects if it has `/` in the beginning.
for i := range deleteObjectsReq.Objects { for i := range deleteObjectsReq.Objects {
deleteObjectsReq.Objects[i].ObjectName = trimLeadingSlash(deleteObjectsReq.Objects[i].ObjectName) deleteObjectsReq.Objects[i].ObjectName = trimLeadingSlash(deleteObjectsReq.Objects[i].ObjectName)
objects[i] = deleteObjectsReq.Objects[i].ObjectV
} }
// Make sure to update context to print ObjectNames for multi objects.
ctx = updateReqContext(ctx, objects...)
// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo() // Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
// Ignore errors here to preserve the S3 error behavior of GetBucketInfo() // Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "") checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
@ -527,8 +532,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
if replicateDeletes { if replicateDeletes {
dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{ dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
ObjectName: object.ObjectName, ObjectV: ObjectV{
VersionID: object.VersionID, ObjectName: object.ObjectName,
VersionID: object.VersionID,
},
}, goi, opts, gerr) }, goi, opts, gerr)
if dsc.ReplicateAny() { if dsc.ReplicateAny() {
if object.VersionID != "" { if object.VersionID != "" {
@ -581,8 +588,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
// created during DeleteMarker creation when client didn't // created during DeleteMarker creation when client didn't
// specify a versionID. // specify a versionID.
objToDel := ObjectToDelete{ objToDel := ObjectToDelete{
ObjectName: dObjects[i].ObjectName, ObjectV: ObjectV{
VersionID: dObjects[i].VersionID, ObjectName: dObjects[i].ObjectName,
VersionID: dObjects[i].VersionID,
},
VersionPurgeStatus: dObjects[i].VersionPurgeStatus(), VersionPurgeStatus: dObjects[i].VersionPurgeStatus(),
VersionPurgeStatuses: dObjects[i].ReplicationState.VersionPurgeStatusInternal, VersionPurgeStatuses: dObjects[i].ReplicationState.VersionPurgeStatusInternal,
DeleteMarkerReplicationStatus: dObjects[i].ReplicationState.ReplicationStatusInternal, DeleteMarkerReplicationStatus: dObjects[i].ReplicationState.ReplicationStatusInternal,

View File

@ -698,7 +698,9 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
getObjectToDeleteList := func(objectNames []string) (objectList []ObjectToDelete) { getObjectToDeleteList := func(objectNames []string) (objectList []ObjectToDelete) {
for _, objectName := range objectNames { for _, objectName := range objectNames {
objectList = append(objectList, ObjectToDelete{ objectList = append(objectList, ObjectToDelete{
ObjectName: objectName, ObjectV: ObjectV{
ObjectName: objectName,
},
}) })
} }
@ -717,10 +719,21 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
return deleteErrorList return deleteErrorList
} }
objects := []ObjectToDelete{}
objects = append(objects, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: "private/object",
},
})
objects = append(objects, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: "public/object",
},
})
requestList := []DeleteObjectsRequest{ requestList := []DeleteObjectsRequest{
{Quiet: false, Objects: getObjectToDeleteList(objectNames[:5])}, {Quiet: false, Objects: getObjectToDeleteList(objectNames[:5])},
{Quiet: true, Objects: getObjectToDeleteList(objectNames[5:])}, {Quiet: true, Objects: getObjectToDeleteList(objectNames[5:])},
{Quiet: false, Objects: []ObjectToDelete{{ObjectName: "private/object"}, {ObjectName: "public/object"}}}, {Quiet: false, Objects: objects},
} }
// generate multi objects delete response. // generate multi objects delete response.

View File

@ -193,8 +193,10 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
numKeys := len(scorer.fileObjInfos()) numKeys := len(scorer.fileObjInfos())
for i, obj := range scorer.fileObjInfos() { for i, obj := range scorer.fileObjInfos() {
objects = append(objects, ObjectToDelete{ objects = append(objects, ObjectToDelete{
ObjectName: obj.Name, ObjectV: ObjectV{
VersionID: obj.VersionID, ObjectName: obj.Name,
VersionID: obj.VersionID,
},
}) })
if len(objects) < maxDeleteList && (i < numKeys-1) { if len(objects) < maxDeleteList && (i < numKeys-1) {
// skip deletion until maxDeleteList or end of slice // skip deletion until maxDeleteList or end of slice

View File

@ -502,8 +502,10 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
var tgtStatuses map[string]replication.StatusType var tgtStatuses map[string]replication.StatusType
if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() { if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{ dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
ObjectName: oi.Name, ObjectV: ObjectV{
VersionID: oi.VersionID, ObjectName: oi.Name,
VersionID: oi.VersionID,
},
}, oi, ObjectOptions{}, nil) }, oi, ObjectOptions{}, nil)
} else { } else {
dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{ dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{

View File

@ -1022,8 +1022,10 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
} }
toDel = append(toDel, ObjectToDelete{ toDel = append(toDel, ObjectToDelete{
ObjectName: fi.Name, ObjectV: ObjectV{
VersionID: fi.VersionID, ObjectName: fi.Name,
VersionID: fi.VersionID,
},
}) })
} }

View File

@ -177,7 +177,11 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
toObjectNames := func(testCases []testCaseType) []ObjectToDelete { toObjectNames := func(testCases []testCaseType) []ObjectToDelete {
names := make([]ObjectToDelete, len(testCases)) names := make([]ObjectToDelete, len(testCases))
for i := range testCases { for i := range testCases {
names[i] = ObjectToDelete{ObjectName: testCases[i].object} names[i] = ObjectToDelete{
ObjectV: ObjectV{
ObjectName: testCases[i].object,
},
}
} }
return names return names
} }

View File

@ -3387,7 +3387,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
os.SetTransitionState(goi.TransitionedObject) os.SetTransitionState(goi.TransitionedObject)
} }
dsc := checkReplicateDelete(ctx, bucket, ObjectToDelete{ObjectName: object, VersionID: opts.VersionID}, goi, opts, gerr) dsc := checkReplicateDelete(ctx, bucket, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: object,
VersionID: opts.VersionID,
},
}, goi, opts, gerr)
if dsc.ReplicateAny() { if dsc.ReplicateAny() {
opts.SetDeleteReplicationState(dsc, opts.VersionID) opts.SetDeleteReplicationState(dsc, opts.VersionID)
} }
@ -3414,8 +3419,10 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
} }
if vID != "" { if vID != "" {
apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{ apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
ObjectName: object, ObjectV: ObjectV{
VersionID: vID, ObjectName: object,
VersionID: vID,
},
}, goi, gerr) }, goi, gerr)
if apiErr != ErrNone && apiErr != ErrNoSuchKey { if apiErr != ErrNone && apiErr != ErrNoSuchKey {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL) writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)

View File

@ -574,7 +574,9 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *check) {
c.Assert(response.StatusCode, http.StatusOK) c.Assert(response.StatusCode, http.StatusOK)
// Append all objects. // Append all objects.
delObjReq.Objects = append(delObjReq.Objects, ObjectToDelete{ delObjReq.Objects = append(delObjReq.Objects, ObjectToDelete{
ObjectName: objName, ObjectV: ObjectV{
ObjectName: objName,
},
}) })
} }
// Marshal delete request. // Marshal delete request.

View File

@ -752,6 +752,21 @@ func likelyUnescapeGeneric(p string, escapeFn func(string) (string, error)) stri
return ep return ep
} }
func updateReqContext(ctx context.Context, objects ...ObjectV) context.Context {
req := logger.GetReqInfo(ctx)
if req != nil {
req.Objects = make([]logger.ObjectVersion, 0, len(objects))
for _, ov := range objects {
req.Objects = append(req.Objects, logger.ObjectVersion{
ObjectName: ov.ObjectName,
VersionID: ov.VersionID,
})
}
return logger.SetReqInfo(ctx, req)
}
return ctx
}
// Returns context with ReqInfo details set in the context. // Returns context with ReqInfo details set in the context.
func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context { func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context {
vars := mux.Vars(r) vars := mux.Vars(r)
@ -770,6 +785,7 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
API: api, API: api,
BucketName: bucket, BucketName: bucket,
ObjectName: object, ObjectName: object,
VersionID: strings.TrimSpace(r.Form.Get(xhttp.VersionID)),
} }
return logger.SetReqInfo(r.Context(), reqInfo) return logger.SetReqInfo(r.Context(), reqInfo)
} }

View File

@ -131,7 +131,7 @@ FLAGS:
Header json.RawMessage Header json.RawMessage
Metadata json.RawMessage Metadata json.RawMessage
} }
var versions = make([]version, nVers) versions := make([]version, nVers)
err = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error { err = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error {
var header xlMetaV2VersionHeaderV2 var header xlMetaV2VersionHeaderV2
if _, err := header.UnmarshalMsg(hdr); err != nil { if _, err := header.UnmarshalMsg(hdr); err != nil {
@ -462,7 +462,6 @@ func (x xlMetaInlineData) files(fn func(name string, data []byte)) error {
fn(string(key), val) fn(string(key), val)
} }
return nil return nil
} }
const ( const (

View File

@ -208,6 +208,13 @@ func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request, reqCl
entry.API.Name = reqInfo.API entry.API.Name = reqInfo.API
entry.API.Bucket = reqInfo.BucketName entry.API.Bucket = reqInfo.BucketName
entry.API.Object = reqInfo.ObjectName entry.API.Object = reqInfo.ObjectName
entry.API.Objects = make([]audit.ObjectVersion, 0, len(reqInfo.Objects))
for _, ov := range reqInfo.Objects {
entry.API.Objects = append(entry.API.Objects, audit.ObjectVersion{
ObjectName: ov.ObjectName,
VersionID: ov.VersionID,
})
}
entry.API.Status = http.StatusText(statusCode) entry.API.Status = http.StatusText(statusCode)
entry.API.StatusCode = statusCode entry.API.StatusCode = statusCode
entry.API.InputBytes = r.ContentLength entry.API.InputBytes = r.ContentLength

View File

@ -62,23 +62,6 @@ var matchingFuncNames = [...]string{
"http.HandlerFunc.ServeHTTP", "http.HandlerFunc.ServeHTTP",
"cmd.serverMain", "cmd.serverMain",
"cmd.StartGateway", "cmd.StartGateway",
"cmd.(*webAPIHandlers).ListBuckets",
"cmd.(*webAPIHandlers).MakeBucket",
"cmd.(*webAPIHandlers).DeleteBucket",
"cmd.(*webAPIHandlers).ListObjects",
"cmd.(*webAPIHandlers).RemoveObject",
"cmd.(*webAPIHandlers).Login",
"cmd.(*webAPIHandlers).SetAuth",
"cmd.(*webAPIHandlers).CreateURLToken",
"cmd.(*webAPIHandlers).Upload",
"cmd.(*webAPIHandlers).Download",
"cmd.(*webAPIHandlers).DownloadZip",
"cmd.(*webAPIHandlers).GetBucketPolicy",
"cmd.(*webAPIHandlers).ListAllBucketPolicies",
"cmd.(*webAPIHandlers).SetBucketPolicy",
"cmd.(*webAPIHandlers).PresignedGet",
"cmd.(*webAPIHandlers).ServerInfo",
"cmd.(*webAPIHandlers).StorageInfo",
// add more here .. // add more here ..
} }
@ -338,6 +321,15 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
if req.DeploymentID == "" { if req.DeploymentID == "" {
req.DeploymentID = globalDeploymentID req.DeploymentID = globalDeploymentID
} }
objects := make([]log.ObjectVersion, 0, len(req.Objects))
for _, ov := range req.Objects {
objects = append(objects, log.ObjectVersion{
ObjectName: ov.ObjectName,
VersionID: ov.VersionID,
})
}
entry := log.Entry{ entry := log.Entry{
DeploymentID: req.DeploymentID, DeploymentID: req.DeploymentID,
Level: ErrorLvl.String(), Level: ErrorLvl.String(),
@ -350,8 +342,10 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
API: &log.API{ API: &log.API{
Name: API, Name: API,
Args: &log.Args{ Args: &log.Args{
Bucket: req.BucketName, Bucket: req.BucketName,
Object: req.ObjectName, Object: req.ObjectName,
VersionID: req.VersionID,
Objects: objects,
}, },
}, },
Trace: &log.Trace{ Trace: &log.Trace{

View File

@ -29,6 +29,12 @@ import (
// Version - represents the current version of audit log structure. // Version - represents the current version of audit log structure.
const Version = "1" const Version = "1"
// ObjectVersion object version key/versionId
type ObjectVersion struct {
ObjectName string `json:"objectName"`
VersionID string `json:"VersionId,omitempty"`
}
// Entry - audit entry logs. // Entry - audit entry logs.
type Entry struct { type Entry struct {
Version string `json:"version"` Version string `json:"version"`
@ -36,15 +42,16 @@ type Entry struct {
Time time.Time `json:"time"` Time time.Time `json:"time"`
Trigger string `json:"trigger"` Trigger string `json:"trigger"`
API struct { API struct {
Name string `json:"name,omitempty"` Name string `json:"name,omitempty"`
Bucket string `json:"bucket,omitempty"` Bucket string `json:"bucket,omitempty"`
Object string `json:"object,omitempty"` Object string `json:"object,omitempty"`
Status string `json:"status,omitempty"` Objects []ObjectVersion `json:"objects,omitempty"`
StatusCode int `json:"statusCode,omitempty"` Status string `json:"status,omitempty"`
InputBytes int64 `json:"rx"` StatusCode int `json:"statusCode,omitempty"`
OutputBytes int64 `json:"tx"` InputBytes int64 `json:"rx"`
TimeToFirstByte string `json:"timeToFirstByte,omitempty"` OutputBytes int64 `json:"tx"`
TimeToResponse string `json:"timeToResponse,omitempty"` TimeToFirstByte string `json:"timeToFirstByte,omitempty"`
TimeToResponse string `json:"timeToResponse,omitempty"`
} `json:"api"` } `json:"api"`
RemoteHost string `json:"remotehost,omitempty"` RemoteHost string `json:"remotehost,omitempty"`
RequestID string `json:"requestID,omitempty"` RequestID string `json:"requestID,omitempty"`

View File

@ -22,11 +22,19 @@ import (
"time" "time"
) )
// ObjectVersion object version key/versionId
type ObjectVersion struct {
ObjectName string `json:"objectName"`
VersionID string `json:"VersionId,omitempty"`
}
// Args - defines the arguments for the API. // Args - defines the arguments for the API.
type Args struct { type Args struct {
Bucket string `json:"bucket,omitempty"` Bucket string `json:"bucket,omitempty"`
Object string `json:"object,omitempty"` Object string `json:"object,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"` VersionID string `json:"versionId,omitempty"`
Objects []ObjectVersion `json:"objects,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
} }
// Trace - defines the trace. // Trace - defines the trace.

View File

@ -34,32 +34,40 @@ type KeyVal struct {
Val interface{} Val interface{}
} }
// ObjectVersion object version key/versionId
type ObjectVersion struct {
ObjectName string
VersionID string `json:"VersionId,omitempty"`
}
// ReqInfo stores the request info. // ReqInfo stores the request info.
type ReqInfo struct { type ReqInfo struct {
RemoteHost string // Client Host/IP RemoteHost string // Client Host/IP
Host string // Node Host/IP Host string // Node Host/IP
UserAgent string // User Agent UserAgent string // User Agent
DeploymentID string // x-minio-deployment-id DeploymentID string // x-minio-deployment-id
RequestID string // x-amz-request-id RequestID string // x-amz-request-id
API string // API name - GetObject PutObject NewMultipartUpload etc. API string // API name - GetObject PutObject NewMultipartUpload etc.
BucketName string // Bucket name BucketName string `json:",omitempty"` // Bucket name
ObjectName string // Object name ObjectName string `json:",omitempty"` // Object name
AccessKey string // Access Key VersionID string `json:",omitempty"` // corresponding versionID for the object
tags []KeyVal // Any additional info not accommodated by above fields Objects []ObjectVersion `json:",omitempty"` // Only set during MultiObject delete handler.
AccessKey string // Access Key
tags []KeyVal // Any additional info not accommodated by above fields
sync.RWMutex sync.RWMutex
} }
// NewReqInfo : // NewReqInfo :
func NewReqInfo(remoteHost, userAgent, deploymentID, requestID, api, bucket, object string) *ReqInfo { func NewReqInfo(remoteHost, userAgent, deploymentID, requestID, api, bucket, object string) *ReqInfo {
req := ReqInfo{} return &ReqInfo{
req.RemoteHost = remoteHost RemoteHost: remoteHost,
req.UserAgent = userAgent UserAgent: userAgent,
req.API = api API: api,
req.DeploymentID = deploymentID DeploymentID: deploymentID,
req.RequestID = requestID RequestID: requestID,
req.BucketName = bucket BucketName: bucket,
req.ObjectName = object ObjectName: object,
return &req }
} }
// AppendTags - appends key/val to ReqInfo.tags // AppendTags - appends key/val to ReqInfo.tags

View File

@ -20,6 +20,7 @@ package console
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"strconv"
"strings" "strings"
"github.com/minio/minio/internal/color" "github.com/minio/minio/internal/color"
@ -83,11 +84,19 @@ func (c *Target) Send(e interface{}, logKind string) error {
var apiString string var apiString string
if entry.API != nil { if entry.API != nil {
apiString = "API: " + entry.API.Name + "(" apiString = "API: " + entry.API.Name + "("
if entry.API.Args != nil && entry.API.Args.Bucket != "" { if entry.API.Args != nil {
apiString = apiString + "bucket=" + entry.API.Args.Bucket if entry.API.Args.Bucket != "" {
} apiString = apiString + "bucket=" + entry.API.Args.Bucket
if entry.API.Args != nil && entry.API.Args.Object != "" { }
apiString = apiString + ", object=" + entry.API.Args.Object if entry.API.Args.Object != "" {
apiString = apiString + ", object=" + entry.API.Args.Object
}
if entry.API.Args.VersionID != "" {
apiString = apiString + ", versionId=" + entry.API.Args.VersionID
}
if len(entry.API.Args.Objects) > 0 {
apiString = apiString + ", multiObject=true, numberOfObjects=" + strconv.Itoa(len(entry.API.Args.Objects))
}
} }
apiString += ")" apiString += ")"
} else { } else {