MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: audit log to support object names in multipleObjectNames() handler (#14017)

Browse Source
This commit is contained in:
Harshavardhana 2022-01-03 01:28:52 -08:00 committed by GitHub
parent 42ba0da6b0
commit a60ac7ca17
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 171 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -37,7 +37,7 @@ lint: ## runs golangci-lint suite of linters
@echo "Running $@ check"
@${GOPATH}/bin/golangci-lint cache clean
@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
@${GOPATH}/bin/gofumpt -s -l .
@${GOPATH}/bin/gofumpt -l .
check: test
test: verifiers build ## builds minio, runs linters, tests

9
cmd/api-datatypes.go
View File

@ -48,10 +48,15 @@ func (t DeleteMarkerMTime) MarshalXML(e *xml.Encoder, startElement xml.StartElem
return e.EncodeElement(t.Time.Format(time.RFC3339), startElement)
}
// ObjectToDelete carries key name for the object to delete.
type ObjectToDelete struct {
// ObjectV object version key/versionId
type ObjectV struct {
ObjectName string `xml:"Key"`
VersionID string `xml:"VersionId"`
}
// ObjectToDelete carries key name for the object to delete.
type ObjectToDelete struct {
ObjectV
// Replication status of DeleteMarker
DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus"`
// Status of versioned delete (of object or DeleteMarker)

17
cmd/bucket-handlers.go
View File

@ -422,11 +422,16 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
return
}
objects := make([]ObjectV, len(deleteObjectsReq.Objects))
// Convert object name delete objects if it has `/` in the beginning.
for i := range deleteObjectsReq.Objects {
deleteObjectsReq.Objects[i].ObjectName = trimLeadingSlash(deleteObjectsReq.Objects[i].ObjectName)
objects[i] = deleteObjectsReq.Objects[i].ObjectV
}
// Make sure to update context to print ObjectNames for multi objects.
ctx = updateReqContext(ctx, objects...)
// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
@ -527,8 +532,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
if replicateDeletes {
dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
ObjectName: object.ObjectName,
VersionID: object.VersionID,
ObjectV: ObjectV{
ObjectName: object.ObjectName,
VersionID: object.VersionID,
},
}, goi, opts, gerr)
if dsc.ReplicateAny() {
if object.VersionID != "" {
@ -581,8 +588,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
// created during DeleteMarker creation when client didn't
// specify a versionID.
objToDel := ObjectToDelete{
ObjectName: dObjects[i].ObjectName,
VersionID: dObjects[i].VersionID,
ObjectV: ObjectV{
ObjectName: dObjects[i].ObjectName,
VersionID: dObjects[i].VersionID,
},
VersionPurgeStatus: dObjects[i].VersionPurgeStatus(),
VersionPurgeStatuses: dObjects[i].ReplicationState.VersionPurgeStatusInternal,
DeleteMarkerReplicationStatus: dObjects[i].ReplicationState.ReplicationStatusInternal,

17
cmd/bucket-handlers_test.go
View File

@ -698,7 +698,9 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
getObjectToDeleteList := func(objectNames []string) (objectList []ObjectToDelete) {
for _, objectName := range objectNames {
objectList = append(objectList, ObjectToDelete{
ObjectName: objectName,
ObjectV: ObjectV{
ObjectName: objectName,
},
})
}
@ -717,10 +719,21 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
return deleteErrorList
}
objects := []ObjectToDelete{}
objects = append(objects, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: "private/object",
},
})
objects = append(objects, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: "public/object",
},
})
requestList := []DeleteObjectsRequest{
{Quiet: false, Objects: getObjectToDeleteList(objectNames[:5])},
{Quiet: true, Objects: getObjectToDeleteList(objectNames[5:])},
{Quiet: false, Objects: []ObjectToDelete{{ObjectName: "private/object"}, {ObjectName: "public/object"}}},
{Quiet: false, Objects: objects},
}
// generate multi objects delete response.

6
cmd/bucket-quota.go
View File

@ -193,8 +193,10 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
numKeys := len(scorer.fileObjInfos())
for i, obj := range scorer.fileObjInfos() {
objects = append(objects, ObjectToDelete{
ObjectName: obj.Name,
VersionID: obj.VersionID,
ObjectV: ObjectV{
ObjectName: obj.Name,
VersionID: obj.VersionID,
},
})
if len(objects) < maxDeleteList && (i < numKeys-1) {
// skip deletion until maxDeleteList or end of slice

6
cmd/bucket-replication-utils.go
View File

@ -502,8 +502,10 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
var tgtStatuses map[string]replication.StatusType
if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
ObjectName: oi.Name,
VersionID: oi.VersionID,
ObjectV: ObjectV{
ObjectName: oi.Name,
VersionID: oi.VersionID,
},
}, oi, ObjectOptions{}, nil)
} else {
dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{

6
cmd/data-scanner.go
View File

@ -1022,8 +1022,10 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
}
toDel = append(toDel, ObjectToDelete{
ObjectName: fi.Name,
VersionID: fi.VersionID,
ObjectV: ObjectV{
ObjectName: fi.Name,
VersionID: fi.VersionID,
},
})
}

6
cmd/erasure-object_test.go
View File

@ -177,7 +177,11 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
toObjectNames := func(testCases []testCaseType) []ObjectToDelete {
names := make([]ObjectToDelete, len(testCases))
for i := range testCases {
names[i] = ObjectToDelete{ObjectName: testCases[i].object}
names[i] = ObjectToDelete{
ObjectV: ObjectV{
ObjectName: testCases[i].object,
},
}
}
return names
}

13
cmd/object-handlers.go
View File

@ -3387,7 +3387,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
os.SetTransitionState(goi.TransitionedObject)
}
dsc := checkReplicateDelete(ctx, bucket, ObjectToDelete{ObjectName: object, VersionID: opts.VersionID}, goi, opts, gerr)
dsc := checkReplicateDelete(ctx, bucket, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: object,
VersionID: opts.VersionID,
},
}, goi, opts, gerr)
if dsc.ReplicateAny() {
opts.SetDeleteReplicationState(dsc, opts.VersionID)
}
@ -3414,8 +3419,10 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
}
if vID != "" {
apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
ObjectName: object,
VersionID: vID,
ObjectV: ObjectV{
ObjectName: object,
VersionID: vID,
},
}, goi, gerr)
if apiErr != ErrNone && apiErr != ErrNoSuchKey {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)

4
cmd/server_test.go
View File

@ -574,7 +574,9 @@ func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *check) {
c.Assert(response.StatusCode, http.StatusOK)
// Append all objects.
delObjReq.Objects = append(delObjReq.Objects, ObjectToDelete{
ObjectName: objName,
ObjectV: ObjectV{
ObjectName: objName,
},
})
}
// Marshal delete request.

16
cmd/utils.go
View File

@ -752,6 +752,21 @@ func likelyUnescapeGeneric(p string, escapeFn func(string) (string, error)) stri
return ep
}
func updateReqContext(ctx context.Context, objects ...ObjectV) context.Context {
req := logger.GetReqInfo(ctx)
if req != nil {
req.Objects = make([]logger.ObjectVersion, 0, len(objects))
for _, ov := range objects {
req.Objects = append(req.Objects, logger.ObjectVersion{
ObjectName: ov.ObjectName,
VersionID: ov.VersionID,
})
}
return logger.SetReqInfo(ctx, req)
}
return ctx
}
// Returns context with ReqInfo details set in the context.
func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context {
vars := mux.Vars(r)
@ -770,6 +785,7 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
API: api,
BucketName: bucket,
ObjectName: object,
VersionID: strings.TrimSpace(r.Form.Get(xhttp.VersionID)),
}
return logger.SetReqInfo(r.Context(), reqInfo)
}

3
docs/debugging/xl-meta/main.go
View File

@ -131,7 +131,7 @@ FLAGS:
Header json.RawMessage
Metadata json.RawMessage
}
var versions = make([]version, nVers)
versions := make([]version, nVers)
err = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error {
var header xlMetaV2VersionHeaderV2
if _, err := header.UnmarshalMsg(hdr); err != nil {
@ -462,7 +462,6 @@ func (x xlMetaInlineData) files(fn func(name string, data []byte)) error {
fn(string(key), val)
}
return nil
}
const (

7
internal/logger/audit.go
View File

@ -208,6 +208,13 @@ func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request, reqCl
entry.API.Name = reqInfo.API
entry.API.Bucket = reqInfo.BucketName
entry.API.Object = reqInfo.ObjectName
entry.API.Objects = make([]audit.ObjectVersion, 0, len(reqInfo.Objects))
for _, ov := range reqInfo.Objects {
entry.API.Objects = append(entry.API.Objects, audit.ObjectVersion{
ObjectName: ov.ObjectName,
VersionID: ov.VersionID,
})
}
entry.API.Status = http.StatusText(statusCode)
entry.API.StatusCode = statusCode
entry.API.InputBytes = r.ContentLength

32
internal/logger/logger.go
View File

@ -62,23 +62,6 @@ var matchingFuncNames = [...]string{
"http.HandlerFunc.ServeHTTP",
"cmd.serverMain",
"cmd.StartGateway",
"cmd.(*webAPIHandlers).ListBuckets",
"cmd.(*webAPIHandlers).MakeBucket",
"cmd.(*webAPIHandlers).DeleteBucket",
"cmd.(*webAPIHandlers).ListObjects",
"cmd.(*webAPIHandlers).RemoveObject",
"cmd.(*webAPIHandlers).Login",
"cmd.(*webAPIHandlers).SetAuth",
"cmd.(*webAPIHandlers).CreateURLToken",
"cmd.(*webAPIHandlers).Upload",
"cmd.(*webAPIHandlers).Download",
"cmd.(*webAPIHandlers).DownloadZip",
"cmd.(*webAPIHandlers).GetBucketPolicy",
"cmd.(*webAPIHandlers).ListAllBucketPolicies",
"cmd.(*webAPIHandlers).SetBucketPolicy",
"cmd.(*webAPIHandlers).PresignedGet",
"cmd.(*webAPIHandlers).ServerInfo",
"cmd.(*webAPIHandlers).StorageInfo",
// add more here ..
}
@ -338,6 +321,15 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
if req.DeploymentID == "" {
req.DeploymentID = globalDeploymentID
}
objects := make([]log.ObjectVersion, 0, len(req.Objects))
for _, ov := range req.Objects {
objects = append(objects, log.ObjectVersion{
ObjectName: ov.ObjectName,
VersionID: ov.VersionID,
})
}
entry := log.Entry{
DeploymentID: req.DeploymentID,
Level: ErrorLvl.String(),
@ -350,8 +342,10 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
API: &log.API{
Name: API,
Args: &log.Args{
Bucket: req.BucketName,
Object: req.ObjectName,
Bucket: req.BucketName,
Object: req.ObjectName,
VersionID: req.VersionID,
Objects: objects,
},
},
Trace: &log.Trace{

25
internal/logger/message/audit/entry.go
View File

@ -29,6 +29,12 @@ import (
// Version - represents the current version of audit log structure.
const Version = "1"
// ObjectVersion object version key/versionId
type ObjectVersion struct {
ObjectName string `json:"objectName"`
VersionID string `json:"VersionId,omitempty"`
}
// Entry - audit entry logs.
type Entry struct {
Version string `json:"version"`
@ -36,15 +42,16 @@ type Entry struct {
Time time.Time `json:"time"`
Trigger string `json:"trigger"`
API struct {
Name string `json:"name,omitempty"`
Bucket string `json:"bucket,omitempty"`
Object string `json:"object,omitempty"`
Status string `json:"status,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
InputBytes int64 `json:"rx"`
OutputBytes int64 `json:"tx"`
TimeToFirstByte string `json:"timeToFirstByte,omitempty"`
TimeToResponse string `json:"timeToResponse,omitempty"`
Name string `json:"name,omitempty"`
Bucket string `json:"bucket,omitempty"`
Object string `json:"object,omitempty"`
Objects []ObjectVersion `json:"objects,omitempty"`
Status string `json:"status,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
InputBytes int64 `json:"rx"`
OutputBytes int64 `json:"tx"`
TimeToFirstByte string `json:"timeToFirstByte,omitempty"`
TimeToResponse string `json:"timeToResponse,omitempty"`
} `json:"api"`
RemoteHost string `json:"remotehost,omitempty"`
RequestID string `json:"requestID,omitempty"`

14
internal/logger/message/log/entry.go
View File

@ -22,11 +22,19 @@ import (
"time"
)
// ObjectVersion object version key/versionId
type ObjectVersion struct {
ObjectName string `json:"objectName"`
VersionID string `json:"VersionId,omitempty"`
}
// Args - defines the arguments for the API.
type Args struct {
Bucket string `json:"bucket,omitempty"`
Object string `json:"object,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
Bucket string `json:"bucket,omitempty"`
Object string `json:"object,omitempty"`
VersionID string `json:"versionId,omitempty"`
Objects []ObjectVersion `json:"objects,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
}
// Trace - defines the trace.

46
internal/logger/reqinfo.go
View File

@ -34,32 +34,40 @@ type KeyVal struct {
Val interface{}
}
// ObjectVersion object version key/versionId
type ObjectVersion struct {
ObjectName string
VersionID string `json:"VersionId,omitempty"`
}
// ReqInfo stores the request info.
type ReqInfo struct {
RemoteHost string // Client Host/IP
Host string // Node Host/IP
UserAgent string // User Agent
DeploymentID string // x-minio-deployment-id
RequestID string // x-amz-request-id
API string // API name - GetObject PutObject NewMultipartUpload etc.
BucketName string // Bucket name
ObjectName string // Object name
AccessKey string // Access Key
tags []KeyVal // Any additional info not accommodated by above fields
RemoteHost string // Client Host/IP
Host string // Node Host/IP
UserAgent string // User Agent
DeploymentID string // x-minio-deployment-id
RequestID string // x-amz-request-id
API string // API name - GetObject PutObject NewMultipartUpload etc.
BucketName string `json:",omitempty"` // Bucket name
ObjectName string `json:",omitempty"` // Object name
VersionID string `json:",omitempty"` // corresponding versionID for the object
Objects []ObjectVersion `json:",omitempty"` // Only set during MultiObject delete handler.
AccessKey string // Access Key
tags []KeyVal // Any additional info not accommodated by above fields
sync.RWMutex
}
// NewReqInfo :
func NewReqInfo(remoteHost, userAgent, deploymentID, requestID, api, bucket, object string) *ReqInfo {
req := ReqInfo{}
req.RemoteHost = remoteHost
req.UserAgent = userAgent
req.API = api
req.DeploymentID = deploymentID
req.RequestID = requestID
req.BucketName = bucket
req.ObjectName = object
return &req
return &ReqInfo{
RemoteHost: remoteHost,
UserAgent: userAgent,
API: api,
DeploymentID: deploymentID,
RequestID: requestID,
BucketName: bucket,
ObjectName: object,
}
}
// AppendTags - appends key/val to ReqInfo.tags

19
internal/logger/target/console/console.go
View File

@ -20,6 +20,7 @@ package console
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/minio/minio/internal/color"
@ -83,11 +84,19 @@ func (c *Target) Send(e interface{}, logKind string) error {
var apiString string
if entry.API != nil {
apiString = "API: " + entry.API.Name + "("
if entry.API.Args != nil && entry.API.Args.Bucket != "" {
apiString = apiString + "bucket=" + entry.API.Args.Bucket
}
if entry.API.Args != nil && entry.API.Args.Object != "" {
apiString = apiString + ", object=" + entry.API.Args.Object
if entry.API.Args != nil {
if entry.API.Args.Bucket != "" {
apiString = apiString + "bucket=" + entry.API.Args.Bucket
}
if entry.API.Args.Object != "" {
apiString = apiString + ", object=" + entry.API.Args.Object
}
if entry.API.Args.VersionID != "" {
apiString = apiString + ", versionId=" + entry.API.Args.VersionID
}
if len(entry.API.Args.Objects) > 0 {
apiString = apiString + ", multiObject=true, numberOfObjects=" + strconv.Itoa(len(entry.API.Args.Objects))
}
}
apiString += ")"
} else {