MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-04-25 04:33:20 -04:00
Code Issues Packages Projects Releases Wiki Activity

simplify audit logging for replication and ILM (#12610)

Browse Source 
auditLog should be attempted right before the
return of the function and not multiple times
per function, this ensures that we only trigger
it once per function call.
This commit is contained in:
Harshavardhana 2021-07-01 14:02:44 -07:00 committed by GitHub
parent a1df230518
commit 4f6c74a257
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 102 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/bucket-lifecycle.go
View File

@ -235,7 +235,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
} }
// Send audit for the lifecycle delete operation // Send audit for the lifecycle delete operation
auditLogLifecycle(ctx, oi.Bucket, oi.Name, oi.VersionID, ILMExpiryActivity) auditLogLifecycle(ctx, *oi, ILMExpiry)
eventName := event.ObjectRemovedDelete eventName := event.ObjectRemovedDelete
if lcOpts.DeleteMarker { if lcOpts.DeleteMarker {

158
cmd/bucket-replication.go
View File

@ -257,13 +257,29 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently // target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds // deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
// on target. // on target.
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) { func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer, trigger string) {
var versionPurgeStatus VersionPurgeStatusType
var replicationStatus string
bucket := dobj.Bucket bucket := dobj.Bucket
versionID := dobj.DeleteMarkerVersionID versionID := dobj.DeleteMarkerVersionID
if versionID == "" { if versionID == "" {
versionID = dobj.VersionID versionID = dobj.VersionID
} }
defer func() {
replStatus := replicationStatus
if !versionPurgeStatus.Empty() {
replStatus = string(versionPurgeStatus)
}
auditLogInternal(context.Background(), bucket, dobj.ObjectName, AuditLogOptions{
Trigger: trigger,
APIName: ReplicateDeleteAPI,
VersionID: versionID,
Status: replStatus,
})
}()
rcfg, err := getReplicationConfig(ctx, bucket) rcfg, err := getReplicationConfig(ctx, bucket)
if err != nil || rcfg == nil { if err != nil || rcfg == nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
@ -308,8 +324,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
}, },
}) })
replicationStatus := dobj.DeleteMarkerReplicationStatus replicationStatus = dobj.DeleteMarkerReplicationStatus
versionPurgeStatus := dobj.VersionPurgeStatus versionPurgeStatus = dobj.VersionPurgeStatus
if rmErr != nil { if rmErr != nil {
if dobj.VersionID == "" { if dobj.VersionID == "" {
@ -332,7 +348,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
currStatus = string(versionPurgeStatus) currStatus = string(versionPurgeStatus)
} }
// to decrement pending count later. // to decrement pending count later.
globalReplicationStats.Update(dobj.Bucket, 0, replication.StatusType(currStatus), replication.StatusType(prevStatus), replication.DeleteReplicationType) globalReplicationStats.Update(dobj.Bucket, 0, replication.StatusType(currStatus),
replication.StatusType(prevStatus), replication.DeleteReplicationType)
var eventName = event.ObjectReplicationComplete var eventName = event.ObjectReplicationComplete
if replicationStatus == string(replication.Failed) || versionPurgeStatus == Failed { if replicationStatus == string(replication.Failed) || versionPurgeStatus == Failed {
@ -606,13 +623,24 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo) replicationActio
// replicateObject replicates the specified version of the object to destination bucket // replicateObject replicates the specified version of the object to destination bucket
// The source object is then updated to reflect the replication status. // The source object is then updated to reflect the replication status.
func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) { func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer, trigger string) {
auditLogInternal(context.Background(), ri.Bucket, ri.Name, AuditLogOptions{ var replicationStatus replication.StatusType
Trigger: ReplicationIncomingActivity, defer func() {
APIName: "s3:ReplicateObject", if replicationStatus.Empty() {
// replication status is empty means
// replication was not attempted for some
// reason, notify the state of the object
// on disk.
replicationStatus = ri.ReplicationStatus
}
auditLogInternal(ctx, ri.Bucket, ri.Name, AuditLogOptions{
Trigger: trigger,
APIName: ReplicateObjectAPI,
VersionID: ri.VersionID, VersionID: ri.VersionID,
Status: ri.ReplicationStatus.String(), Status: replicationStatus.String(),
}) })
}()
objInfo := ri.ObjectInfo objInfo := ri.ObjectInfo
bucket := objInfo.Bucket bucket := objInfo.Bucket
object := objInfo.Name object := objInfo.Name
@ -697,8 +725,11 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
// object with same VersionID already exists, replication kicked off by // object with same VersionID already exists, replication kicked off by
// PutObject might have completed // PutObject might have completed
if objInfo.ReplicationStatus == replication.Pending || objInfo.ReplicationStatus == replication.Failed { if objInfo.ReplicationStatus == replication.Pending || objInfo.ReplicationStatus == replication.Failed {
// if metadata is not updated for some reason after replication, such as 503 encountered while updating metadata - make sure // if metadata is not updated for some reason after replication, such as
// to set ReplicationStatus as Completed.Note that replication Stats would have been updated despite metadata update failure. // 503 encountered while updating metadata - make sure to set ReplicationStatus
// as Completed.
//
// Note: Replication Stats would have been updated despite metadata update failure.
gr.Close() gr.Close()
closeOnDefer = false closeOnDefer = false
popts := ObjectOptions{ popts := ObjectOptions{
@ -715,19 +746,13 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
} }
if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil { if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w", bucket, objInfo.Name, objInfo.VersionID, err)) logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w", bucket, objInfo.Name, objInfo.VersionID, err))
} else {
auditLogInternal(context.Background(), ri.Bucket, ri.Name, AuditLogOptions{
Trigger: ReplicationIncomingActivity,
APIName: "s3:ReplicateObject",
VersionID: ri.VersionID,
Status: ri.ReplicationStatus.String(),
})
} }
} }
return return
} }
} }
replicationStatus := replication.Completed
replicationStatus = replication.Completed
// use core client to avoid doing multipart on PUT // use core client to avoid doing multipart on PUT
c := &miniogo.Core{Client: tgt.Client} c := &miniogo.Core{Client: tgt.Client}
if rtype != replicateAll { if rtype != replicateAll {
@ -774,7 +799,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
if uploadID, err := replicateObjectWithMultipart(ctx, c, dest.Bucket, object, r, objInfo, putOpts); err != nil { if uploadID, err := replicateObjectWithMultipart(ctx, c, dest.Bucket, object, r, objInfo, putOpts); err != nil {
replicationStatus = replication.Failed replicationStatus = replication.Failed
logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err)) logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
defer c.AbortMultipartUpload(ctx, dest.Bucket, object, uploadID) // block and abort remote upload upon failure.
c.AbortMultipartUpload(ctx, dest.Bucket, object, uploadID)
} }
} else { } else {
if _, err = c.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts); err != nil { if _, err = c.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts); err != nil {
@ -818,16 +844,9 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
popts.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags popts.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
} }
if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil { if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w", bucket, objInfo.Name, objInfo.VersionID, err)) logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w",
} else { bucket, objInfo.Name, objInfo.VersionID, err))
auditLogInternal(context.Background(), objInfo.Bucket, objInfo.Name, AuditLogOptions{
Trigger: ReplicationIncomingActivity,
APIName: "s3:ReplicateObject",
VersionID: objInfo.VersionID,
Status: replicationStatus.String(),
})
} }
opType := replication.MetadataReplicationType opType := replication.MetadataReplicationType
if rtype == replicateAll { if rtype == replicateAll {
opType = replication.ObjectReplicationType opType = replication.ObjectReplicationType
@ -914,21 +933,29 @@ type DeletedObjectReplicationInfo struct {
ResetID string ResetID string
} }
// Replication specific APIName
const ( const (
// ReplicationQueuedActivity - replication being queued activity trail ReplicateObjectAPI = "ReplicateObject"
ReplicationQueuedActivity = "replication:queue" ReplicateDeleteAPI = "ReplicateDelete"
// ReplicationExistingActivity - activity trail for existing objects replication )
ReplicationExistingActivity = "replication:existing"
// ReplicationMRFActivity - activity trail for replication from Most Recent Failures (MRF) queue const (
ReplicationMRFActivity = "replication:mrf" // ReplicateQueued - replication being queued trail
// ReplicationIncomingActivity - activity trail indicating replication started [could be from incoming/existing/heal activity] ReplicateQueued = "replicate:queue"
ReplicationIncomingActivity = "replication:incoming"
// ReplicationHealActivity - activity trail for healing of failed/pending replications // ReplicateExisting - audit trail for existing objects replication
ReplicationHealActivity = "replication:heal" ReplicateExisting = "replicate:existing"
// ReplicationDeleteActivity - activity trail for delete replication // ReplicateExistingDelete - audit trail for delete replication triggered for existing delete markers
ReplicationDeleteActivity = "replication:delete" ReplicateExistingDelete = "replicate:existing:delete"
// ReplicationExistingDeleteActivity - activity trail for delete replication triggered for existing delete markers
ReplicationExistingDeleteActivity = "replication:delete:existing" // ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue
ReplicateMRF = "replicate:mrf"
// ReplicateIncoming - audit trail indicating replication started [could be from incoming/existing/heal activity]
ReplicateIncoming = "replicate:incoming"
// ReplicateHeal - audit trail for healing of failed/pending replications
ReplicateHeal = "replicate:heal"
// ReplicateDelete - audit trail for delete replication
ReplicateDelete = "replicate:delete"
) )
var ( var (
@ -986,7 +1013,7 @@ func (p *ReplicationPool) AddMRFWorker() {
if !ok { if !ok {
return return
} }
replicateObject(p.ctx, oi, p.objLayer) replicateObject(p.ctx, oi, p.objLayer, ReplicateMRF)
case <-p.mrfWorkerKillCh: case <-p.mrfWorkerKillCh:
return return
} }
@ -1004,12 +1031,12 @@ func (p *ReplicationPool) AddWorker() {
if !ok { if !ok {
return return
} }
replicateObject(p.ctx, oi, p.objLayer) replicateObject(p.ctx, oi, p.objLayer, ReplicateIncoming)
case doi, ok := <-p.replicaDeleteCh: case doi, ok := <-p.replicaDeleteCh:
if !ok { if !ok {
return return
} }
replicateDelete(p.ctx, doi, p.objLayer) replicateDelete(p.ctx, doi, p.objLayer, ReplicateDelete)
case <-p.workerKillCh: case <-p.workerKillCh:
return return
} }
@ -1027,12 +1054,12 @@ func (p *ReplicationPool) AddExistingObjectReplicateWorker() {
if !ok { if !ok {
return return
} }
replicateObject(p.ctx, oi, p.objLayer) replicateObject(p.ctx, oi, p.objLayer, ReplicateExisting)
case doi, ok := <-p.existingReplicaDeleteCh: case doi, ok := <-p.existingReplicaDeleteCh:
if !ok { if !ok {
return return
} }
replicateDelete(p.ctx, doi, p.objLayer) replicateDelete(p.ctx, doi, p.objLayer, ReplicateExistingDelete)
} }
} }
} }
@ -1081,12 +1108,6 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
close(p.existingReplicaCh) close(p.existingReplicaCh)
}) })
case p.mrfReplicaCh <- ri: case p.mrfReplicaCh <- ri:
auditLogInternal(context.Background(), ri.Bucket, ri.Name, AuditLogOptions{
Trigger: ReplicationMRFActivity,
APIName: "s3:ReplicateObject",
VersionID: ri.VersionID,
Status: ri.ReplicationStatus.String(),
})
default: default:
} }
} }
@ -1096,14 +1117,11 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
return return
} }
var ch chan ReplicateObjectInfo var ch chan ReplicateObjectInfo
trigger := ReplicationQueuedActivity
switch ri.OpType { switch ri.OpType {
case replication.ExistingObjectReplicationType: case replication.ExistingObjectReplicationType:
ch = p.existingReplicaCh ch = p.existingReplicaCh
trigger = ReplicationExistingActivity
case replication.HealReplicationType: case replication.HealReplicationType:
ch = p.replicaCh fallthrough
trigger = ReplicationHealActivity
default: default:
ch = p.replicaCh ch = p.replicaCh
} }
@ -1115,12 +1133,6 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
close(p.existingReplicaCh) close(p.existingReplicaCh)
}) })
case ch <- ri: case ch <- ri:
auditLogInternal(context.Background(), ri.Bucket, ri.Name, AuditLogOptions{
Trigger: trigger,
APIName: "s3:ReplicateObject",
VersionID: ri.VersionID,
Status: string(ri.ReplicationStatus),
})
default: default:
} }
} }
@ -1129,15 +1141,13 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
if p == nil { if p == nil {
return return
} }
trigger := ReplicationDeleteActivity
var ch chan DeletedObjectReplicationInfo var ch chan DeletedObjectReplicationInfo
switch doi.OpType { switch doi.OpType {
case replication.ExistingObjectReplicationType: case replication.ExistingObjectReplicationType:
ch = p.existingReplicaDeleteCh ch = p.existingReplicaDeleteCh
trigger = ReplicationExistingDeleteActivity
case replication.HealReplicationType: case replication.HealReplicationType:
ch = p.replicaDeleteCh fallthrough
trigger = ReplicationHealActivity
default: default:
ch = p.replicaDeleteCh ch = p.replicaDeleteCh
} }
@ -1149,16 +1159,6 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
close(p.existingReplicaDeleteCh) close(p.existingReplicaDeleteCh)
}) })
case ch <- doi: case ch <- doi:
replStatus := doi.DeleteMarkerReplicationStatus
if doi.VersionPurgeStatus != "" {
replStatus = string(doi.VersionPurgeStatus)
}
auditLogInternal(context.Background(), doi.Bucket, doi.ObjectName, AuditLogOptions{
Trigger: trigger,
APIName: "s3:ReplicateDelete",
VersionID: doi.VersionID,
Status: replStatus,
})
default: default:
} }
} }
@ -1314,7 +1314,7 @@ func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, op
func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer, sync bool, opType replication.Type) { func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer, sync bool, opType replication.Type) {
if sync { if sync {
replicateObject(ctx, ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType}, o) replicateObject(ctx, ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType}, o, ReplicateIncoming)
} else { } else {
globalReplicationPool.queueReplicaTask(ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType}) globalReplicationPool.queueReplicaTask(ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType})
} }

26
cmd/data-scanner.go
View File

@ -990,7 +990,7 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, meta
opts.VersionID = meta.oi.VersionID opts.VersionID = meta.oi.VersionID
_, err = o.DeleteObject(ctx, meta.oi.Bucket, meta.oi.Name, opts) _, err = o.DeleteObject(ctx, meta.oi.Bucket, meta.oi.Name, opts)
if err == nil { if err == nil {
auditLogLifecycle(ctx, meta.oi.Bucket, meta.oi.Name, meta.oi.VersionID, ILMFreeVersionDeleteActivity) auditLogLifecycle(ctx, meta.oi, ILMFreeVersionDelete)
} }
if ignoreNotFoundErr(err) != nil { if ignoreNotFoundErr(err) != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
@ -1136,7 +1136,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
} }
// Send audit for the lifecycle delete operation // Send audit for the lifecycle delete operation
auditLogLifecycle(ctx, obj.Bucket, obj.Name, obj.VersionID, ILMExpiryActivity) auditLogLifecycle(ctx, obj, ILMExpiry)
eventName := event.ObjectRemovedDelete eventName := event.ObjectRemovedDelete
if obj.DeleteMarker { if obj.DeleteMarker {
@ -1379,23 +1379,23 @@ func (d *dynamicSleeper) Update(factor float64, maxWait time.Duration) error {
} }
const ( const (
// ILMExpiryActivity - activity trail for ILM expiry // ILMExpiry - audit trail for ILM expiry
ILMExpiryActivity = "ilm:expiry" ILMExpiry = "ilm:expiry"
// ILMFreeVersionDeleteActivity - activity trail for ILM free-version delete // ILMFreeVersionDelete - audit trail for ILM free-version delete
ILMFreeVersionDeleteActivity = "ilm:free-version-delete" ILMFreeVersionDelete = "ilm:free-version-delete"
) )
func auditLogLifecycle(ctx context.Context, bucket, object, versionID string, trigger string) { func auditLogLifecycle(ctx context.Context, oi ObjectInfo, trigger string) {
var apiName string var apiName string
switch trigger { switch trigger {
case ILMExpiryActivity: case ILMExpiry:
apiName = "s3:ExpireObject" apiName = "ILMExpiry"
case ILMFreeVersionDeleteActivity: case ILMFreeVersionDelete:
apiName = "s3:DeleteFreeVersion" apiName = "ILMFreeVersionDelete"
} }
auditLogInternal(ctx, bucket, object, AuditLogOptions{ auditLogInternal(ctx, oi.Bucket, oi.Name, AuditLogOptions{
Trigger: trigger, Trigger: trigger,
APIName: apiName, APIName: apiName,
VersionID: versionID, VersionID: oi.VersionID,
}) })
} }

9
internal/logger/audit.go
View File

@ -142,7 +142,11 @@ func GetAuditEntry(ctx context.Context) *audit.Entry {
if ok { if ok {
return r return r
} }
r = &audit.Entry{} r = &audit.Entry{
Version: audit.Version,
DeploymentID: globalDeploymentID,
Time: time.Now().UTC().Format(time.RFC3339Nano),
}
SetAuditEntry(ctx, r) SetAuditEntry(ctx, r)
return r return r
} }
@ -165,7 +169,8 @@ func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request, reqCl
} }
entry = audit.ToEntry(w, r, reqClaims, globalDeploymentID) entry = audit.ToEntry(w, r, reqClaims, globalDeploymentID)
entry.Trigger = "external-request" // indicates all requests for this API call are inbound
entry.Trigger = "incoming"
for _, filterKey := range filterKeys { for _, filterKey := range filterKeys {
delete(entry.ReqClaims, filterKey) delete(entry.ReqClaims, filterKey)