flatten out audit tags, do not send as free-form (#20256)

move away from map[string]interface{} to map[string]string
to simplify the audit, and also provide concise information.

avoids large allocations under load(), reduces the amount
of audit information generated, as the current implementation
was a bit free-form. instead all datastructures must be
flattened.
This commit is contained in:
Harshavardhana
2024-08-13 15:22:04 -07:00
committed by GitHub
parent 516af01a12
commit e7a56f35b9
11 changed files with 100 additions and 109 deletions

View File

@@ -70,7 +70,7 @@ func countOnlineDisks(onlineDisks []StorageAPI) (online int) {
// update metadata.
func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) {
if !dstOpts.NoAuditLog {
auditObjectErasureSet(ctx, dstObject, &er)
auditObjectErasureSet(ctx, "CopyObject", dstObject, &er)
}
// This call shouldn't be used for anything other than metadata updates or adding self referential versions.
@@ -199,7 +199,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
// Read(Closer). When err != nil, the returned reader is always nil.
func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
if !opts.NoAuditLog {
auditObjectErasureSet(ctx, object, &er)
auditObjectErasureSet(ctx, "GetObject", object, &er)
}
var unlockOnDefer bool
@@ -439,7 +439,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
// GetObjectInfo - reads object metadata and replies back ObjectInfo.
func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
if !opts.NoAuditLog {
auditObjectErasureSet(ctx, object, &er)
auditObjectErasureSet(ctx, "GetObjectInfo", object, &er)
}
if !opts.NoLock {
@@ -456,7 +456,7 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
return er.getObjectInfo(ctx, bucket, object, opts)
}
func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID string, tags map[string]interface{}) {
func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID string, tags map[string]string) {
if len(logger.AuditTargets()) == 0 {
return
}
@@ -472,13 +472,16 @@ func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID
auditLogInternal(ctx, opts)
}
func joinErrs(errs []error) []string {
s := make([]string, len(errs))
func joinErrs(errs []error) string {
var s string
for i := range s {
if s != "" {
s += ","
}
if errs[i] == nil {
s[i] = "<nil>"
s += "<nil>"
} else {
s[i] = errs[i].Error()
s += errs[i].Error()
}
}
return s
@@ -491,20 +494,18 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
// can be deleted safely, in such a scenario return ReadQuorum error.
return FileInfo{}, errErasureReadQuorum
}
tags := make(map[string]interface{}, 4)
tags["set"] = er.setIndex
tags["pool"] = er.poolIndex
tags := make(map[string]string, 16)
tags["set"] = strconv.Itoa(er.setIndex)
tags["pool"] = strconv.Itoa(er.poolIndex)
tags["merrs"] = joinErrs(errs)
tags["derrs"] = dataErrsByPart
tags["derrs"] = fmt.Sprintf("%v", dataErrsByPart)
if m.IsValid() {
tags["size"] = m.Size
tags["mtime"] = m.ModTime.Format(http.TimeFormat)
tags["data"] = m.Erasure.DataBlocks
tags["parity"] = m.Erasure.ParityBlocks
tags["sz"] = strconv.FormatInt(m.Size, 10)
tags["mt"] = m.ModTime.Format(iso8601Format)
tags["d:p"] = fmt.Sprintf("%d:%d", m.Erasure.DataBlocks, m.Erasure.ParityBlocks)
} else {
tags["invalid-meta"] = true
tags["data"] = er.setDriveCount - er.defaultParityCount
tags["parity"] = er.defaultParityCount
tags["invalid"] = "1"
tags["d:p"] = fmt.Sprintf("%d:%d", er.setDriveCount-er.defaultParityCount, er.defaultParityCount)
}
// count the number of offline disks
@@ -527,7 +528,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
}
}
if offline > 0 {
tags["offline"] = offline
tags["offline"] = strconv.Itoa(offline)
}
_, file, line, cok := runtime.Caller(1)
@@ -556,22 +557,16 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
}, index)
}
rmDisks := make(map[string]string, len(disks))
for index, err := range g.Wait() {
var errStr, diskName string
var errStr string
if err != nil {
errStr = err.Error()
} else {
errStr = "<nil>"
}
if disks[index] != nil {
diskName = disks[index].String()
} else {
diskName = fmt.Sprintf("disk-%d", index)
}
rmDisks[diskName] = errStr
tags[fmt.Sprintf("ddisk-%d", index)] = errStr
}
tags["cleanupResult"] = rmDisks
return m, nil
}
@@ -1251,7 +1246,7 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
// putObject wrapper for erasureObjects PutObject
func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
if !opts.NoAuditLog {
auditObjectErasureSet(ctx, object, &er)
auditObjectErasureSet(ctx, "PutObject", object, &er)
}
data := r.Reader
@@ -1626,7 +1621,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
if !opts.NoAuditLog {
for _, obj := range objects {
auditObjectErasureSet(ctx, obj.ObjectV.ObjectName, &er)
auditObjectErasureSet(ctx, "DeleteObjects", obj.ObjectV.ObjectName, &er)
}
}
@@ -1846,7 +1841,7 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
// response to the client request.
func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
if !opts.NoAuditLog {
auditObjectErasureSet(ctx, object, &er)
auditObjectErasureSet(ctx, "DeleteObject", object, &er)
}
var lc *lifecycle.Lifecycle