From 36385010f5f09dc504e4589d6321898e382e3a76 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 16 Sep 2023 19:08:59 -0700 Subject: [PATCH] use optimized pathJoin instead of path.Join (#18042) this avoids allocations in scanner routine, they are tiny but they allocate a lot over many cycles of the scanner. --- cmd/batch-handlers.go | 5 ++--- cmd/data-scanner.go | 12 ++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/batch-handlers.go b/cmd/batch-handlers.go index 55ef8f3f6..d02a6f32b 100644 --- a/cmd/batch-handlers.go +++ b/cmd/batch-handlers.go @@ -28,7 +28,6 @@ import ( "math/rand" "net/http" "net/url" - "path" "runtime" "strconv" "strings" @@ -114,7 +113,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec srcObject := srcObjInfo.Name tgtObject := srcObjInfo.Name if r.Target.Prefix != "" { - tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name) + tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name) } versionID := srcObjInfo.VersionID @@ -182,7 +181,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a srcObject := srcObjInfo.Name tgtObject := srcObjInfo.Name if r.Target.Prefix != "" { - tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name) + tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name) } if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 { opts.VersionID = "" diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 04b0f66b8..2210e157b 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -409,9 +409,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int var existingFolders, newFolders []cachedFolder var foundObjects bool - err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error { + err := readDirFn(pathJoin(f.root, folder.name), func(entName string, typ os.FileMode) error { // Parse - entName = pathClean(path.Join(folder.name, entName)) + entName = pathClean(pathJoin(folder.name, entName)) if entName == "" || entName == folder.name { if f.dataUsageScannerDebug { console.Debugf(scannerLogPrefix+" no entity (%s,%s)\n", f.root, entName) @@ -461,7 +461,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int // Get file size, ignore errors. item := scannerItem{ - Path: path.Join(f.root, entName), + Path: pathJoin(f.root, entName), Typ: typ, bucket: bucket, prefix: path.Dir(prefix), @@ -496,7 +496,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int // Object already accounted for, remove from heal map, // simply because getSize() function already heals the // object. - delete(abandonedChildren, path.Join(item.bucket, item.objectPath())) + delete(abandonedChildren, pathJoin(item.bucket, item.objectPath())) into.addSizes(sz) into.Objects++ @@ -873,7 +873,7 @@ type getSizeFn func(item scannerItem) (sizeSummary, error) func (i *scannerItem) transformMetaDir() { split := strings.Split(i.prefix, SlashSeparator) if len(split) > 1 { - i.prefix = path.Join(split[:len(split)-1]...) + i.prefix = pathJoin(split[:len(split)-1]...) } else { i.prefix = "" } @@ -1247,7 +1247,7 @@ func applyLifecycleAction(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) // objectPath returns the prefix and object name. func (i *scannerItem) objectPath() string { - return path.Join(i.prefix, i.objectName) + return pathJoin(i.prefix, i.objectName) } // healReplication will heal a scanned item that has failed replication.