fix: metacache should only rename entries during cleanup (#11503)

To avoid large delays in metacache cleanup, use rename
instead of recursive delete calls, renames are cheaper
move the content to minioMetaTmpBucket and then cleanup
this folder once in 24hrs instead.

If the new cache can replace an existing one, we should
let it replace since that is currently being saved anyways,
this avoids pile up of 1000's of metacache entires for
same listing calls that are not necessary to be stored
on disk.

cmd/metacache-server-pool.go

@@ -19,7 +19,9 @@ package cmd
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
+	"os"
 	"path"
 	"strings"
 	"sync"
@@ -28,6 +30,24 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
+func renameAllBucketMetacache(epPath string) error {
+	// Rename all previous `.minio.sys/buckets/<bucketname>/.metacache` to
+	// to `.minio.sys/tmp/` for deletion.
+	return readDirFilterFn(pathJoin(epPath, minioMetaBucket, bucketMetaPrefix), func(name string, typ os.FileMode) error {
+		if typ == os.ModeDir {
+			tmpMetacacheOld := pathJoin(epPath, minioMetaTmpBucket+"-old", mustGetUUID())
+			if err := renameAll(pathJoin(epPath, minioMetaBucket, metacachePrefixForID(name, slashSeparator)),
+				tmpMetacacheOld); err != nil && err != errFileNotFound {
+				return fmt.Errorf("unable to rename (%s -> %s) %w",
+					pathJoin(epPath, minioMetaBucket+metacachePrefixForID(minioMetaBucket, slashSeparator)),
+					tmpMetacacheOld,
+					osErrToFileErr(err))
+			}
+		}
+		return nil
+	})
+}
+
 // listPath will return the requested entries.
 // If no more entries are in the listing io.EOF is returned,
 // otherwise nil or an unexpected error is returned.