fix: metacache should only rename entries during cleanup (#11503)

To avoid large delays in metacache cleanup, use rename
instead of recursive delete calls, renames are cheaper
move the content to minioMetaTmpBucket and then cleanup
this folder once in 24hrs instead.

If the new cache can replace an existing one, we should
let it replace since that is currently being saved anyways,
this avoids pile up of 1000's of metacache entires for
same listing calls that are not necessary to be stored
on disk.

cmd/erasure-server-pool.go

@@ -1121,22 +1121,24 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, fo
 // data is not distributed across sets.
 // Errors are logged but individual disk failures are not returned.
 func (z *erasureServerPools) deleteAll(ctx context.Context, bucket, prefix string) {
-	var wg sync.WaitGroup
 	for _, servers := range z.serverPools {
 		for _, set := range servers.sets {
-			for _, disk := range set.getDisks() {
-				if disk == nil {
-					continue
-				}
-				wg.Add(1)
-				go func(disk StorageAPI) {
-					defer wg.Done()
-					disk.Delete(ctx, bucket, prefix, true)
-				}(disk)
-			}
+			set.deleteAll(ctx, bucket, prefix)
+		}
+	}
+}
+
+// renameAll will rename bucket+prefix unconditionally across all disks to
+// minioMetaTmpBucket + unique uuid,
+// Note that set distribution is ignored so it should only be used in cases where
+// data is not distributed across sets. Errors are logged but individual
+// disk failures are not returned.
+func (z *erasureServerPools) renameAll(ctx context.Context, bucket, prefix string) {
+	for _, servers := range z.serverPools {
+		for _, set := range servers.sets {
+			set.renameAll(ctx, bucket, prefix)
 		}
 	}
-	wg.Wait()
 }
 
 // This function is used to undo a successful DeleteBucket operation.