Clean entire tmp-old on restart (#15979)

cmd/erasure-sets.go

@@ -117,7 +117,7 @@ func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI {
 // Initializes a new StorageAPI from the endpoint argument, returns
 // StorageAPI and also `format` which exists on the disk.
 func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
-	disk, err := newStorageAPIWithoutHealthCheck(endpoint)
+	disk, err := newStorageAPI(endpoint, false)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -251,7 +251,7 @@ func (s *erasureSets) connectDisks() {
 				s.erasureDisks[setIndex][diskIndex] = disk
 			} else {
 				// Enable healthcheck disk for remote endpoint.
-				disk, err = newStorageAPI(endpoint)
+				disk, err = newStorageAPI(endpoint, true)
 				if err != nil {
 					printEndpointError(endpoint, err, false)
 					s.erasureDisksMu.Unlock()
@@ -1250,7 +1250,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 
 // HealFormat - heals missing `format.json` on fresh unformatted disks.
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
-	storageDisks, _ := initStorageDisksWithErrorsWithoutHealthCheck(s.endpoints.Endpoints)
+	storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, false)
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {

cmd/format-erasure.go

@@ -666,30 +666,16 @@ func closeStorageDisks(storageDisks ...StorageAPI) {
 	wg.Wait()
 }
 
-func initStorageDisksWithErrorsWithoutHealthCheck(endpoints Endpoints) ([]StorageAPI, []error) {
-	// Bootstrap disks.
-	storageDisks := make([]StorageAPI, len(endpoints))
-	g := errgroup.WithNErrs(len(endpoints))
-	for index := range endpoints {
-		index := index
-		g.Go(func() (err error) {
-			storageDisks[index], err = newStorageAPIWithoutHealthCheck(endpoints[index])
-			return err
-		}, index)
-	}
-	return storageDisks, g.Wait()
-}
-
 // Initialize storage disks for each endpoint.
 // Errors are returned for each endpoint with matching index.
-func initStorageDisksWithErrors(endpoints Endpoints) ([]StorageAPI, []error) {
+func initStorageDisksWithErrors(endpoints Endpoints, healthCheck bool) ([]StorageAPI, []error) {
 	// Bootstrap disks.
 	storageDisks := make([]StorageAPI, len(endpoints))
 	g := errgroup.WithNErrs(len(endpoints))
 	for index := range endpoints {
 		index := index
 		g.Go(func() (err error) {
-			storageDisks[index], err = newStorageAPI(endpoints[index])
+			storageDisks[index], err = newStorageAPI(endpoints[index], healthCheck)
 			return err
 		}, index)
 	}

cmd/format-erasure_test.go

@@ -38,7 +38,7 @@ func TestFixFormatV3(t *testing.T) {
 	}
 	endpoints := mustGetNewEndpoints(erasureDirs...)
 
-	storageDisks, errs := initStorageDisksWithErrors(endpoints)
+	storageDisks, errs := initStorageDisksWithErrors(endpoints, true)
 	for _, err := range errs {
 		if err != nil && err != errDiskNotFound {
 			t.Fatal(err)
@@ -559,7 +559,7 @@ func benchmarkInitStorageDisksN(b *testing.B, nDisks int) {
 	b.RunParallel(func(pb *testing.PB) {
 		endpoints := endpoints
 		for pb.Next() {
-			initStorageDisksWithErrors(endpoints)
+			initStorageDisksWithErrors(endpoints, true)
 		}
 	})
 }

cmd/object-api-common.go

@@ -55,27 +55,15 @@ var globalObjectAPI ObjectLayer
 // Global cacheObjects, only accessed by newCacheObjectsFn().
 var globalCacheObjectAPI CacheObjectLayer
 
-func newStorageAPIWithoutHealthCheck(endpoint Endpoint) (storage StorageAPI, err error) {
-	if endpoint.IsLocal {
-		storage, err := newXLStorage(endpoint)
-		if err != nil {
-			return nil, err
-		}
-		return newXLStorageDiskIDCheck(storage), nil
-	}
-
-	return newStorageRESTClient(endpoint, false), nil
-}
-
 // Depending on the disk type network or local, initialize storage API.
-func newStorageAPI(endpoint Endpoint) (storage StorageAPI, err error) {
+func newStorageAPI(endpoint Endpoint, healthCheck bool) (storage StorageAPI, err error) {
 	if endpoint.IsLocal {
-		storage, err := newXLStorage(endpoint)
+		storage, err := newXLStorage(endpoint, healthCheck)
 		if err != nil {
 			return nil, err
 		}
 		return newXLStorageDiskIDCheck(storage), nil
 	}
 
-	return newStorageRESTClient(endpoint, true), nil
+	return newStorageRESTClient(endpoint, healthCheck), nil
 }

cmd/prepare-storage.go

@@ -97,7 +97,8 @@ func bgFormatErasureCleanupTmp(diskPath string) {
 			err))
 	}
 
-	go removeAll(tmpOld)
+	// Remove the entire folder in case there are leftovers that didn't get cleaned up before restart.
+	go removeAll(pathJoin(diskPath, minioMetaTmpBucket+"-old"))
 	// Renames and schedules for purging all bucket metacache.
 	go renameAllBucketMetacache(diskPath)
 }
@@ -146,7 +147,7 @@ func isServerResolvable(endpoint Endpoint, timeout time.Duration) error {
 // time. additionally make sure to close all the disks used in this attempt.
 func connectLoadInitFormats(verboseLogging bool, firstDisk bool, endpoints Endpoints, poolCount, setCount, setDriveCount int, deploymentID, distributionAlgo string) (storageDisks []StorageAPI, format *formatErasureV3, err error) {
 	// Initialize all storage disks
-	storageDisks, errs := initStorageDisksWithErrors(endpoints)
+	storageDisks, errs := initStorageDisksWithErrors(endpoints, true)
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {

cmd/storage-rest-server.go

@@ -1323,7 +1323,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 			go func(poolIdx, setIdx int, endpoint Endpoint) {
 				defer wg.Done()
 				var err error
-				storageDisks[poolIdx][setIdx], err = newXLStorage(endpoint)
+				storageDisks[poolIdx][setIdx], err = newXLStorage(endpoint, false)
 				if err != nil {
 					// if supported errors don't fail, we proceed to
 					// printing message and moving forward.

cmd/xl-storage.go

@@ -206,11 +206,11 @@ func newLocalXLStorage(path string) (*xlStorage, error) {
 	return newXLStorage(Endpoint{
 		URL:     &u,
 		IsLocal: true,
-	})
+	}, true)
 }
 
 // Initialize a new storage disk.
-func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
+func newXLStorage(ep Endpoint, cleanUp bool) (s *xlStorage, err error) {
 	path := ep.Path
 	if path, err = getValidPath(path); err != nil {
 		return nil, err
@@ -247,7 +247,9 @@ func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
 		diskIndex:  -1,
 	}
 
-	bgFormatErasureCleanupTmp(s.diskPath) // cleanup any old data.
+	if cleanUp {
+		bgFormatErasureCleanupTmp(s.diskPath) // cleanup any old data.
+	}
 
 	formatData, formatFi, err := formatErasureMigrate(s.diskPath)
 	if err != nil && !errors.Is(err, os.ErrNotExist) {