xl: Avoid multi-disks node to exit when one disk fails (#12423)

It makes sense that a node that has multiple disks starts when one
disk fails, returning an i/o error for example. This commit will make this
faulty tolerance available in this specific use case.

cmd/endpoint.go

@@ -257,18 +257,17 @@ func (l EndpointServerPools) Localhost() string {
 	return ""
 }
 
-// FirstLocalDiskPath returns the disk path of first (in cmdline args order)
+// LocalDisksPaths returns the disk paths of the local disks
-// local endpoint.
+func (l EndpointServerPools) LocalDisksPaths() []string {
-func (l EndpointServerPools) FirstLocalDiskPath() string {
+	var disks []string
-	var diskPath string
 	for _, ep := range l {
 		for _, endpoint := range ep.Endpoints {
 			if endpoint.IsLocal {
-				return endpoint.Path
+				disks = append(disks, endpoint.Path)
 			}
 		}
 	}
-	return diskPath
+	return disks
 }
 
 // FirstLocal returns true if the first endpoint is local.

cmd/prepare-storage.go

@@ -20,18 +20,15 @@ package cmd
 import (
 	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
-	"os"
 	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 )
 
 var printEndpointError = func() func(Endpoint, error, bool) {
@@ -72,41 +69,8 @@ var printEndpointError = func() func(Endpoint, error, bool) {
 	}
 }()
 
-// Migrates backend format of local disks.
+// Cleans up tmp directory of the local disk.
-func formatErasureMigrateLocalEndpoints(endpoints Endpoints) error {
+func formatErasureCleanupTmp(diskPath string) error {
-	g := errgroup.WithNErrs(len(endpoints))
-	for index, endpoint := range endpoints {
-		if !endpoint.IsLocal {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			epPath := endpoints[index].Path
-			err := formatErasureMigrate(epPath)
-			if err != nil && !errors.Is(err, os.ErrNotExist) {
-				return err
-			}
-			return nil
-		}, index)
-	}
-	for _, err := range g.Wait() {
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// Cleans up tmp directory of local disks.
-func formatErasureCleanupTmpLocalEndpoints(endpoints Endpoints) error {
-	g := errgroup.WithNErrs(len(endpoints))
-	for index, endpoint := range endpoints {
-		if !endpoint.IsLocal {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			epPath := endpoints[index].Path
 	// Need to move temporary objects left behind from previous run of minio
 	// server to a unique directory under `minioMetaTmpBucket-old` to clean
 	// up `minioMetaTmpBucket` for the current run.
@@ -117,35 +81,27 @@ func formatErasureCleanupTmpLocalEndpoints(endpoints Endpoints) error {
 	//
 	// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
 	// temporary objects from one of the previous runs of minio server.
-			tmpOld := pathJoin(epPath, minioMetaTmpBucket+"-old", mustGetUUID())
+	tmpOld := pathJoin(diskPath, minioMetaTmpBucket+"-old", mustGetUUID())
-			if err := renameAll(pathJoin(epPath, minioMetaTmpBucket),
+	if err := renameAll(pathJoin(diskPath, minioMetaTmpBucket),
 		tmpOld); err != nil && err != errFileNotFound {
 		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
-					pathJoin(epPath, minioMetaTmpBucket),
+			pathJoin(diskPath, minioMetaTmpBucket),
 			tmpOld,
 			osErrToFileErr(err)))
 	}
 
-			// Renames and schedules for puring all bucket metacache.
+	// Renames and schedules for purging all bucket metacache.
-			renameAllBucketMetacache(epPath)
+	renameAllBucketMetacache(diskPath)
 
 	// Removal of tmp-old folder is backgrounded completely.
-			go removeAll(pathJoin(epPath, minioMetaTmpBucket+"-old"))
+	go removeAll(pathJoin(diskPath, minioMetaTmpBucket+"-old"))
 
-			if err := mkdirAll(pathJoin(epPath, minioMetaTmpBucket), 0777); err != nil {
+	if err := mkdirAll(pathJoin(diskPath, minioMetaTmpBucket), 0777); err != nil {
 		logger.LogIf(GlobalContext, fmt.Errorf("unable to create (%s) %w, drive may be faulty please investigate",
-					pathJoin(epPath, minioMetaTmpBucket),
+			pathJoin(diskPath, minioMetaTmpBucket),
 			err))
 	}
 	return nil
-		}, index)
-	}
-	for _, err := range g.Wait() {
-		if err != nil {
-			return err
-		}
-	}
-	return nil
 }
 
 // Following error message is added to fix a regression in release
@@ -337,14 +293,6 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCou
 		return nil, nil, errInvalidArgument
 	}
 
-	if err := formatErasureMigrateLocalEndpoints(endpoints); err != nil {
-		return nil, nil, err
-	}
-
-	if err := formatErasureCleanupTmpLocalEndpoints(endpoints); err != nil {
-		return nil, nil, err
-	}
-
 	// prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks.
 	// All times are rounded to avoid showing milli, micro and nano seconds
 	formatStartTime := time.Now().Round(time.Second)

cmd/server-main.go

@@ -555,7 +555,7 @@ func serverMain(ctx *cli.Context) {
 
 	if globalIsErasure { // to be done after config init
 		initBackgroundReplication(GlobalContext, newObject)
-		globalTierJournal, err = initTierDeletionJournal(GlobalContext.Done())
+		globalTierJournal, err = initTierDeletionJournal(GlobalContext)
 		if err != nil {
 			logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
 		}

cmd/tier-journal.go

@@ -56,25 +56,30 @@ var (
 	errUnsupportedJournalVersion = errors.New("unsupported pending deletes journal version")
 )
 
-func initTierDeletionJournal(done <-chan struct{}) (*tierJournal, error) {
+func initTierDeletionJournal(ctx context.Context) (*tierJournal, error) {
-	diskPath := globalEndpoints.FirstLocalDiskPath()
+	for _, diskPath := range globalEndpoints.LocalDisksPaths() {
 		j := &tierJournal{
 			diskPath: diskPath,
 		}
 
 		if err := os.MkdirAll(filepath.Dir(j.JournalPath()), os.FileMode(0700)); err != nil {
-		return nil, err
+			logger.LogIf(ctx, err)
+			continue
 		}
 
 		err := j.Open()
 		if err != nil {
-		return nil, err
+			logger.LogIf(ctx, err)
+			continue
 		}
 
-	go j.deletePending(done)
+		go j.deletePending(ctx.Done())
 		return j, nil
 	}
 
+	return nil, errors.New("no local disk found")
+}
+
 // rotate rotates the journal. If a read-only journal already exists it does
 // nothing. Otherwise renames the active journal to a read-only journal and
 // opens a new active journal.

cmd/xl-storage.go

@@ -300,7 +300,15 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) {
 		return p, err
 	}
 	w.Close()
-	defer Remove(filePath)
+	Remove(filePath)
+
+	if err := formatErasureMigrate(p.diskPath); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return p, err
+	}
+
+	if err := formatErasureCleanupTmp(p.diskPath); err != nil {
+		return p, err
+	}
 
 	// Success.
 	return p, nil