Add reliable RemoveAll to handle racy situations (#6227)

cmd/format-xl.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"os"
 	"reflect"
 	"sync"
 
@@ -267,10 +266,11 @@ func formatXLMigrateV2ToV3(export string) error {
 		return err
 	}
 
-	if err = os.RemoveAll(pathJoin(export, minioMetaMultipartBucket)); err != nil {
+	if err = removeAll(pathJoin(export, minioMetaMultipartBucket)); err != nil {
 		return err
 	}
-	if err = os.MkdirAll(pathJoin(export, minioMetaMultipartBucket), 0755); err != nil {
+
+	if err = mkdirAll(pathJoin(export, minioMetaMultipartBucket), 0755); err != nil {
 		return err
 	}
 

cmd/fs-v1-helpers.go

@@ -64,7 +64,7 @@ func fsRemoveAll(ctx context.Context, dirPath string) (err error) {
 		return err
 	}
 
-	if err = os.RemoveAll(dirPath); err != nil {
+	if err = removeAll(dirPath); err != nil {
 		if os.IsPermission(err) {
 			logger.LogIf(ctx, errVolumeAccessDenied)
 			return errVolumeAccessDenied

cmd/os-reliable.go

@@ -22,6 +22,53 @@ import (
 	"path"
 )
 
+// Wrapper functions to os.RemoveAll, which calls reliableRemoveAll
+// this is to ensure that if there is a racy parent directory
+// create in between we can simply retry the operation.
+func removeAll(dirPath string) (err error) {
+	if dirPath == "" {
+		return errInvalidArgument
+	}
+
+	if err = checkPathLength(dirPath); err != nil {
+		return err
+	}
+
+	if err = reliableRemoveAll(dirPath); err != nil {
+		switch {
+		case isSysErrNotDir(err):
+			// File path cannot be verified since one of
+			// the parents is a file.
+			return errFileAccessDenied
+		case isSysErrPathNotFound(err):
+			// This is a special case should be handled only for
+			// windows, because windows API does not return "not a
+			// directory" error message. Handle this specifically
+			// here.
+			return errFileAccessDenied
+		}
+	}
+	return err
+}
+
+// Reliably retries os.RemoveAll if for some reason os.RemoveAll returns
+// syscall.ENOTEMPTY (children has files).
+func reliableRemoveAll(dirPath string) (err error) {
+	i := 0
+	for {
+		// Removes all the directories and files.
+		if err = os.RemoveAll(dirPath); err != nil {
+			// Retry only for the first retryable error.
+			if isSysErrNotEmpty(err) && i == 0 {
+				i++
+				continue
+			}
+		}
+		break
+	}
+	return err
+}
+
 // Wrapper functions to os.MkdirAll, which calls reliableMkdirAll
 // this is to ensure that if there is a racy parent directory
 // delete in between we can simply retry the operation.

cmd/prepare-storage.go

@@ -80,10 +80,10 @@ func formatXLCleanupTmpLocalEndpoints(endpoints EndpointList) error {
 			}
 			return err
 		}
-		if err := os.RemoveAll(pathJoin(endpoint.Path, minioMetaTmpBucket)); err != nil {
+		if err := removeAll(pathJoin(endpoint.Path, minioMetaTmpBucket)); err != nil {
 			return err
 		}
-		if err := os.MkdirAll(pathJoin(endpoint.Path, minioMetaTmpBucket), 0777); err != nil {
+		if err := mkdirAll(pathJoin(endpoint.Path, minioMetaTmpBucket), 0777); err != nil {
 			return err
 		}
 	}