Add reliable RemoveAll to handle racy situations (#6227)

cmd/os-reliable.go

@@ -22,6 +22,53 @@ import (
 	"path"
 )
 
+// Wrapper functions to os.RemoveAll, which calls reliableRemoveAll
+// this is to ensure that if there is a racy parent directory
+// create in between we can simply retry the operation.
+func removeAll(dirPath string) (err error) {
+	if dirPath == "" {
+		return errInvalidArgument
+	}
+
+	if err = checkPathLength(dirPath); err != nil {
+		return err
+	}
+
+	if err = reliableRemoveAll(dirPath); err != nil {
+		switch {
+		case isSysErrNotDir(err):
+			// File path cannot be verified since one of
+			// the parents is a file.
+			return errFileAccessDenied
+		case isSysErrPathNotFound(err):
+			// This is a special case should be handled only for
+			// windows, because windows API does not return "not a
+			// directory" error message. Handle this specifically
+			// here.
+			return errFileAccessDenied
+		}
+	}
+	return err
+}
+
+// Reliably retries os.RemoveAll if for some reason os.RemoveAll returns
+// syscall.ENOTEMPTY (children has files).
+func reliableRemoveAll(dirPath string) (err error) {
+	i := 0
+	for {
+		// Removes all the directories and files.
+		if err = os.RemoveAll(dirPath); err != nil {
+			// Retry only for the first retryable error.
+			if isSysErrNotEmpty(err) && i == 0 {
+				i++
+				continue
+			}
+		}
+		break
+	}
+	return err
+}
+
 // Wrapper functions to os.MkdirAll, which calls reliableMkdirAll
 // this is to ensure that if there is a racy parent directory
 // delete in between we can simply retry the operation.