relax O_DIRECT in single drive mode if unsupported (#15045)

cmd/xl-storage.go

@@ -94,8 +94,8 @@ type xlStorage struct {
 	endpoint Endpoint
 
 	globalSync bool
-
+	oDirect    bool // indicates if this disk supports ODirect
-	rootDisk bool
+	rootDisk   bool
 
 	diskID string
 
@@ -264,15 +264,27 @@ func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
 		filePath := pathJoin(s.diskPath, ".writable-check-"+uuid+".tmp")
 		w, err := s.openFileDirect(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL)
 		if err != nil {
-			return s, err
+			// relax single drive mode without O_DIRECT support
+			if globalIsErasureSD && errors.Is(err, errUnsupportedDisk) {
+				s.oDirect = false
+			} else {
+				return s, err
+			}
 		}
 		_, err = w.Write(alignedBuf)
 		w.Close()
 		if err != nil {
 			if isSysErrInvalidArg(err) {
-				return s, errUnsupportedDisk
+				err = errUnsupportedDisk
 			}
-			return s, err
+			// relax single drive mode without O_DIRECT support
+			if globalIsErasureSD && errors.Is(err, errUnsupportedDisk) {
+				s.oDirect = false
+			} else {
+				return s, err
+			}
+		} else {
+			s.oDirect = true
 		}
 		renameAll(filePath, pathJoin(s.diskPath, minioMetaTmpDeletedBucket, uuid))
 
@@ -289,6 +301,7 @@ func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
 		s.diskID = format.Erasure.This
 		s.formatLastCheck = time.Now()
 		s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoV1
+		s.oDirect = true
 	}
 
 	// Success.
@@ -1372,7 +1385,13 @@ func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath
 		return nil, time.Time{}, ctx.Err()
 	}
 
-	f, err := OpenFileDirectIO(filePath, readMode, 0o666)
+	odirectEnabled := !globalAPIConfig.isDisableODirect() && s.oDirect
+	var f *os.File
+	if odirectEnabled {
+		f, err = OpenFileDirectIO(filePath, readMode, 0o666)
+	} else {
+		f, err = OpenFile(filePath, readMode, 0o666)
+	}
 	if err != nil {
 		if osIsNotExist(err) {
 			// Check if the object doesn't exist because its bucket
@@ -1662,7 +1681,14 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		return nil, err
 	}
 
-	file, err := OpenFileDirectIO(filePath, readMode, 0o666)
+	odirectEnabled := !globalAPIConfig.isDisableODirect() && s.oDirect
+
+	var file *os.File
+	if odirectEnabled {
+		file, err = OpenFileDirectIO(filePath, readMode, 0o666)
+	} else {
+		file, err = OpenFile(filePath, readMode, 0o666)
+	}
 	if err != nil {
 		switch {
 		case osIsNotExist(err):
@@ -1706,7 +1732,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 	}
 
 	alignment := offset%xioutil.DirectioAlignSize == 0
-	if !alignment || globalAPIConfig.isDisableODirect() {
+	if !alignment && odirectEnabled {
 		if err = disk.DisableDirectIO(file); err != nil {
 			file.Close()
 			return nil, err
@@ -1736,13 +1762,11 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		io.Reader
 		io.Closer
 	}{Reader: io.LimitReader(diskHealthReader(ctx, or), length), Closer: closeWrapper(func() error {
-		if !alignment || offset+length%xioutil.DirectioAlignSize != 0 {
+		if (!alignment || offset+length%xioutil.DirectioAlignSize != 0) && odirectEnabled {
 			// invalidate page-cache for unaligned reads.
-			if !globalAPIConfig.isDisableODirect() {
+			// skip removing from page-cache only
-				// skip removing from page-cache only
+			// if O_DIRECT was disabled.
-				// if O_DIRECT was disabled.
+			disk.FadviseDontNeed(file)
-				disk.FadviseDontNeed(file)
-			}
 		}
 		return or.Close()
 	})}
@@ -1814,7 +1838,13 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		return osErrToFileErr(err)
 	}
 
-	w, err := OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
+	odirectEnabled := s.oDirect
+	var w *os.File
+	if odirectEnabled {
+		w, err = OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
+	} else {
+		w, err = OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
+	}
 	if err != nil {
 		return osErrToFileErr(err)
 	}
@@ -1834,7 +1864,12 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		defer xioutil.ODirectPoolLarge.Put(bufp)
 	}
 
-	written, err := xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w)
+	var written int64
+	if odirectEnabled {
+		written, err = xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w)
+	} else {
+		written, err = io.CopyBuffer(diskHealthWriter(ctx, w), r, *bufp)
+	}
 	if err != nil {
 		return err
 	}