avoid sendFile() for ranges or object lengths < 4MiB (#20141)

cmd/storage-rest-server.go

@@ -36,6 +36,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/grid"
 	"github.com/tinylib/msgp/msgp"
 
@@ -594,44 +595,47 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
 	}
 	volume := r.Form.Get(storageRESTVolume)
 	filePath := r.Form.Get(storageRESTFilePath)
-	offset, err := strconv.Atoi(r.Form.Get(storageRESTOffset))
+	offset, err := strconv.ParseInt(r.Form.Get(storageRESTOffset), 10, 64)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-	length, err := strconv.Atoi(r.Form.Get(storageRESTLength))
+	length, err := strconv.ParseInt(r.Form.Get(storageRESTLength), 10, 64)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
 
-	w.Header().Set(xhttp.ContentLength, strconv.Itoa(length))
+	w.Header().Set(xhttp.ContentLength, strconv.FormatInt(length, 10))
 
-	rc, err := s.getStorage().ReadFileStream(r.Context(), volume, filePath, int64(offset), int64(length))
+	rc, err := s.getStorage().ReadFileStream(r.Context(), volume, filePath, offset, length)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
 	defer rc.Close()
 
-	rf, ok := w.(io.ReaderFrom)
+	noReadFrom := runtime.GOOS == "windows" || length < 4*humanize.MiByte
-	if ok && runtime.GOOS != "windows" {
+	if !noReadFrom {
-		// Attempt to use splice/sendfile() optimization, A very specific behavior mentioned below is necessary.
+		rf, ok := w.(io.ReaderFrom)
-		// See https://github.com/golang/go/blob/f7c5cbb82087c55aa82081e931e0142783700ce8/src/net/sendfile_linux.go#L20
-		// Windows can lock up with this optimization, so we fall back to regular copy.
-		sr, ok := rc.(*sendFileReader)
 		if ok {
-			// Sendfile sends in 4MiB chunks per sendfile syscall which is more than enough
+			// Attempt to use splice/sendfile() optimization, A very specific behavior mentioned below is necessary.
-			// for most setups.
+			// See https://github.com/golang/go/blob/f7c5cbb82087c55aa82081e931e0142783700ce8/src/net/sendfile_linux.go#L20
-			_, err = rf.ReadFrom(sr.Reader)
+			// Windows can lock up with this optimization, so we fall back to regular copy.
-			if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
+			sr, ok := rc.(*sendFileReader)
-				storageLogIf(r.Context(), err)
+			if ok {
-			}
+				// Sendfile sends in 4MiB chunks per sendfile syscall which is more than enough
-			if err == nil || !errors.Is(err, xhttp.ErrNotImplemented) {
+				// for most setups.
-				return
+				_, err = rf.ReadFrom(sr.Reader)
+				if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
+					storageLogIf(r.Context(), err)
+				}
+				if err == nil || !errors.Is(err, xhttp.ErrNotImplemented) {
+					return
+				}
 			}
 		}
-	} // Fallback to regular copy
+	} // noReadFrom means use io.Copy()
 
 	_, err = xioutil.Copy(w, rc)
 	if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients

cmd/xl-storage.go

@@ -2067,7 +2067,6 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 			return nil, err
 		}
 	}
-
 	return &sendFileReader{Reader: io.LimitReader(file, length), Closer: file}, nil
 }
 

internal/ioutil/ioutil.go

@@ -243,6 +243,15 @@ func NopCloser(w io.Writer) io.WriteCloser {
 	return nopCloser{w}
 }
 
+const copyBufferSize = 32 * 1024
+
+var copyBufPool = sync.Pool{
+	New: func() interface{} {
+		b := make([]byte, copyBufferSize)
+		return &b
+	},
+}
+
 // SkipReader skips a given number of bytes and then returns all
 // remaining data.
 type SkipReader struct {
@@ -285,21 +294,11 @@ func NewSkipReader(r io.Reader, n int64) io.Reader {
 	return &SkipReader{r, n}
 }
 
-const copyBufferSize = 32 * 1024
-
-var copyBufPool = sync.Pool{
-	New: func() interface{} {
-		b := make([]byte, copyBufferSize)
-		return &b
-	},
-}
-
 // Copy is exactly like io.Copy but with reusable buffers.
 func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
-	bufp := copyBufPool.Get().(*[]byte)
+	bufp := ODirectPoolLarge.Get().(*[]byte)
 	buf := *bufp
-	buf = buf[:copyBufferSize]
+	defer ODirectPoolLarge.Put(bufp)
-	defer copyBufPool.Put(bufp)
 
 	return io.CopyBuffer(dst, src, buf)
 }