MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-21 10:16:03 -05:00
Code Issues Packages Projects Releases Wiki Activity

re-use io.Copy buffers with 32k pools (#13553)

Browse Source 
Borrowed idea from Go's usage of this
optimization for ReadFrom() on client
side, we should re-use the 32k buffers
io.Copy() allocates for generic copy
from a reader to writer.

the performance increase for reads for
really tiny objects is at this range
after this change.

> * Fastest: +7.89% (+1.3 MiB/s) throughput, +7.89% (+1308.1) obj/s
This commit is contained in:
Harshavardhana
2021-11-02 08:11:50 -07:00
committed by GitHub
parent 30ba85bc67
commit 14d8a931fe
8 changed files with 53 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
cmd/storage-rest-server.go
View File

@@ -41,6 +41,7 @@ import (
"github.com/gorilla/mux"
"github.com/minio/minio/internal/config"
xhttp "github.com/minio/minio/internal/http"
xioutil "github.com/minio/minio/internal/ioutil"
xjwt "github.com/minio/minio/internal/jwt"
"github.com/minio/minio/internal/logger"
xnet "github.com/minio/pkg/net"
@@ -574,7 +575,7 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
defer rc.Close()
w.Header().Set(xhttp.ContentLength, strconv.Itoa(length))
if _, err = io.Copy(w, rc); err != nil {
if _, err = xioutil.Copy(w, rc); err != nil {
if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
logger.LogIf(r.Context(), err)
}
@@ -954,13 +955,22 @@ func streamHTTPResponse(w http.ResponseWriter) *httpStreamResponse {
return &h
}
var poolBuf8k = sync.Pool{
New: func() interface{} {
b := make([]byte, 8192)
return &b
},
}
// waitForHTTPStream will wait for responses where
// streamHTTPResponse has been used.
// The returned reader contains the payload and must be closed if no error is returned.
func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
var tmp [1]byte
// 8K copy buffer, reused for less allocs...
var buf [8 << 10]byte
bufp := poolBuf8k.Get().(*[]byte)
buf := *bufp
defer poolBuf8k.Put(bufp)
for {
_, err := io.ReadFull(respBody, tmp[:])
if err != nil {
@@ -970,7 +980,7 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
switch tmp[0] {
case 0:
// 0 is unbuffered, copy the rest.
_, err := io.CopyBuffer(w, respBody, buf[:])
_, err := io.CopyBuffer(w, respBody, buf)
if err == io.EOF {
return nil
}
@@ -989,7 +999,7 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
return err
}
length := binary.LittleEndian.Uint32(tmp[:])
_, err = io.CopyBuffer(w, io.LimitReader(respBody, int64(length)), buf[:])
_, err = io.CopyBuffer(w, io.LimitReader(respBody, int64(length)), buf)
if err != nil {
return err
}