mirror of
https://github.com/minio/minio.git
synced 2025-11-07 12:52:58 -05:00
fix: use buffers only when necessary for io.Copy() (#11229)
Use separate sync.Pool for writes/reads
Avoid passing buffers for io.CopyBuffer()
if the writer or reader implement io.WriteTo or io.ReadFrom
respectively then its useless for sync.Pool to allocate
buffers on its own since that will be completely ignored
by the io.CopyBuffer Go implementation.
Improve this wherever we see this to be optimal.
This allows us to be more efficient on memory usage.
```
385 // copyBuffer is the actual implementation of Copy and CopyBuffer.
386 // if buf is nil, one is allocated.
387 func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) {
388 // If the reader has a WriteTo method, use it to do the copy.
389 // Avoids an allocation and a copy.
390 if wt, ok := src.(WriterTo); ok {
391 return wt.WriteTo(dst)
392 }
393 // Similarly, if the writer has a ReadFrom method, use it to do the copy.
394 if rt, ok := dst.(ReaderFrom); ok {
395 return rt.ReadFrom(src)
396 }
```
From readahead package
```
// WriteTo writes data to w until there's no more data to write or when an error occurs.
// The return value n is the number of bytes written.
// Any error encountered during the write is also returned.
func (a *reader) WriteTo(w io.Writer) (n int64, err error) {
if a.err != nil {
return 0, a.err
}
n = 0
for {
err = a.fill()
if err != nil {
return n, err
}
n2, err := w.Write(a.cur.buffer())
a.cur.inc(n2)
n += int64(n2)
if err != nil {
return n, err
}
```
This commit is contained in:
@@ -703,8 +703,8 @@ func TestXLStorageListVols(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestXLStorageXlStorageListDir - TestXLStorages validate the directory listing functionality provided by xlStorage.ListDir .
|
||||
func TestXLStorageXlStorageListDir(t *testing.T) {
|
||||
// TestXLStorageListDir - TestXLStorages validate the directory listing functionality provided by xlStorage.ListDir .
|
||||
func TestXLStorageListDir(t *testing.T) {
|
||||
// create xlStorage test setup
|
||||
xlStorage, path, err := newXLStorageTestSetup()
|
||||
if err != nil {
|
||||
@@ -1692,8 +1692,9 @@ func TestXLStorageVerifyFile(t *testing.T) {
|
||||
w := newStreamingBitrotWriter(xlStorage, volName, fileName, size, algo, shardSize)
|
||||
reader := bytes.NewReader(data)
|
||||
for {
|
||||
// Using io.CopyBuffer instead of this loop will not work for us as io.CopyBuffer
|
||||
// will use bytes.Buffer.ReadConfig() which will not do shardSize'ed writes causing error.
|
||||
// Using io.Copy instead of this loop will not work for us as io.Copy
|
||||
// will use bytes.Reader.WriteTo() which will not do shardSize'ed writes
|
||||
// causing error.
|
||||
n, err := reader.Read(shard)
|
||||
w.Write(shard[:n])
|
||||
if err == nil {
|
||||
|
||||
Reference in New Issue
Block a user