MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

reduce memory usage in metacache reader (#12334)

This commit is contained in:
Harshavardhana 2021-05-20 09:00:11 -07:00 committed by GitHub
parent a27339826c
commit 32d8a48d4e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 64 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/erasure-utils.go
View File

@ -84,7 +84,9 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
if write < int64(len(block)) { if write < int64(len(block)) {
n, err := io.Copy(dst, bytes.NewReader(block[:write])) n, err := io.Copy(dst, bytes.NewReader(block[:write]))
if err != nil { if err != nil {
if err != io.ErrClosedPipe { // The writer will be closed incase of range queries, which will emit ErrClosedPipe.
// The reader pipe might be closed at ListObjects io.EOF ignore it.
if err != io.ErrClosedPipe && err != io.EOF {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
} }
return 0, err return 0, err
@ -97,7 +99,8 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
n, err := io.Copy(dst, bytes.NewReader(block)) n, err := io.Copy(dst, bytes.NewReader(block))
if err != nil { if err != nil {
// The writer will be closed incase of range queries, which will emit ErrClosedPipe. // The writer will be closed incase of range queries, which will emit ErrClosedPipe.
if err != io.ErrClosedPipe { // The reader pipe might be closed at ListObjects io.EOF ignore it.
if err != io.ErrClosedPipe && err != io.EOF {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
} }
return 0, err return 0, err

52
cmd/metacache-set.go
View File

@ -418,11 +418,6 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
// We got a stream to start at. // We got a stream to start at.
loadedPart := 0 loadedPart := 0
buf := bufferPool.Get().(*bytes.Buffer)
defer func() {
buf.Reset()
bufferPool.Put(buf)
}()
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -471,9 +466,27 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
} }
} }
} }
buf.Reset()
err := er.getObjectWithFileInfo(ctx, minioMetaBucket, o.objectPath(partN), 0, fi.Size, buf, fi, metaArr, onlineDisks) pr, pw := io.Pipe()
if err != nil { go func() {
werr := er.getObjectWithFileInfo(ctx, minioMetaBucket, o.objectPath(partN), 0,
fi.Size, pw, fi, metaArr, onlineDisks)
pw.CloseWithError(werr)
}()
tmp := newMetacacheReader(pr)
e, err := tmp.filter(o)
pr.CloseWithError(err)
entries.o = append(entries.o, e.o...)
if o.Limit > 0 && entries.len() > o.Limit {
entries.truncate(o.Limit)
return entries, nil
}
if err == nil {
// We stopped within the listing, we are done for now...
return entries, nil
}
if err != nil && err.Error() != io.EOF.Error() {
switch toObjectErr(err, minioMetaBucket, o.objectPath(partN)).(type) { switch toObjectErr(err, minioMetaBucket, o.objectPath(partN)).(type) {
case ObjectNotFound: case ObjectNotFound:
retries++ retries++
@ -488,24 +501,6 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
return entries, err return entries, err
} }
} }
tmp, err := newMetacacheReader(buf)
if err != nil {
return entries, err
}
e, err := tmp.filter(o)
entries.o = append(entries.o, e.o...)
if o.Limit > 0 && entries.len() > o.Limit {
entries.truncate(o.Limit)
return entries, nil
}
if err == nil {
// We stopped within the listing, we are done for now...
return entries, nil
}
if !errors.Is(err, io.EOF) {
logger.LogIf(ctx, err)
return entries, err
}
// We finished at the end of the block. // We finished at the end of the block.
// And should not expect any more results. // And should not expect any more results.
@ -824,10 +819,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
// Make sure we close the pipe so blocked writes doesn't stay around. // Make sure we close the pipe so blocked writes doesn't stay around.
defer r.CloseWithError(context.Canceled) defer r.CloseWithError(context.Canceled)
readers[i], err = newMetacacheReader(r) readers[i] = newMetacacheReader(r)
if err != nil {
return err
}
d := disks[i] d := disks[i]
// Send request to each disk. // Send request to each disk.

19
cmd/metacache-stream.go
View File

@ -18,7 +18,6 @@
package cmd package cmd
import ( import (
"bytes"
"context" "context"
"errors" "errors"
"fmt" "fmt"
@ -30,6 +29,7 @@ import (
"github.com/klauspost/compress/s2" "github.com/klauspost/compress/s2"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/tinylib/msgp/msgp" "github.com/tinylib/msgp/msgp"
"github.com/valyala/bytebufferpool"
) )
// metadata stream format: // metadata stream format:
@ -246,11 +246,11 @@ type metacacheReader struct {
// newMetacacheReader creates a new cache reader. // newMetacacheReader creates a new cache reader.
// Nothing will be read from the stream yet. // Nothing will be read from the stream yet.
func newMetacacheReader(r io.Reader) (*metacacheReader, error) { func newMetacacheReader(r io.Reader) *metacacheReader {
dec := s2DecPool.Get().(*s2.Reader) dec := s2DecPool.Get().(*s2.Reader)
dec.Reset(r) dec.Reset(r)
mr := msgp.NewReader(dec) mr := msgp.NewReader(dec)
m := metacacheReader{ return &metacacheReader{
mr: mr, mr: mr,
closer: func() { closer: func() {
dec.Reset(nil) dec.Reset(nil)
@ -269,7 +269,6 @@ func newMetacacheReader(r io.Reader) (*metacacheReader, error) {
return nil return nil
}, },
} }
return &m, nil
} }
func (r *metacacheReader) checkInit() { func (r *metacacheReader) checkInit() {
@ -747,12 +746,6 @@ type metacacheBlockWriter struct {
blockEntries int blockEntries int
} }
var bufferPool = sync.Pool{
New: func() interface{} {
return new(bytes.Buffer)
},
}
// newMetacacheBlockWriter provides a streaming block writer. // newMetacacheBlockWriter provides a streaming block writer.
// Each block is the size of the capacity of the input channel. // Each block is the size of the capacity of the input channel.
// The caller should close to indicate the stream has ended. // The caller should close to indicate the stream has ended.
@ -763,11 +756,13 @@ func newMetacacheBlockWriter(in <-chan metaCacheEntry, nextBlock func(b *metacac
defer w.wg.Done() defer w.wg.Done()
var current metacacheBlock var current metacacheBlock
var n int var n int
buf := bufferPool.Get().(*bytes.Buffer)
buf := bytebufferpool.Get()
defer func() { defer func() {
buf.Reset() buf.Reset()
bufferPool.Put(buf) bytebufferpool.Put(buf)
}() }()
block := newMetacacheWriter(buf, 1<<20) block := newMetacacheWriter(buf, 1<<20)
defer block.Close() defer block.Close()
finishBlock := func() { finishBlock := func() {

11
cmd/metacache-stream_test.go
View File

@ -34,11 +34,7 @@ func loadMetacacheSample(t testing.TB) *metacacheReader {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
r, err := newMetacacheReader(bytes.NewReader(b)) return newMetacacheReader(bytes.NewReader(b))
if err != nil {
t.Fatal(err)
}
return r
} }
func loadMetacacheSampleEntries(t testing.TB) metaCacheEntriesSorted { func loadMetacacheSampleEntries(t testing.TB) metaCacheEntriesSorted {
@ -388,10 +384,7 @@ func Test_newMetacacheStream(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
r, err = newMetacacheReader(&buf) r = newMetacacheReader(&buf)
if err != nil {
t.Fatal(err)
}
defer r.Close() defer r.Close()
names, err := r.readNames(-1) names, err := r.readNames(-1)
if err != io.EOF { if err != io.EOF {

43
cmd/object-api-errors.go
View File

@ -29,30 +29,33 @@ import (
// handle all cases where we have known types of errors returned by // handle all cases where we have known types of errors returned by
// underlying storage layer. // underlying storage layer.
func toObjectErr(err error, params ...string) error { func toObjectErr(err error, params ...string) error {
switch err { if err == nil {
case errVolumeNotFound: return nil
}
switch err.Error() {
case errVolumeNotFound.Error():
apiErr := BucketNotFound{} apiErr := BucketNotFound{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
} }
return apiErr return apiErr
case errVolumeNotEmpty: case errVolumeNotEmpty.Error():
apiErr := BucketNotEmpty{} apiErr := BucketNotEmpty{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
} }
return apiErr return apiErr
case errVolumeExists: case errVolumeExists.Error():
apiErr := BucketExists{} apiErr := BucketExists{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
} }
return apiErr return apiErr
case errDiskFull: case errDiskFull.Error():
return StorageFull{} return StorageFull{}
case errTooManyOpenFiles: case errTooManyOpenFiles.Error():
return SlowDown{} return SlowDown{}
case errFileAccessDenied: case errFileAccessDenied.Error():
apiErr := PrefixAccessDenied{} apiErr := PrefixAccessDenied{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -61,7 +64,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case errFileParentIsFile: case errFileParentIsFile.Error():
apiErr := ParentIsObject{} apiErr := ParentIsObject{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -70,7 +73,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case errIsNotRegular: case errIsNotRegular.Error():
apiErr := ObjectExistsAsDirectory{} apiErr := ObjectExistsAsDirectory{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -79,7 +82,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case errFileVersionNotFound: case errFileVersionNotFound.Error():
apiErr := VersionNotFound{} apiErr := VersionNotFound{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -91,7 +94,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.VersionID = params[2] apiErr.VersionID = params[2]
} }
return apiErr return apiErr
case errMethodNotAllowed: case errMethodNotAllowed.Error():
apiErr := MethodNotAllowed{} apiErr := MethodNotAllowed{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -100,7 +103,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case errFileNotFound: case errFileNotFound.Error():
apiErr := ObjectNotFound{} apiErr := ObjectNotFound{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -109,7 +112,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case errUploadIDNotFound: case errUploadIDNotFound.Error():
apiErr := InvalidUploadID{} apiErr := InvalidUploadID{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -121,7 +124,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.UploadID = params[2] apiErr.UploadID = params[2]
} }
return apiErr return apiErr
case errFileNameTooLong: case errFileNameTooLong.Error():
apiErr := ObjectNameInvalid{} apiErr := ObjectNameInvalid{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -130,7 +133,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case errDataTooLarge: case errDataTooLarge.Error():
apiErr := ObjectTooLarge{} apiErr := ObjectTooLarge{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -139,7 +142,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case errDataTooSmall: case errDataTooSmall.Error():
apiErr := ObjectTooSmall{} apiErr := ObjectTooSmall{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -148,7 +151,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case errErasureReadQuorum: case errErasureReadQuorum.Error():
apiErr := InsufficientReadQuorum{} apiErr := InsufficientReadQuorum{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -157,7 +160,7 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case errErasureWriteQuorum: case errErasureWriteQuorum.Error():
apiErr := InsufficientWriteQuorum{} apiErr := InsufficientWriteQuorum{}
if len(params) >= 1 { if len(params) >= 1 {
apiErr.Bucket = params[0] apiErr.Bucket = params[0]
@ -166,9 +169,9 @@ func toObjectErr(err error, params ...string) error {
apiErr.Object = decodeDirObject(params[1]) apiErr.Object = decodeDirObject(params[1])
} }
return apiErr return apiErr
case io.ErrUnexpectedEOF, io.ErrShortWrite: case io.ErrUnexpectedEOF.Error(), io.ErrShortWrite.Error():
return IncompleteBody{} return IncompleteBody{}
case context.Canceled, context.DeadlineExceeded: case context.Canceled.Error(), context.DeadlineExceeded.Error():
return IncompleteBody{} return IncompleteBody{}
} }
return err return err

2
go.mod
View File

@ -78,7 +78,7 @@ require (
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/streadway/amqp v1.0.0 github.com/streadway/amqp v1.0.0
github.com/tinylib/msgp v1.1.3 github.com/tinylib/msgp v1.1.3
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 // indirect github.com/valyala/bytebufferpool v1.0.0
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
github.com/willf/bloom v2.0.3+incompatible github.com/willf/bloom v2.0.3+incompatible
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c

11
go.sum
View File

@ -694,11 +694,11 @@ github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZ
github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8= github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4=
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
@ -797,8 +797,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -859,8 +857,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -920,7 +918,6 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -993,8 +990,6 @@ golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.1-0.20210201201750-4d4ee958a9b7 h1:/wdPW261t381NDQd8TBo63/FyvACfLICwtH8wMRoHJQ=
golang.org/x/tools v0.1.1-0.20210201201750-4d4ee958a9b7/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

38
pkg/ioutil/read_file.go
View File

@ -15,11 +15,6 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
// Forked from golang.org/pkg/os.ReadFile with NOATIME support.
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the https://golang.org/LICENSE file.
package ioutil package ioutil
import ( import (
@ -40,36 +35,5 @@ func ReadFile(name string) ([]byte, error) {
} }
defer f.Close() defer f.Close()
var size int return io.ReadAll(f)
if info, err := f.Stat(); err == nil {
size64 := info.Size()
if int64(int(size64)) == size64 {
size = int(size64)
}
}
size++ // one byte for final read at EOF
// If a file claims a small size, read at least 512 bytes.
// In particular, files in Linux's /proc claim size 0 but
// then do not work right if read in small pieces,
// so an initial read of 1 byte would not work correctly.
if size < 512 {
size = 512
}
data := make([]byte, 0, size)
for {
if len(data) >= cap(data) {
d := append(data[:cap(data)], 0)
data = d[:len(data)]
}
n, err := f.Read(data[len(data):cap(data)])
data = data[:len(data)+n]
if err != nil {
if err == io.EOF {
err = nil
}
return data, err
}
}
} }

5
pkg/ioutil/wait_pipe.go
View File

@ -15,11 +15,6 @@
// You should have received a copy of the GNU Affero General Public License // You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
// Forked from golang.org/pkg/os.ReadFile with NOATIME support.
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the https://golang.org/LICENSE file.
package ioutil package ioutil
import ( import (