fix: reduce using memory and temporary files. (#17206)

This commit is contained in:
Harshavardhana
2023-05-15 14:08:54 -07:00
committed by GitHub
parent d063596430
commit ef2fc0f99e
8 changed files with 157 additions and 178 deletions

View File

@@ -39,8 +39,10 @@ import (
// are not empty then it will check whether the computed
// match the reference values.
type Reader struct {
src io.Reader
bytesRead int64
src io.Reader
bytesRead int64
expectedMin int64
expectedMax int64
size int64
actualSize int64
@@ -111,7 +113,7 @@ func newReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualSize i
}
}
if r.size >= 0 && size >= 0 && r.size != size {
return nil, ErrSizeMismatch{Want: r.size, Got: size}
return nil, SizeMismatch{Want: r.size, Got: size}
}
r.checksum = MD5
@@ -171,6 +173,16 @@ func NewLimitReader(src io.Reader, size int64, md5Hex, sha256Hex string, actualS
// ErrInvalidChecksum is returned when an invalid checksum is provided in headers.
var ErrInvalidChecksum = errors.New("invalid checksum")
// SetExpectedMin set expected minimum data expected from reader
func (r *Reader) SetExpectedMin(expectedMin int64) {
r.expectedMin = expectedMin
}
// SetExpectedMax set expected max data expected from reader
func (r *Reader) SetExpectedMax(expectedMax int64) {
r.expectedMax = expectedMax
}
// AddChecksum will add checksum checks as specified in
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
// Returns ErrInvalidChecksum if a problem with the checksum is found.
@@ -209,6 +221,17 @@ func (r *Reader) Read(p []byte) (int, error) {
}
if err == io.EOF { // Verify content SHA256, if set.
if r.expectedMin > 0 {
if r.bytesRead < r.expectedMin {
return 0, SizeTooSmall{Want: r.expectedMin, Got: r.bytesRead}
}
}
if r.expectedMax > 0 {
if r.bytesRead > r.expectedMax {
return 0, SizeTooLarge{Want: r.expectedMax, Got: r.bytesRead}
}
}
if r.sha256 != nil {
if sum := r.sha256.Sum(nil); !bytes.Equal(r.contentSHA256, sum) {
return n, SHA256Mismatch{