MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-27 14:43:18 -05:00
Code Issues Packages Projects Releases Wiki Activity

s3: Provide enough buffer when the object final size is unknown (#20847)

Browse Source 
When compression is enabled, the final object size is not calculated in
that case, we need to make sure that the provided buffer is always
more significant than the shard size, the bitrot will always calculate 
the hash of blocks with shard size, except the last block.
This commit is contained in:
Anis Eleuch 2025-01-17 20:19:30 +01:00 committed by GitHub
parent b9196757fd
commit dbcb71828d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 14 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/bitrot-streaming.go
View File

@ -20,6 +20,7 @@ package cmd
import (
"bytes"
"context"
"errors"
"hash"
"io"
"sync"
@ -37,12 +38,22 @@ type streamingBitrotWriter struct {
shardSize int64
canClose *sync.WaitGroup
byteBuf []byte
finished bool
}
func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
if b.finished {
return 0, errors.New("bitrot write not allowed")
}
if int64(len(p)) > b.shardSize {
return 0, errors.New("unexpected bitrot buffer size")
}
if int64(len(p)) < b.shardSize {
b.finished = true
}
b.h.Reset()
b.h.Write(p)
hashBytes := b.h.Sum(nil)

10
cmd/erasure-multipart.go
View File

@ -626,17 +626,13 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
switch size := data.Size(); {
case size == 0:
buffer = make([]byte, 1) // Allocate at least a byte to reach EOF
case size == -1:
if size := data.ActualSize(); size > 0 && size < fi.Erasure.BlockSize {
// Account for padding and forced compression overhead and encryption.
buffer = make([]byte, data.ActualSize()+256+32+32, data.ActualSize()*2+512)
case size >= fi.Erasure.BlockSize || size == -1:
if int64(globalBytePoolCap.Load().Width()) < fi.Erasure.BlockSize {
buffer = make([]byte, fi.Erasure.BlockSize, 2*fi.Erasure.BlockSize)
} else {
buffer = globalBytePoolCap.Load().Get()
defer globalBytePoolCap.Load().Put(buffer)
}
case size >= fi.Erasure.BlockSize:
buffer = globalBytePoolCap.Load().Get()
defer globalBytePoolCap.Load().Put(buffer)
case size < fi.Erasure.BlockSize:
// No need to allocate fully fi.Erasure.BlockSize buffer if the incoming data is smaller.
buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))