MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support CRC32 Checksums on single drive setup (#15873)

This commit is contained in:
Harshavardhana 2022-10-15 11:58:47 -07:00 committed by GitHub
parent c68910005b
commit 328d660106
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 106 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/erasure-multipart.go
View File

@ -1103,7 +1103,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
PartNumber: part.PartNumber,
}
}
checksumCombined = append(checksumCombined, cs.Raw()...)
checksumCombined = append(checksumCombined, cs.Raw...)
}
// All parts except the last part has to be at least 5MB.

78
cmd/erasure-single-drive.go
View File

@ -1017,6 +1017,11 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
}
fi.DataDir = mustGetUUID()
fi.Checksum = opts.WantChecksum.AppendTo(nil)
if opts.EncryptFn != nil {
fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
}
uniqueID := mustGetUUID()
tempObj := uniqueID
@ -2593,6 +2598,7 @@ func (es *erasureSingle) ListObjectParts(ctx context.Context, bucket, object, up
result.MaxParts = maxParts
result.PartNumberMarker = partNumberMarker
result.UserDefined = cloneMSS(fi.Metadata)
result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum]
// For empty number of parts or maxParts as zero, return right here.
if len(fi.Parts) == 0 || maxParts == 0 {
@ -2613,10 +2619,15 @@ func (es *erasureSingle) ListObjectParts(ctx context.Context, bucket, object, up
count := maxParts
for _, part := range parts {
result.Parts = append(result.Parts, PartInfo{
PartNumber: part.Number,
ETag: part.ETag,
LastModified: fi.ModTime,
Size: part.Size,
PartNumber: part.Number,
ETag: part.ETag,
LastModified: fi.ModTime,
ActualSize: part.ActualSize,
Size: part.Size,
ChecksumCRC32: part.Checksums["CRC32"],
ChecksumCRC32C: part.Checksums["CRC32C"],
ChecksumSHA1: part.Checksums["SHA1"],
ChecksumSHA256: part.Checksums["SHA256"],
})
count--
if count == 0 {
@ -2685,6 +2696,21 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
return oi, err
}
// Checksum type set when upload started.
var checksumType hash.ChecksumType
if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
checksumType = hash.NewChecksumType(cs)
if opts.WantChecksum != nil && !opts.WantChecksum.Type.Is(checksumType) {
return oi, InvalidArgument{
Bucket: bucket,
Object: fi.Name,
Err: fmt.Errorf("checksum type mismatch"),
}
}
}
var checksumCombined []byte
// However, in case of encryption, the persisted part ETags don't match
// what we have sent to the client during PutObjectPart. The reason is
// that ETags are encrypted. Hence, the client will send a list of complete
@ -2731,6 +2757,7 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
}
return oi, invp
}
expPart := currentFI.Parts[partIdx]
// ensure that part ETag is canonicalized to strip off extraneous quotes
part.ETag = canonicalizeETag(part.ETag)
@ -2744,27 +2771,58 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
return oi, invp
}
if checksumType.IsSet() {
crc := expPart.Checksums[checksumType.String()]
if crc == "" {
return oi, InvalidPart{
PartNumber: part.PartNumber,
}
}
wantCS := map[string]string{
hash.ChecksumCRC32.String(): part.ChecksumCRC32,
hash.ChecksumCRC32C.String(): part.ChecksumCRC32C,
hash.ChecksumSHA1.String(): part.ChecksumSHA1,
hash.ChecksumSHA256.String(): part.ChecksumSHA256,
}
if wantCS[checksumType.String()] != crc {
return oi, InvalidPart{
PartNumber: part.PartNumber,
ExpETag: wantCS[checksumType.String()],
GotETag: crc,
}
}
cs := hash.NewChecksumString(checksumType.String(), crc)
if !cs.Valid() {
return oi, InvalidPart{
PartNumber: part.PartNumber,
}
}
checksumCombined = append(checksumCombined, cs.Raw...)
}
// All parts except the last part has to be atleast 5MB.
if (i < len(parts)-1) && !isMinAllowedPartSize(currentFI.Parts[partIdx].ActualSize) {
return oi, PartTooSmall{
PartNumber: part.PartNumber,
PartSize: currentFI.Parts[partIdx].ActualSize,
PartSize: expPart.ActualSize,
PartETag: part.ETag,
}
}
// Save for total object size.
objectSize += currentFI.Parts[partIdx].Size
objectSize += expPart.Size
// Save the consolidated actual size.
objectActualSize += currentFI.Parts[partIdx].ActualSize
objectActualSize += expPart.ActualSize
// Add incoming parts.
fi.Parts[i] = ObjectPartInfo{
Number: part.PartNumber,
Size: currentFI.Parts[partIdx].Size,
ActualSize: currentFI.Parts[partIdx].ActualSize,
Index: currentFI.Parts[partIdx].Index,
Size: expPart.Size,
ActualSize: expPart.ActualSize,
ModTime: expPart.ModTime,
Index: expPart.Index,
Checksums: nil, // Not transferred since we do not need it.
}
}

6
cmd/object-handlers.go
View File

@ -1797,7 +1797,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err = actualReader.AddChecksum(r, false); err != nil {
if err = actualReader.AddChecksum(r, globalIsGateway); err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
@ -1818,7 +1818,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err := hashReader.AddChecksum(r, size < 0); err != nil {
if err := hashReader.AddChecksum(r, size < 0 || globalIsGateway); err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
@ -2129,7 +2129,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err = hreader.AddChecksum(r, false); err != nil {
if err = hreader.AddChecksum(r, globalIsGateway); err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}

4
cmd/object-multipart-handlers.go
View File

@ -390,7 +390,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err = actualReader.AddChecksum(r, false); err != nil {
if err = actualReader.AddChecksum(r, globalIsGateway); err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
@ -411,7 +411,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err := hashReader.AddChecksum(r, size < 0); err != nil {
if err := hashReader.AddChecksum(r, size < 0 || globalIsGateway); err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}

60
internal/hash/checksum.go
View File

@ -18,9 +18,11 @@
package hash
import (
"bytes"
"crypto/sha1"
"encoding/base64"
"encoding/binary"
"fmt"
"hash"
"hash/crc32"
"net/http"
@ -61,6 +63,7 @@ const (
type Checksum struct {
Type ChecksumType
Encoded string
Raw []byte
}
// Is returns if c is all of t.
@ -169,7 +172,8 @@ func NewChecksumFromData(t ChecksumType, data []byte) *Checksum {
}
h := t.Hasher()
h.Write(data)
c := Checksum{Type: t, Encoded: base64.StdEncoding.EncodeToString(h.Sum(nil))}
raw := h.Sum(nil)
c := Checksum{Type: t, Encoded: base64.StdEncoding.EncodeToString(raw), Raw: raw}
if !c.Valid() {
return nil
}
@ -200,19 +204,27 @@ func ReadCheckSums(b []byte) map[string]string {
return res
}
// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value.
func NewChecksumString(alg, value string) *Checksum {
t := NewChecksumType(alg)
if !t.IsSet() {
// NewChecksumWithType is similar to NewChecksumString but expects input algo of ChecksumType.
func NewChecksumWithType(alg ChecksumType, value string) *Checksum {
if !alg.IsSet() {
return nil
}
c := Checksum{Type: t, Encoded: value}
bvalue, err := base64.StdEncoding.DecodeString(value)
if err != nil {
return nil
}
c := Checksum{Type: alg, Encoded: value, Raw: bvalue}
if !c.Valid() {
return nil
}
return &c
}
// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value.
func NewChecksumString(alg, value string) *Checksum {
return NewChecksumWithType(NewChecksumType(alg), value)
}
// AppendTo will append the checksum to b.
// ReadCheckSums reads the values back.
func (c *Checksum) AppendTo(b []byte) []byte {
@ -221,7 +233,7 @@ func (c *Checksum) AppendTo(b []byte) []byte {
}
var tmp [binary.MaxVarintLen32]byte
n := binary.PutUvarint(tmp[:], uint64(c.Type))
crc := c.Raw()
crc := c.Raw
if len(crc) != c.Type.RawByteLen() {
return b
}
@ -238,19 +250,10 @@ func (c Checksum) Valid() bool {
if len(c.Encoded) == 0 || c.Type.Is(ChecksumTrailing) {
return c.Type.Is(ChecksumNone) || c.Type.Is(ChecksumTrailing)
}
raw := c.Raw()
raw := c.Raw
return c.Type.RawByteLen() == len(raw)
}
// Raw returns the Raw checksum.
func (c Checksum) Raw() []byte {
if len(c.Encoded) == 0 {
return nil
}
v, _ := base64.StdEncoding.DecodeString(c.Encoded)
return v
}
// Matches returns whether given content matches c.
func (c Checksum) Matches(content []byte) error {
if len(c.Encoded) == 0 {
@ -261,11 +264,11 @@ func (c Checksum) Matches(content []byte) error {
if err != nil {
return err
}
got := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
if got != c.Encoded {
sum := hasher.Sum(nil)
if !bytes.Equal(sum, c.Raw) {
return ChecksumMismatch{
Want: c.Encoded,
Got: got,
Got: base64.StdEncoding.EncodeToString(sum),
}
}
return nil
@ -292,13 +295,13 @@ func TransferChecksumHeader(w http.ResponseWriter, r *http.Request) {
// AddChecksumHeader will transfer any checksum value that has been checked.
func AddChecksumHeader(w http.ResponseWriter, c map[string]string) {
for k, v := range c {
typ := NewChecksumType(k)
if !typ.IsSet() {
fmt.Println(c, v)
cksum := NewChecksumString(k, v)
if cksum == nil {
continue
}
crc := Checksum{Type: typ, Encoded: v}
if crc.Valid() {
w.Header().Set(typ.Key(), v)
if cksum.Valid() {
w.Header().Set(cksum.Type.Key(), v)
}
}
}
@ -314,12 +317,11 @@ func GetContentChecksum(r *http.Request) (*Checksum, error) {
}
return nil, ErrInvalidChecksum
}
c := Checksum{Type: t, Encoded: s}
if !c.Valid() {
cksum := NewChecksumWithType(t, s)
if cksum == nil {
return nil, ErrInvalidChecksum
}
return &c, nil
return cksum, nil
}
// getContentChecksum returns content checksum type and value.

2
internal/hash/reader.go
View File

@ -186,7 +186,7 @@ func (r *Reader) Read(p []byte) (int, error) {
}
}
if r.contentHasher != nil {
if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw(), sum) {
if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw, sum) {
err := ChecksumMismatch{
Want: r.contentHash.Encoded,
Got: base64.StdEncoding.EncodeToString(sum),