MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-10 05:59:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

Simplify data verification with HashReader. (#5071)

Browse Source 
Verify() was being called by caller after the data
has been successfully read after io.EOF. This disconnection
opens a race under concurrent access to such an object.
Verification is not necessary outside of Read() call,
we can simply just do checksum verification right inside
Read() call at io.EOF.

This approach simplifies the usage.
This commit is contained in:
Harshavardhana
2017-10-21 22:30:34 -07:00
committed by Nitish Tiwari
parent 65a817fe8c
commit 1d8a8c63db
51 changed files with 749 additions and 499 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
cmd/object-api-interface.go
View File

@@ -17,13 +17,9 @@
package cmd
import (
"bytes"
"crypto/md5"
"encoding/hex"
"hash"
"io"
sha256 "github.com/minio/sha256-simd"
"github.com/minio/minio/pkg/hash"
)
// ObjectLayer implements primitives for object API layer.
@@ -42,7 +38,7 @@ type ObjectLayer interface {
// Object operations.
GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
PutObject(bucket, object string, data *HashReader, metadata map[string]string) (objInfo ObjectInfo, err error)
PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error)
CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error)
DeleteObject(bucket, object string) error
@@ -50,7 +46,7 @@ type ObjectLayer interface {
ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)
NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error)
CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error)
PutObjectPart(bucket, object, uploadID string, partID int, data *HashReader) (info PartInfo, err error)
PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error)
ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error)
AbortMultipartUpload(bucket, object, uploadID string) error
CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (objInfo ObjectInfo, err error)
@@ -63,81 +59,3 @@ type ObjectLayer interface {
ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
delimiter string, maxUploads int) (ListMultipartsInfo, error)
}
// HashReader writes what it reads from an io.Reader to an MD5 and SHA256 hash.Hash.
// HashReader verifies that the content of the io.Reader matches the expected checksums.
type HashReader struct {
src io.Reader
size int64
md5Hash, sha256Hash hash.Hash
md5Sum, sha256Sum string // hex representation
}
// NewHashReader returns a new HashReader computing the MD5 sum and SHA256 sum
// (if set) of the provided io.Reader.
func NewHashReader(src io.Reader, size int64, md5Sum, sha256Sum string) *HashReader {
var sha256Hash hash.Hash
if sha256Sum != "" {
sha256Hash = sha256.New()
}
if size >= 0 {
src = io.LimitReader(src, size)
} else {
size = -1
}
return &HashReader{
src: src,
size: size,
md5Sum: md5Sum,
sha256Sum: sha256Sum,
md5Hash: md5.New(),
sha256Hash: sha256Hash,
}
}
func (r *HashReader) Read(p []byte) (n int, err error) {
n, err = r.src.Read(p)
if err != nil && err != io.EOF {
return
}
if r.md5Hash != nil {
r.md5Hash.Write(p[:n])
}
if r.sha256Hash != nil {
r.sha256Hash.Write(p[:n])
}
return
}
// Size returns the absolute number of bytes the HashReader
// will return during reading. It returns -1 for unlimited
// data.
func (r *HashReader) Size() int64 { return r.size }
// MD5 returns the MD5 sum of the processed data. Any
// further reads will change the MD5 sum.
func (r *HashReader) MD5() []byte { return r.md5Hash.Sum(nil) }
// Verify verifies if the computed MD5 sum - and SHA256 sum - are
// equal to the ones specified when creating the HashReader.
func (r *HashReader) Verify() error {
if r.sha256Hash != nil {
sha256Sum, err := hex.DecodeString(r.sha256Sum)
if err != nil {
return SHA256Mismatch{}
}
if !bytes.Equal(sha256Sum, r.sha256Hash.Sum(nil)) {
return errContentSHA256Mismatch
}
}
if r.md5Hash != nil && r.md5Sum != "" {
md5Sum, err := hex.DecodeString(r.md5Sum)
if err != nil {
return BadDigest{r.md5Sum, hex.EncodeToString(r.md5Hash.Sum(nil))}
}
if sum := r.md5Hash.Sum(nil); !bytes.Equal(md5Sum, sum) {
return BadDigest{r.md5Sum, hex.EncodeToString(sum)}
}
}
return nil
}