mirror of
https://github.com/minio/minio.git
synced 2025-11-09 13:39:46 -05:00
Simplify data verification with HashReader. (#5071)
Verify() was being called by caller after the data has been successfully read after io.EOF. This disconnection opens a race under concurrent access to such an object. Verification is not necessary outside of Read() call, we can simply just do checksum verification right inside Read() call at io.EOF. This approach simplifies the usage.
This commit is contained in:
Harshavardhana
committed by
Nitish Tiwari
Nitish Tiwari
parent
65a817fe8c
commit
1d8a8c63db
@@ -26,6 +26,7 @@ import (
|
||||
"testing"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
func md5Header(data []byte) map[string]string {
|
||||
@@ -33,7 +34,7 @@ func md5Header(data []byte) map[string]string {
|
||||
}
|
||||
|
||||
// Wrapper for calling PutObject tests for both XL multiple disks and single node setup.
|
||||
func TestObjectAPIPutObject(t *testing.T) {
|
||||
func TestObjectAPIPutObjectSingle(t *testing.T) {
|
||||
ExecObjectLayerTest(t, testObjectAPIPutObject)
|
||||
}
|
||||
|
||||
@@ -94,22 +95,25 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
|
||||
|
||||
// Test case - 7.
|
||||
// Input to replicate Md5 mismatch.
|
||||
{bucket, object, []byte(""), map[string]string{"etag": "a35"}, "", 0, "",
|
||||
BadDigest{ExpectedMD5: "a35", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"}},
|
||||
{bucket, object, []byte(""), map[string]string{"etag": "d41d8cd98f00b204e9800998ecf8427f"}, "", 0, "",
|
||||
hash.BadDigest{"d41d8cd98f00b204e9800998ecf8427f", "d41d8cd98f00b204e9800998ecf8427e"}},
|
||||
|
||||
// Test case - 8.
|
||||
// With incorrect sha256.
|
||||
{bucket, object, []byte("abcd"), map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331f"}, "incorrect-sha256", int64(len("abcd")), "", SHA256Mismatch{}},
|
||||
{bucket, object, []byte("abcd"), map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331f"},
|
||||
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580", int64(len("abcd")),
|
||||
"", hash.SHA256Mismatch{"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
|
||||
"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589"}},
|
||||
|
||||
// Test case - 9.
|
||||
// Input with size more than the size of actual data inside the reader.
|
||||
{bucket, object, []byte("abcd"), map[string]string{"etag": "a35"}, "", int64(len("abcd") + 1), "",
|
||||
IncompleteBody{}},
|
||||
{bucket, object, []byte("abcd"), map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331e"}, "", int64(len("abcd") + 1), "",
|
||||
hash.BadDigest{"e2fc714c4727ee9395f324cd2e7f331e", "e2fc714c4727ee9395f324cd2e7f331f"}},
|
||||
|
||||
// Test case - 10.
|
||||
// Input with size less than the size of actual data inside the reader.
|
||||
{bucket, object, []byte("abcd"), map[string]string{"etag": "a35"}, "", int64(len("abcd") - 1), "",
|
||||
BadDigest{ExpectedMD5: "a35", CalculatedMD5: "900150983cd24fb0d6963f7d28e17f72"}},
|
||||
{bucket, object, []byte("abcd"), map[string]string{"etag": "900150983cd24fb0d6963f7d28e17f73"}, "", int64(len("abcd") - 1), "",
|
||||
hash.BadDigest{"900150983cd24fb0d6963f7d28e17f73", "900150983cd24fb0d6963f7d28e17f72"}},
|
||||
|
||||
// Test case - 11-14.
|
||||
// Validating for success cases.
|
||||
@@ -138,9 +142,12 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
|
||||
|
||||
// Test case 24-26.
|
||||
// data with invalid md5sum in header
|
||||
{bucket, object, data, invalidMD5Header, "", int64(len(data)), getMD5Hash(data), BadDigest{invalidMD5, getMD5Hash(data)}},
|
||||
{bucket, object, nilBytes, invalidMD5Header, "", int64(len(nilBytes)), getMD5Hash(nilBytes), BadDigest{invalidMD5, getMD5Hash(nilBytes)}},
|
||||
{bucket, object, fiveMBBytes, invalidMD5Header, "", int64(len(fiveMBBytes)), getMD5Hash(fiveMBBytes), BadDigest{invalidMD5, getMD5Hash(fiveMBBytes)}},
|
||||
{bucket, object, data, invalidMD5Header, "", int64(len(data)), getMD5Hash(data),
|
||||
hash.BadDigest{invalidMD5, getMD5Hash(data)}},
|
||||
{bucket, object, nilBytes, invalidMD5Header, "", int64(len(nilBytes)), getMD5Hash(nilBytes),
|
||||
hash.BadDigest{invalidMD5, getMD5Hash(nilBytes)}},
|
||||
{bucket, object, fiveMBBytes, invalidMD5Header, "", int64(len(fiveMBBytes)), getMD5Hash(fiveMBBytes),
|
||||
hash.BadDigest{invalidMD5, getMD5Hash(fiveMBBytes)}},
|
||||
|
||||
// Test case 27-29.
|
||||
// data with size different from the actual number of bytes available in the reader
|
||||
@@ -154,7 +161,7 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, NewHashReader(bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), testCase.inputMeta)
|
||||
objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), testCase.inputMeta)
|
||||
actualErr = errorCause(actualErr)
|
||||
if actualErr != nil && testCase.expectedError == nil {
|
||||
t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error())
|
||||
@@ -228,7 +235,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di
|
||||
|
||||
sha256sum := ""
|
||||
for i, testCase := range testCases {
|
||||
objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, NewHashReader(bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta)
|
||||
objInfo, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta)
|
||||
actualErr = errorCause(actualErr)
|
||||
if actualErr != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
|
||||
@@ -278,7 +285,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di
|
||||
InsufficientWriteQuorum{},
|
||||
}
|
||||
|
||||
_, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, NewHashReader(bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta)
|
||||
_, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta)
|
||||
actualErr = errorCause(actualErr)
|
||||
if actualErr != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", len(testCases)+1, instanceType, actualErr.Error())
|
||||
@@ -311,7 +318,7 @@ func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disk
|
||||
|
||||
data := []byte("hello, world")
|
||||
// Create object.
|
||||
_, err = obj.PutObject(bucket, object, NewHashReader(bytes.NewReader(data), int64(len(data)), "", ""), nil)
|
||||
_, err = obj.PutObject(bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil)
|
||||
if err != nil {
|
||||
// Failed to create object, abort.
|
||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||
@@ -356,7 +363,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
|
||||
md5Writer.Write(fiveMBBytes)
|
||||
etag1 := hex.EncodeToString(md5Writer.Sum(nil))
|
||||
sha256sum := ""
|
||||
_, err = obj.PutObjectPart(bucket, object, uploadID, 1, NewHashReader(bytes.NewReader(fiveMBBytes), int64(len(fiveMBBytes)), etag1, sha256sum))
|
||||
_, err = obj.PutObjectPart(bucket, object, uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), int64(len(fiveMBBytes)), etag1, sha256sum))
|
||||
if err != nil {
|
||||
// Failed to upload object part, abort.
|
||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||
@@ -367,7 +374,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
|
||||
md5Writer = md5.New()
|
||||
md5Writer.Write(data)
|
||||
etag2 := hex.EncodeToString(md5Writer.Sum(nil))
|
||||
_, err = obj.PutObjectPart(bucket, object, uploadID, 2, NewHashReader(bytes.NewReader(data), int64(len(data)), etag2, sha256sum))
|
||||
_, err = obj.PutObjectPart(bucket, object, uploadID, 2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), etag2, sha256sum))
|
||||
if err != nil {
|
||||
// Failed to upload object part, abort.
|
||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||
|
||||
Reference in New Issue
Block a user