MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 21:49:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

Simplify data verification with HashReader. (#5071)

Browse Source 
Verify() was being called by caller after the data
has been successfully read after io.EOF. This disconnection
opens a race under concurrent access to such an object.
Verification is not necessary outside of Read() call,
we can simply just do checksum verification right inside
Read() call at io.EOF.

This approach simplifies the usage.
This commit is contained in:
Harshavardhana
2017-10-21 22:30:34 -07:00
committed by Nitish Tiwari
parent 65a817fe8c
commit 1d8a8c63db
51 changed files with 749 additions and 499 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/fs-v1.go
View File

@@ -17,7 +17,6 @@
package cmd
import (
"encoding/hex"
"fmt"
"io"
"io/ioutil"
@@ -27,6 +26,7 @@ import (
"sort"
"syscall"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/lock"
)
@@ -361,7 +361,12 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
}()
objInfo, err := fs.PutObject(dstBucket, dstObject, NewHashReader(pipeReader, length, metadata["etag"], ""), metadata)
hashReader, err := hash.NewReader(pipeReader, length, "", "")
if err != nil {
return oi, toObjectErr(err, dstBucket, dstObject)
}
objInfo, err := fs.PutObject(dstBucket, dstObject, hashReader, metadata)
if err != nil {
return oi, toObjectErr(err, dstBucket, dstObject)
}
@@ -540,7 +545,7 @@ func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
// until EOF, writes data directly to configured filesystem path.
// Additionally writes `fs.json` which carries the necessary metadata
// for future object operations.
func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
// No metadata is set, allocate a new one.
if metadata == nil {
metadata = make(map[string]string)
@@ -588,6 +593,8 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
return ObjectInfo{}, traceError(errInvalidArgument)
}
metadata["etag"] = data.MD5HexString()
var wlk *lock.LockedFile
if bucket != minioMetaBucket {
bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
@@ -617,6 +624,7 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
if size := data.Size(); size > 0 && bufSize > size {
bufSize = size
}
buf := make([]byte, int(bufSize))
fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj)
bytesWritten, err := fsCreateFile(fsTmpObjPath, data, buf, data.Size())
@@ -638,12 +646,6 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
// nothing to delete.
defer fsRemoveFile(fsTmpObjPath)
if err = data.Verify(); err != nil { // verify MD5 and SHA256
return ObjectInfo{}, traceError(err)
}
metadata["etag"] = hex.EncodeToString(data.MD5())
// Entire object was written to the temp location, now it's safe to rename it to the actual location.
fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil {