diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go index 3f7f1949a..260cf13b5 100644 --- a/cmd/fs-v1-metadata.go +++ b/cmd/fs-v1-metadata.go @@ -27,6 +27,7 @@ import ( "github.com/minio/minio/pkg/lock" "github.com/minio/minio/pkg/mimedb" + "github.com/tidwall/gjson" ) const ( @@ -66,7 +67,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo Name: object, } - // We set file into only if its valid. + // We set file info only if its valid. objInfo.ModTime = timeSentinel if fi != nil { objInfo.ModTime = fi.ModTime() @@ -144,29 +145,76 @@ func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) { return int64(len(metadataBytes)), nil } +func parseFSVersion(fsMetaBuf []byte) string { + return gjson.GetBytes(fsMetaBuf, "version").String() +} + +func parseFSFormat(fsMetaBuf []byte) string { + return gjson.GetBytes(fsMetaBuf, "format").String() +} + +func parseFSRelease(fsMetaBuf []byte) string { + return gjson.GetBytes(fsMetaBuf, "minio.release").String() +} + +func parseFSMetaMap(fsMetaBuf []byte) map[string]string { + // Get xlMetaV1.Meta map. + metaMapResult := gjson.GetBytes(fsMetaBuf, "meta").Map() + metaMap := make(map[string]string) + for key, valResult := range metaMapResult { + metaMap[key] = valResult.String() + } + return metaMap +} + +func parseFSParts(fsMetaBuf []byte) []objectPartInfo { + // Parse the FS Parts. + partsResult := gjson.GetBytes(fsMetaBuf, "parts").Array() + partInfo := make([]objectPartInfo, len(partsResult)) + for i, p := range partsResult { + info := objectPartInfo{} + info.Number = int(p.Get("number").Int()) + info.Name = p.Get("name").String() + info.ETag = p.Get("etag").String() + info.Size = p.Get("size").Int() + partInfo[i] = info + } + return partInfo +} + func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) { - var metadataBytes []byte + var fsMetaBuf []byte fi, err := lk.Stat() if err != nil { return 0, traceError(err) } - metadataBytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size())) + fsMetaBuf, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size())) if err != nil { return 0, traceError(err) } - if len(metadataBytes) == 0 { + if len(fsMetaBuf) == 0 { return 0, traceError(io.EOF) } - // Decode `fs.json` into fsMeta structure. - if err = json.Unmarshal(metadataBytes, m); err != nil { - return 0, traceError(err) - } + // obtain version. + m.Version = parseFSVersion(fsMetaBuf) + + // obtain format. + m.Format = parseFSFormat(fsMetaBuf) + + // obtain metadata. + m.Meta = parseFSMetaMap(fsMetaBuf) + + // obtain parts info list. + m.Parts = parseFSParts(fsMetaBuf) + + // obtain minio release date. + m.Minio.Release = parseFSRelease(fsMetaBuf) // Success. - return int64(len(metadataBytes)), nil + return int64(len(fsMetaBuf)), nil } // FS metadata constants. diff --git a/cmd/fs-v1-metadata_test.go b/cmd/fs-v1-metadata_test.go index c31008949..79e3f35d0 100644 --- a/cmd/fs-v1-metadata_test.go +++ b/cmd/fs-v1-metadata_test.go @@ -18,7 +18,6 @@ package cmd import ( "bytes" - "os" "path/filepath" "testing" ) @@ -73,18 +72,6 @@ func TestReadFSMetadata(t *testing.T) { if _, err = fsMeta.ReadFrom(rlk.LockedFile); err != nil { t.Fatal("Unexpected error ", err) } - - // Corrupted fs.json - file, err := os.OpenFile(preparePath(fsPath), os.O_APPEND|os.O_WRONLY, 0666) - if err != nil { - t.Fatal("Unexpected error ", err) - } - file.Write([]byte{'a'}) - file.Close() - fsMeta = fsMetaV1{} - if _, err := fsMeta.ReadFrom(rlk.LockedFile); err == nil { - t.Fatal("Should fail", err) - } } // TestWriteFSMetadata - tests of writeFSMetadata with healthy disk. diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 99a39b1e5..8a6a3430e 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -22,6 +22,7 @@ import ( "fmt" "hash" "io" + "io/ioutil" "os" "path/filepath" "sort" @@ -687,6 +688,41 @@ func (fs fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc { return listDir } +// getObjectETag is a helper function, which returns only the md5sum +// of the file on the disk. +func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) { + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, entry, fsMetaJSONFile) + + // Read `fs.json` to perhaps contend with + // parallel Put() operations. + rlk, err := fs.rwPool.Open(fsMetaPath) + // Ignore if `fs.json` is not available, this is true for pre-existing data. + if err != nil && err != errFileNotFound { + return "", toObjectErr(traceError(err), bucket, entry) + } + + // If file is not found, we don't need to proceed forward. + if err == errFileNotFound { + return "", nil + } + + // Read from fs metadata only if it exists. + defer fs.rwPool.Close(fsMetaPath) + + fsMetaBuf, err := ioutil.ReadAll(rlk.LockedFile) + if err != nil { + // `fs.json` can be empty due to previously failed + // PutObject() transaction, if we arrive at such + // a situation we just ignore and continue. + if errorCause(err) != io.EOF { + return "", toObjectErr(err, bucket, entry) + } + } + + fsMetaMap := parseFSMetaMap(fsMetaBuf) + return fsMetaMap["md5Sum"], nil +} + // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool // state for future re-entrant list requests. func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { @@ -731,14 +767,33 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey objInfo.IsDir = true return } + + // Protect reading `fs.json`. + objectLock := globalNSMutex.NewNSLock(bucket, entry) + objectLock.RLock() + var md5Sum string + md5Sum, err = fs.getObjectETag(bucket, entry) + objectLock.RUnlock() + if err != nil { + return ObjectInfo{}, err + } + // Stat the file to get file size. var fi os.FileInfo fi, err = fsStatFile(pathJoin(fs.fsPath, bucket, entry)) if err != nil { return ObjectInfo{}, toObjectErr(err, bucket, entry) } - fsMeta := fsMetaV1{} - return fsMeta.ToObjectInfo(bucket, entry, fi), nil + + // Success. + return ObjectInfo{ + Name: entry, + Bucket: bucket, + Size: fi.Size(), + ModTime: fi.ModTime(), + IsDir: fi.IsDir(), + MD5Sum: md5Sum, + }, nil } heal := false // true only for xl.ListObjectsHeal() diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go index 2b25441a0..f57b9503a 100644 --- a/cmd/object-api-listobjects_test.go +++ b/cmd/object-api-listobjects_test.go @@ -81,7 +81,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { { IsTruncated: false, Objects: []ObjectInfo{ - {Name: "Asia-maps.png", ContentType: "image/png"}, + {Name: "Asia-maps.png"}, {Name: "Asia/India/India-summer-photos-1"}, {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, {Name: "newPrefix0"}, @@ -97,7 +97,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { { IsTruncated: true, Objects: []ObjectInfo{ - {Name: "Asia-maps.png", ContentType: "image/png"}, + {Name: "Asia-maps.png"}, {Name: "Asia/India/India-summer-photos-1"}, {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, {Name: "newPrefix0"}, @@ -109,7 +109,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { { IsTruncated: true, Objects: []ObjectInfo{ - {Name: "Asia-maps.png", ContentType: "image/png"}, + {Name: "Asia-maps.png"}, {Name: "Asia/India/India-summer-photos-1"}, {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, {Name: "newPrefix0"}, @@ -120,7 +120,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { { IsTruncated: true, Objects: []ObjectInfo{ - {Name: "Asia-maps.png", ContentType: "image/png"}, + {Name: "Asia-maps.png"}, {Name: "Asia/India/India-summer-photos-1"}, {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, }, @@ -131,7 +131,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { { IsTruncated: true, Objects: []ObjectInfo{ - {Name: "Asia-maps.png", ContentType: "image/png"}, + {Name: "Asia-maps.png"}, }, }, // ListObjectsResult-5. @@ -234,7 +234,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { { IsTruncated: false, Objects: []ObjectInfo{ - {Name: "Asia-maps.png", ContentType: "image/png"}, + {Name: "Asia-maps.png"}, {Name: "Asia/India/India-summer-photos-1"}, {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, {Name: "newPrefix0"}, @@ -343,7 +343,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { { IsTruncated: false, Objects: []ObjectInfo{ - {Name: "Asia-maps.png", ContentType: "image/png"}, + {Name: "Asia-maps.png"}, {Name: "Asia/India/India-summer-photos-1"}, {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, }, @@ -354,7 +354,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { { IsTruncated: false, Objects: []ObjectInfo{ - {Name: "Asia-maps.png", ContentType: "image/png"}, + {Name: "Asia-maps.png"}, }, }, // ListObjectsResult-26. @@ -549,8 +549,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { if testCase.result.Objects[j].Name != result.Objects[j].Name { t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name) } - if testCase.result.Objects[j].ContentType != result.Objects[j].ContentType { - t.Errorf("Test %d: %s: Expected object contentType to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].ContentType, result.Objects[j].ContentType) + if result.Objects[j].MD5Sum == "" { + t.Errorf("Test %d: %s: Expected md5sum to be not empty, but found empty instead", i+1, instanceType) } } diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index d61d47991..46b1fab17 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -67,6 +67,9 @@ func init() { // Set system resources to maximum. setMaxResources() + + // Quiet logging. + log.logger.Hooks = nil } func prepareFS() (ObjectLayer, string, error) {