mirror of
https://github.com/minio/minio.git
synced 2024-12-24 22:25:54 -05:00
fs: Enable returning ETag along with ListObjects() (#4042)
This is to comply with S3 behavior, we previously removed reading `fs.json` for optimization reasons but we have a reason to believe that providing ETag and using gjson provides needed benefit of not having to deal with unmarshalling overhead of golang stdlib. Fixes #4028
This commit is contained in:
parent
52d8f564bf
commit
4747adfcb4
@ -27,6 +27,7 @@ import (
|
||||
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
"github.com/minio/minio/pkg/mimedb"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -66,7 +67,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
|
||||
Name: object,
|
||||
}
|
||||
|
||||
// We set file into only if its valid.
|
||||
// We set file info only if its valid.
|
||||
objInfo.ModTime = timeSentinel
|
||||
if fi != nil {
|
||||
objInfo.ModTime = fi.ModTime()
|
||||
@ -144,29 +145,76 @@ func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
|
||||
return int64(len(metadataBytes)), nil
|
||||
}
|
||||
|
||||
func parseFSVersion(fsMetaBuf []byte) string {
|
||||
return gjson.GetBytes(fsMetaBuf, "version").String()
|
||||
}
|
||||
|
||||
func parseFSFormat(fsMetaBuf []byte) string {
|
||||
return gjson.GetBytes(fsMetaBuf, "format").String()
|
||||
}
|
||||
|
||||
func parseFSRelease(fsMetaBuf []byte) string {
|
||||
return gjson.GetBytes(fsMetaBuf, "minio.release").String()
|
||||
}
|
||||
|
||||
func parseFSMetaMap(fsMetaBuf []byte) map[string]string {
|
||||
// Get xlMetaV1.Meta map.
|
||||
metaMapResult := gjson.GetBytes(fsMetaBuf, "meta").Map()
|
||||
metaMap := make(map[string]string)
|
||||
for key, valResult := range metaMapResult {
|
||||
metaMap[key] = valResult.String()
|
||||
}
|
||||
return metaMap
|
||||
}
|
||||
|
||||
func parseFSParts(fsMetaBuf []byte) []objectPartInfo {
|
||||
// Parse the FS Parts.
|
||||
partsResult := gjson.GetBytes(fsMetaBuf, "parts").Array()
|
||||
partInfo := make([]objectPartInfo, len(partsResult))
|
||||
for i, p := range partsResult {
|
||||
info := objectPartInfo{}
|
||||
info.Number = int(p.Get("number").Int())
|
||||
info.Name = p.Get("name").String()
|
||||
info.ETag = p.Get("etag").String()
|
||||
info.Size = p.Get("size").Int()
|
||||
partInfo[i] = info
|
||||
}
|
||||
return partInfo
|
||||
}
|
||||
|
||||
func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
|
||||
var metadataBytes []byte
|
||||
var fsMetaBuf []byte
|
||||
fi, err := lk.Stat()
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
|
||||
metadataBytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
|
||||
fsMetaBuf, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
|
||||
if len(metadataBytes) == 0 {
|
||||
if len(fsMetaBuf) == 0 {
|
||||
return 0, traceError(io.EOF)
|
||||
}
|
||||
|
||||
// Decode `fs.json` into fsMeta structure.
|
||||
if err = json.Unmarshal(metadataBytes, m); err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
// obtain version.
|
||||
m.Version = parseFSVersion(fsMetaBuf)
|
||||
|
||||
// obtain format.
|
||||
m.Format = parseFSFormat(fsMetaBuf)
|
||||
|
||||
// obtain metadata.
|
||||
m.Meta = parseFSMetaMap(fsMetaBuf)
|
||||
|
||||
// obtain parts info list.
|
||||
m.Parts = parseFSParts(fsMetaBuf)
|
||||
|
||||
// obtain minio release date.
|
||||
m.Minio.Release = parseFSRelease(fsMetaBuf)
|
||||
|
||||
// Success.
|
||||
return int64(len(metadataBytes)), nil
|
||||
return int64(len(fsMetaBuf)), nil
|
||||
}
|
||||
|
||||
// FS metadata constants.
|
||||
|
@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
@ -73,18 +72,6 @@ func TestReadFSMetadata(t *testing.T) {
|
||||
if _, err = fsMeta.ReadFrom(rlk.LockedFile); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
// Corrupted fs.json
|
||||
file, err := os.OpenFile(preparePath(fsPath), os.O_APPEND|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
file.Write([]byte{'a'})
|
||||
file.Close()
|
||||
fsMeta = fsMetaV1{}
|
||||
if _, err := fsMeta.ReadFrom(rlk.LockedFile); err == nil {
|
||||
t.Fatal("Should fail", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteFSMetadata - tests of writeFSMetadata with healthy disk.
|
||||
|
59
cmd/fs-v1.go
59
cmd/fs-v1.go
@ -22,6 +22,7 @@ import (
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@ -687,6 +688,41 @@ func (fs fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
|
||||
return listDir
|
||||
}
|
||||
|
||||
// getObjectETag is a helper function, which returns only the md5sum
|
||||
// of the file on the disk.
|
||||
func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
|
||||
fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, entry, fsMetaJSONFile)
|
||||
|
||||
// Read `fs.json` to perhaps contend with
|
||||
// parallel Put() operations.
|
||||
rlk, err := fs.rwPool.Open(fsMetaPath)
|
||||
// Ignore if `fs.json` is not available, this is true for pre-existing data.
|
||||
if err != nil && err != errFileNotFound {
|
||||
return "", toObjectErr(traceError(err), bucket, entry)
|
||||
}
|
||||
|
||||
// If file is not found, we don't need to proceed forward.
|
||||
if err == errFileNotFound {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Read from fs metadata only if it exists.
|
||||
defer fs.rwPool.Close(fsMetaPath)
|
||||
|
||||
fsMetaBuf, err := ioutil.ReadAll(rlk.LockedFile)
|
||||
if err != nil {
|
||||
// `fs.json` can be empty due to previously failed
|
||||
// PutObject() transaction, if we arrive at such
|
||||
// a situation we just ignore and continue.
|
||||
if errorCause(err) != io.EOF {
|
||||
return "", toObjectErr(err, bucket, entry)
|
||||
}
|
||||
}
|
||||
|
||||
fsMetaMap := parseFSMetaMap(fsMetaBuf)
|
||||
return fsMetaMap["md5Sum"], nil
|
||||
}
|
||||
|
||||
// ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
|
||||
// state for future re-entrant list requests.
|
||||
func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
@ -731,14 +767,33 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
|
||||
objInfo.IsDir = true
|
||||
return
|
||||
}
|
||||
|
||||
// Protect reading `fs.json`.
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, entry)
|
||||
objectLock.RLock()
|
||||
var md5Sum string
|
||||
md5Sum, err = fs.getObjectETag(bucket, entry)
|
||||
objectLock.RUnlock()
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
// Stat the file to get file size.
|
||||
var fi os.FileInfo
|
||||
fi, err = fsStatFile(pathJoin(fs.fsPath, bucket, entry))
|
||||
if err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, entry)
|
||||
}
|
||||
fsMeta := fsMetaV1{}
|
||||
return fsMeta.ToObjectInfo(bucket, entry, fi), nil
|
||||
|
||||
// Success.
|
||||
return ObjectInfo{
|
||||
Name: entry,
|
||||
Bucket: bucket,
|
||||
Size: fi.Size(),
|
||||
ModTime: fi.ModTime(),
|
||||
IsDir: fi.IsDir(),
|
||||
MD5Sum: md5Sum,
|
||||
}, nil
|
||||
}
|
||||
|
||||
heal := false // true only for xl.ListObjectsHeal()
|
||||
|
@ -81,7 +81,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
{
|
||||
IsTruncated: false,
|
||||
Objects: []ObjectInfo{
|
||||
{Name: "Asia-maps.png", ContentType: "image/png"},
|
||||
{Name: "Asia-maps.png"},
|
||||
{Name: "Asia/India/India-summer-photos-1"},
|
||||
{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
|
||||
{Name: "newPrefix0"},
|
||||
@ -97,7 +97,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
{
|
||||
IsTruncated: true,
|
||||
Objects: []ObjectInfo{
|
||||
{Name: "Asia-maps.png", ContentType: "image/png"},
|
||||
{Name: "Asia-maps.png"},
|
||||
{Name: "Asia/India/India-summer-photos-1"},
|
||||
{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
|
||||
{Name: "newPrefix0"},
|
||||
@ -109,7 +109,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
{
|
||||
IsTruncated: true,
|
||||
Objects: []ObjectInfo{
|
||||
{Name: "Asia-maps.png", ContentType: "image/png"},
|
||||
{Name: "Asia-maps.png"},
|
||||
{Name: "Asia/India/India-summer-photos-1"},
|
||||
{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
|
||||
{Name: "newPrefix0"},
|
||||
@ -120,7 +120,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
{
|
||||
IsTruncated: true,
|
||||
Objects: []ObjectInfo{
|
||||
{Name: "Asia-maps.png", ContentType: "image/png"},
|
||||
{Name: "Asia-maps.png"},
|
||||
{Name: "Asia/India/India-summer-photos-1"},
|
||||
{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
|
||||
},
|
||||
@ -131,7 +131,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
{
|
||||
IsTruncated: true,
|
||||
Objects: []ObjectInfo{
|
||||
{Name: "Asia-maps.png", ContentType: "image/png"},
|
||||
{Name: "Asia-maps.png"},
|
||||
},
|
||||
},
|
||||
// ListObjectsResult-5.
|
||||
@ -234,7 +234,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
{
|
||||
IsTruncated: false,
|
||||
Objects: []ObjectInfo{
|
||||
{Name: "Asia-maps.png", ContentType: "image/png"},
|
||||
{Name: "Asia-maps.png"},
|
||||
{Name: "Asia/India/India-summer-photos-1"},
|
||||
{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
|
||||
{Name: "newPrefix0"},
|
||||
@ -343,7 +343,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
{
|
||||
IsTruncated: false,
|
||||
Objects: []ObjectInfo{
|
||||
{Name: "Asia-maps.png", ContentType: "image/png"},
|
||||
{Name: "Asia-maps.png"},
|
||||
{Name: "Asia/India/India-summer-photos-1"},
|
||||
{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
|
||||
},
|
||||
@ -354,7 +354,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
{
|
||||
IsTruncated: false,
|
||||
Objects: []ObjectInfo{
|
||||
{Name: "Asia-maps.png", ContentType: "image/png"},
|
||||
{Name: "Asia-maps.png"},
|
||||
},
|
||||
},
|
||||
// ListObjectsResult-26.
|
||||
@ -549,8 +549,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
if testCase.result.Objects[j].Name != result.Objects[j].Name {
|
||||
t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name)
|
||||
}
|
||||
if testCase.result.Objects[j].ContentType != result.Objects[j].ContentType {
|
||||
t.Errorf("Test %d: %s: Expected object contentType to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].ContentType, result.Objects[j].ContentType)
|
||||
if result.Objects[j].MD5Sum == "" {
|
||||
t.Errorf("Test %d: %s: Expected md5sum to be not empty, but found empty instead", i+1, instanceType)
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -67,6 +67,9 @@ func init() {
|
||||
|
||||
// Set system resources to maximum.
|
||||
setMaxResources()
|
||||
|
||||
// Quiet logging.
|
||||
log.logger.Hooks = nil
|
||||
}
|
||||
|
||||
func prepareFS() (ObjectLayer, string, error) {
|
||||
|
Loading…
Reference in New Issue
Block a user