MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-12-07 00:02:31 -05:00
Code Issues Packages Projects Releases Wiki Activity

Avoid using fastjson parser pool, move back to jsoniter (#8190)

Browse Source 
It looks like from implementation point of view fastjson
parser pool doesn't behave the same way as expected
when dealing many `xl.json` from multiple disks.

The fastjson parser pool usage ends up returning incorrect
xl.json entries for checksums, with references pointing
to older entries. This led to the subtle bug where checksum
info is duplicated from a previous xl.json read of a different
file from different disk.
This commit is contained in:
Harshavardhana
2019-09-05 15:51:27 -07:00
committed by kannappanr
parent 428836d4e1
commit b52a3e523c
15 changed files with 169 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
cmd/fs-v1-metadata.go
View File

@@ -27,10 +27,10 @@ import (
pathutil "path"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/lock"
"github.com/minio/minio/pkg/mimedb"
"github.com/valyala/fastjson"
)
// FS format, and object metadata.
@@ -203,37 +203,6 @@ func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
return fi.Size(), nil
}
func parseFSVersion(v *fastjson.Value) string {
return string(v.GetStringBytes("version"))
}
func parseFSMetaMap(v *fastjson.Value) map[string]string {
metaMap := make(map[string]string)
// Get fsMetaV1.Meta map.
v.GetObject("meta").Visit(func(k []byte, kv *fastjson.Value) {
metaMap[string(k)] = string(kv.GetStringBytes())
})
return metaMap
}
func parseFSPartsArray(v *fastjson.Value) []ObjectPartInfo {
// Get xlMetaV1.Parts array
var partsArray []ObjectPartInfo
for _, result := range v.GetArray("parts") {
partsArray = append(partsArray, ObjectPartInfo{
Number: result.GetInt("number"),
Name: string(result.GetStringBytes("name")),
ETag: string(result.GetStringBytes("etag")),
Size: result.GetInt64("size"),
ActualSize: result.GetInt64("actualSize"),
})
}
return partsArray
}
// fs.json parser pool
var fsParserPool fastjson.ParserPool
func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64, err error) {
var fsMetaBuf []byte
fi, err := lk.Stat()
@@ -253,18 +222,11 @@ func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64,
return 0, io.EOF
}
parser := fsParserPool.Get()
defer fsParserPool.Put(parser)
var v *fastjson.Value
v, err = parser.ParseBytes(fsMetaBuf)
if err != nil {
var json = jsoniter.ConfigCompatibleWithStandardLibrary
if err = json.Unmarshal(fsMetaBuf, m); err != nil {
return 0, err
}
// obtain version.
m.Version = parseFSVersion(v)
// Verify if the format is valid, return corrupted format
// for unrecognized formats.
if !isFSMetaValid(m.Version) {
@@ -273,12 +235,6 @@ func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64,
return 0, errCorruptedFormat
}
// obtain parts information
m.Parts = parseFSPartsArray(v)
// obtain metadata.
m.Meta = parseFSMetaMap(v)
// Success.
return int64(len(fsMetaBuf)), nil
}