MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-12-07 00:02:31 -05:00
Code Issues Packages Projects Releases Wiki Activity

Avoid using jsoniter, move to fastjson (#8063)

Browse Source 
This is to avoid using unsafe.Pointer type
code dependency for MinIO, this causes
crashes on ARM64 platforms

Refer #8005 collection of runtime crashes due
to unsafe.Pointer usage incorrectly. We have
seen issues like this before when using
jsoniter library in the past.

This PR hopes to fix this using fastjson
This commit is contained in:
Harshavardhana
2019-08-19 08:35:52 -10:00
committed by GitHub
parent b3ca304c01
commit 9ca7470ccc
10 changed files with 335 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
cmd/fs-v1-metadata.go
View File

@@ -30,7 +30,7 @@ import (
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/lock"
"github.com/minio/minio/pkg/mimedb"
"github.com/tidwall/gjson"
"github.com/valyala/fastjson"
)
// FS format, and object metadata.
@@ -203,44 +203,37 @@ func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
return fi.Size(), nil
}
func parseFSVersion(fsMetaBuf []byte) string {
return gjson.GetBytes(fsMetaBuf, "version").String()
func parseFSVersion(v *fastjson.Value) string {
return string(v.GetStringBytes("version"))
}
func parseFSMetaMap(fsMetaBuf []byte) map[string]string {
// Get xlMetaV1.Meta map.
metaMapResult := gjson.GetBytes(fsMetaBuf, "meta").Map()
func parseFSMetaMap(v *fastjson.Value) map[string]string {
metaMap := make(map[string]string)
for key, valResult := range metaMapResult {
metaMap[key] = valResult.String()
}
// Get fsMetaV1.Meta map.
v.GetObject("meta").Visit(func(k []byte, kv *fastjson.Value) {
metaMap[string(k)] = string(kv.GetStringBytes())
})
return metaMap
}
func parseFSPartsArray(fsMetaBuf []byte) []ObjectPartInfo {
func parseFSPartsArray(v *fastjson.Value) []ObjectPartInfo {
// Get xlMetaV1.Parts array
var partsArray []ObjectPartInfo
partsArrayResult := gjson.GetBytes(fsMetaBuf, "parts")
partsArrayResult.ForEach(func(key, part gjson.Result) bool {
partJSON := part.String()
number := gjson.Get(partJSON, "number").Int()
name := gjson.Get(partJSON, "name").String()
etag := gjson.Get(partJSON, "etag").String()
size := gjson.Get(partJSON, "size").Int()
actualSize := gjson.Get(partJSON, "actualSize").Int()
for _, result := range v.GetArray("parts") {
partsArray = append(partsArray, ObjectPartInfo{
Number: int(number),
Name: name,
ETag: etag,
Size: size,
ActualSize: int64(actualSize),
Number: result.GetInt("number"),
Name: string(result.GetStringBytes("name")),
ETag: string(result.GetStringBytes("etag")),
Size: result.GetInt64("size"),
ActualSize: result.GetInt64("actualSize"),
})
return true
})
}
return partsArray
}
// fs.json parser pool
var fsParserPool fastjson.ParserPool
func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64, err error) {
var fsMetaBuf []byte
fi, err := lk.Stat()
@@ -260,8 +253,17 @@ func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64,
return 0, io.EOF
}
parser := fsParserPool.Get()
defer fsParserPool.Put(parser)
var v *fastjson.Value
v, err = parser.ParseBytes(fsMetaBuf)
if err != nil {
return 0, err
}
// obtain version.
m.Version = parseFSVersion(fsMetaBuf)
m.Version = parseFSVersion(v)
// Verify if the format is valid, return corrupted format
// for unrecognized formats.
@@ -272,10 +274,10 @@ func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64,
}
// obtain parts information
m.Parts = parseFSPartsArray(fsMetaBuf)
m.Parts = parseFSPartsArray(v)
// obtain metadata.
m.Meta = parseFSMetaMap(fsMetaBuf)
m.Meta = parseFSMetaMap(v)
// Success.
return int64(len(fsMetaBuf)), nil