mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
fix: minor allocation improvements in xlMetaV2 (#12133)
This commit is contained in:
parent
0bab1c1895
commit
254698f126
@ -190,13 +190,12 @@ func TestFromMinioClientListBucketResultToV2Info(t *testing.T) {
|
||||
|
||||
// Test for gcsParseProjectID
|
||||
func TestGCSParseProjectID(t *testing.T) {
|
||||
f, err := ioutil.TempFile("", "")
|
||||
f, err := ioutil.TempFile("", "TestGCSParseProjectID-*")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
|
||||
contents := `
|
||||
{
|
||||
@ -205,6 +204,7 @@ func TestGCSParseProjectID(t *testing.T) {
|
||||
}
|
||||
`
|
||||
f.WriteString(contents)
|
||||
f.Close()
|
||||
projectID, err := gcsParseProjectID(f.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@ -217,8 +217,20 @@ func TestGCSParseProjectID(t *testing.T) {
|
||||
t.Errorf(`Expected to fail but succeeded reading "non-existent"`)
|
||||
}
|
||||
|
||||
f.WriteString(`,}`)
|
||||
|
||||
contents = `
|
||||
{
|
||||
"type": "service_account",
|
||||
"project_id": "miniotesting"
|
||||
},}
|
||||
`
|
||||
f, err = ioutil.TempFile("", "TestGCSParseProjectID-*")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
f.WriteString(contents)
|
||||
f.Close()
|
||||
if _, err := gcsParseProjectID(f.Name()); err == nil {
|
||||
t.Errorf(`Expected to fail reading corrupted credentials file`)
|
||||
}
|
||||
|
@ -852,6 +852,8 @@ func streamHTTPResponse(w http.ResponseWriter) *httpStreamResponse {
|
||||
// The returned reader contains the payload and must be closed if no error is returned.
|
||||
func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
|
||||
var tmp [1]byte
|
||||
// 8K copy buffer, reused for less allocs...
|
||||
var buf [8 << 10]byte
|
||||
for {
|
||||
_, err := io.ReadFull(respBody, tmp[:])
|
||||
if err != nil {
|
||||
@ -861,7 +863,7 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
|
||||
switch tmp[0] {
|
||||
case 0:
|
||||
// 0 is unbuffered, copy the rest.
|
||||
_, err := io.Copy(w, respBody)
|
||||
_, err := io.CopyBuffer(w, respBody, buf[:])
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
@ -880,7 +882,7 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
|
||||
return err
|
||||
}
|
||||
length := binary.LittleEndian.Uint32(tmp[:])
|
||||
_, err = io.CopyN(w, respBody, int64(length))
|
||||
_, err = io.CopyBuffer(w, io.LimitReader(respBody, int64(length)), buf[:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -18,21 +18,24 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
)
|
||||
|
||||
// versionsSorter sorts FileInfo slices by version.
|
||||
type versionsSorter []FileInfo
|
||||
|
||||
func (v versionsSorter) Len() int { return len(v) }
|
||||
func (v versionsSorter) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
|
||||
func (v versionsSorter) Less(i, j int) bool {
|
||||
if v[i].IsLatest {
|
||||
return true
|
||||
}
|
||||
if v[j].IsLatest {
|
||||
return false
|
||||
}
|
||||
return v[i].ModTime.After(v[j].ModTime)
|
||||
func (v versionsSorter) sort() {
|
||||
sort.Slice(v, func(i, j int) bool {
|
||||
if v[i].IsLatest {
|
||||
return true
|
||||
}
|
||||
if v[j].IsLatest {
|
||||
return false
|
||||
}
|
||||
return v[i].ModTime.After(v[j].ModTime)
|
||||
})
|
||||
}
|
||||
|
||||
func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) {
|
||||
|
@ -524,13 +524,35 @@ func (x *xlMetaInlineData) rename(oldKey, newKey string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// remove will remove a key.
|
||||
// Returns whether the key was found.
|
||||
func (x *xlMetaInlineData) remove(key string) bool {
|
||||
// remove will remove one or more keys.
|
||||
// Returns true if any key was found.
|
||||
func (x *xlMetaInlineData) remove(keys ...string) bool {
|
||||
in := x.afterVersion()
|
||||
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
|
||||
keys := make([][]byte, 0, sz)
|
||||
vals := make([][]byte, 0, sz)
|
||||
newKeys := make([][]byte, 0, sz)
|
||||
newVals := make([][]byte, 0, sz)
|
||||
var removeKey func(s []byte) bool
|
||||
|
||||
// Copy if big number of compares...
|
||||
if len(keys) > 5 && sz > 5 {
|
||||
mKeys := make(map[string]struct{}, len(keys))
|
||||
for _, key := range keys {
|
||||
mKeys[key] = struct{}{}
|
||||
}
|
||||
removeKey = func(s []byte) bool {
|
||||
_, ok := mKeys[string(s)]
|
||||
return ok
|
||||
}
|
||||
} else {
|
||||
removeKey = func(s []byte) bool {
|
||||
for _, key := range keys {
|
||||
if key == string(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Version plus header...
|
||||
plSize := 1 + msgp.MapHeaderSize
|
||||
@ -546,10 +568,10 @@ func (x *xlMetaInlineData) remove(key string) bool {
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if string(foundKey) != key {
|
||||
if !removeKey(foundKey) {
|
||||
plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal)
|
||||
keys = append(keys, foundKey)
|
||||
vals = append(vals, foundVal)
|
||||
newKeys = append(newKeys, foundKey)
|
||||
newVals = append(newVals, foundVal)
|
||||
} else {
|
||||
found = true
|
||||
}
|
||||
@ -559,13 +581,13 @@ func (x *xlMetaInlineData) remove(key string) bool {
|
||||
return false
|
||||
}
|
||||
// If none left...
|
||||
if len(keys) == 0 {
|
||||
if len(newKeys) == 0 {
|
||||
*x = nil
|
||||
return true
|
||||
}
|
||||
|
||||
// Reserialize...
|
||||
x.serialize(plSize, keys, vals)
|
||||
x.serialize(plSize, newKeys, newVals)
|
||||
return true
|
||||
}
|
||||
|
||||
@ -891,11 +913,6 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func newXLMetaV2(fi FileInfo) (xlMetaV2, error) {
|
||||
xlMeta := xlMetaV2{}
|
||||
return xlMeta, xlMeta.AddVersion(fi)
|
||||
}
|
||||
|
||||
func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error) {
|
||||
versionID := ""
|
||||
var uv uuid.UUID
|
||||
@ -937,7 +954,7 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
|
||||
versionID := ""
|
||||
var uv uuid.UUID
|
||||
// check if the version is not "null"
|
||||
if !bytes.Equal(j.VersionID[:], uv[:]) {
|
||||
if j.VersionID != uv {
|
||||
versionID = uuid.UUID(j.VersionID).String()
|
||||
}
|
||||
fi := FileInfo{
|
||||
@ -1233,7 +1250,7 @@ func (z xlMetaV2) ListVersions(volume, path string) ([]FileInfo, time.Time, erro
|
||||
versions = append(versions, fi)
|
||||
}
|
||||
|
||||
sort.Sort(versionsSorter(versions))
|
||||
versionsSorter(versions).sort()
|
||||
|
||||
for i := range versions {
|
||||
versions[i].NumVersions = len(versions)
|
||||
|
@ -852,11 +852,10 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
if versionID == "" {
|
||||
versionID = nullVersionID
|
||||
}
|
||||
xlMeta.data.remove(versionID)
|
||||
// PR #11758 used DataDir, preserve it
|
||||
// for users who might have used master
|
||||
// branch
|
||||
xlMeta.data.remove(dataDir)
|
||||
xlMeta.data.remove(versionID, dataDir)
|
||||
filePath := pathJoin(volumeDir, path, dataDir)
|
||||
if err = checkPathLength(filePath); err != nil {
|
||||
return err
|
||||
@ -940,7 +939,7 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
|
||||
|
||||
var xlMeta xlMetaV2
|
||||
if !isXL2V1Format(buf) {
|
||||
xlMeta, err = newXLMetaV2(fi)
|
||||
err = xlMeta.AddVersion(fi)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
|
Loading…
Reference in New Issue
Block a user