mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Improve performance on multiple versions (#13573)
Existing:
```go
type xlMetaV2 struct {
Versions []xlMetaV2Version `json:"Versions" msg:"Versions"`
}
```
Serialized as regular MessagePack.
```go
//msgp:tuple xlMetaV2VersionHeader
type xlMetaV2VersionHeader struct {
VersionID [16]byte
ModTime int64
Type VersionType
Flags xlFlags
}
```
Serialize as streaming MessagePack, format:
```
int(headerVersion)
int(xlmetaVersion)
int(nVersions)
for each version {
binary blob, xlMetaV2VersionHeader, serialized
binary blob, xlMetaV2Version, serialized.
}
```
xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct
can easily be reused and does not contain pointers, so efficient as a
slice (single allocation)
This allows quickly parsing everything as slices of bytes (no copy).
Versions are always *saved* sorted by modTime, newest *first*.
No more need to sort on load.
* Allows checking if a version exists.
* Allows reading single version without unmarshal all.
* Allows reading latest version of type without unmarshal all.
* Allows reading latest version without unmarshal of all.
* Allows checking if the latest is deleteMarker by reading first entry.
* Allows adding/updating/deleting a version with only header deserialization.
* Reduces allocations on conversion to FileInfo(s).
This commit is contained in:
Klaus Post
GitHub
parent
7152915318
commit
faf013ec84
2
Makefile
2
Makefile
@ -20,7 +20,7 @@ help: ## print this help
|
||||
getdeps: ## fetch necessary dependencies
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.43.0
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@latest
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
|
||||
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
|
||||
|
||||
crosscompile: ## cross compile minio
|
||||
|
||||
@ -724,7 +724,7 @@ const (
|
||||
// matches k1 with all keys, returns 'true' if one of them matches
|
||||
func equals(k1 string, keys ...string) bool {
|
||||
for _, k2 := range keys {
|
||||
if strings.EqualFold(strings.ToLower(k1), strings.ToLower(k2)) {
|
||||
if strings.EqualFold(k1, k2) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@ -476,9 +476,7 @@ func GetInternalReplicationState(m map[string][]byte) ReplicationState {
|
||||
|
||||
// getInternalReplicationState fetches internal replication state from the map m
|
||||
func getInternalReplicationState(m map[string]string) ReplicationState {
|
||||
d := ReplicationState{
|
||||
ResetStatusesMap: make(map[string]string),
|
||||
}
|
||||
d := ReplicationState{}
|
||||
for k, v := range m {
|
||||
switch {
|
||||
case equals(k, ReservedMetadataPrefixLower+ReplicationTimestamp):
|
||||
@ -497,6 +495,9 @@ func getInternalReplicationState(m map[string]string) ReplicationState {
|
||||
d.PurgeTargets = versionPurgeStatusesMap(v)
|
||||
case strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset):
|
||||
arn := strings.TrimPrefix(k, fmt.Sprintf("%s-", ReservedMetadataPrefixLower+ReplicationReset))
|
||||
if d.ResetStatusesMap == nil {
|
||||
d.ResetStatusesMap = make(map[string]string, 1)
|
||||
}
|
||||
d.ResetStatusesMap[arn] = v
|
||||
}
|
||||
}
|
||||
|
||||
@ -148,11 +148,15 @@ func (e *metaCacheEntry) isLatestDeletemarker() bool {
|
||||
if !isXL2V1Format(e.metadata) {
|
||||
return false
|
||||
}
|
||||
if meta, _ := isIndexedMetaV2(e.metadata); meta != nil {
|
||||
return meta.IsLatestDeleteMarker()
|
||||
}
|
||||
// Fall back...
|
||||
var xlMeta xlMetaV2
|
||||
if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.Versions) == 0 {
|
||||
if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.versions) == 0 {
|
||||
return true
|
||||
}
|
||||
return xlMeta.Versions[len(xlMeta.Versions)-1].Type == DeleteType
|
||||
return xlMeta.versions[0].header.Type == DeleteType
|
||||
}
|
||||
|
||||
// fileInfo returns the decoded metadata.
|
||||
|
||||
BIN
cmd/testdata/xl.meta-v1.2.zst
vendored
Normal file
BIN
cmd/testdata/xl.meta-v1.2.zst
vendored
Normal file
Binary file not shown.
@ -19,27 +19,12 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
// versionsSorter sorts FileInfo slices by version.
|
||||
type versionsSorter []FileInfo
|
||||
|
||||
func (v versionsSorter) sort() {
|
||||
sort.Slice(v, func(i, j int) bool {
|
||||
if v[i].IsLatest {
|
||||
return true
|
||||
}
|
||||
if v[j].IsLatest {
|
||||
return false
|
||||
}
|
||||
return v[i].ModTime.After(v[j].ModTime)
|
||||
})
|
||||
}
|
||||
|
||||
func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) {
|
||||
fivs, err := getAllFileInfoVersions(xlMetaBuf, volume, path)
|
||||
if err != nil {
|
||||
@ -54,24 +39,35 @@ func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersion
|
||||
}
|
||||
}
|
||||
fivs.Versions = fivs.Versions[:n]
|
||||
// Update numversions
|
||||
for i := range fivs.Versions {
|
||||
fivs.Versions[i].NumVersions = n
|
||||
}
|
||||
return fivs, nil
|
||||
}
|
||||
|
||||
func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) {
|
||||
if isXL2V1Format(xlMetaBuf) {
|
||||
var xlMeta xlMetaV2
|
||||
if err := xlMeta.Load(xlMetaBuf); err != nil {
|
||||
return FileInfoVersions{}, err
|
||||
}
|
||||
versions, latestModTime, err := xlMeta.ListVersions(volume, path)
|
||||
if err != nil {
|
||||
var versions []FileInfo
|
||||
var err error
|
||||
if buf, _ := isIndexedMetaV2(xlMetaBuf); buf != nil {
|
||||
versions, err = buf.ListVersions(volume, path)
|
||||
} else {
|
||||
var xlMeta xlMetaV2
|
||||
if err := xlMeta.Load(xlMetaBuf); err != nil {
|
||||
return FileInfoVersions{}, err
|
||||
}
|
||||
versions, err = xlMeta.ListVersions(volume, path)
|
||||
}
|
||||
if err != nil || len(versions) == 0 {
|
||||
return FileInfoVersions{}, err
|
||||
}
|
||||
|
||||
return FileInfoVersions{
|
||||
Volume: volume,
|
||||
Name: path,
|
||||
Versions: versions,
|
||||
LatestModTime: latestModTime,
|
||||
LatestModTime: versions[0].ModTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -98,11 +94,20 @@ func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVers
|
||||
|
||||
func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (FileInfo, error) {
|
||||
if isXL2V1Format(xlMetaBuf) {
|
||||
var xlMeta xlMetaV2
|
||||
if err := xlMeta.Load(xlMetaBuf); err != nil {
|
||||
return FileInfo{}, err
|
||||
var fi FileInfo
|
||||
var err error
|
||||
var inData xlMetaInlineData
|
||||
if buf, data := isIndexedMetaV2(xlMetaBuf); buf != nil {
|
||||
inData = data
|
||||
fi, err = buf.ToFileInfo(volume, path, versionID)
|
||||
} else {
|
||||
var xlMeta xlMetaV2
|
||||
if err := xlMeta.Load(xlMetaBuf); err != nil {
|
||||
return FileInfo{}, err
|
||||
}
|
||||
inData = xlMeta.data
|
||||
fi, err = xlMeta.ToFileInfo(volume, path, versionID)
|
||||
}
|
||||
fi, err := xlMeta.ToFileInfo(volume, path, versionID)
|
||||
if !data || err != nil {
|
||||
return fi, err
|
||||
}
|
||||
@ -110,12 +115,12 @@ func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (F
|
||||
if versionID == "" {
|
||||
versionID = nullVersionID
|
||||
}
|
||||
fi.Data = xlMeta.data.find(versionID)
|
||||
fi.Data = inData.find(versionID)
|
||||
if len(fi.Data) == 0 {
|
||||
// PR #11758 used DataDir, preserve it
|
||||
// for users who might have used master
|
||||
// branch
|
||||
fi.Data = xlMeta.data.find(fi.DataDir)
|
||||
fi.Data = inData.find(fi.DataDir)
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
@ -149,3 +154,27 @@ func getXLDiskLoc(diskID string) (poolIdx, setIdx, diskIdx int) {
|
||||
}
|
||||
return -1, -1, -1
|
||||
}
|
||||
|
||||
// hashDeterministicString will return a deterministic hash for the map values.
|
||||
// Trivial collisions are avoided, but this is by no means a strong hash.
|
||||
func hashDeterministicString(m map[string]string) uint64 {
|
||||
// Seed (random)
|
||||
var crc = uint64(0xc2b40bbac11a7295)
|
||||
// Xor each value to make order independent
|
||||
for k, v := range m {
|
||||
// Separate key and value with an individual xor with a random number.
|
||||
// Add values of each, so they cannot be trivially collided.
|
||||
crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.HashString(v) ^ 0x8da4c8da66194257)
|
||||
}
|
||||
return crc
|
||||
}
|
||||
|
||||
// hashDeterministicBytes will return a deterministic (weak) hash for the map values.
|
||||
// Trivial collisions are avoided, but this is by no means a strong hash.
|
||||
func hashDeterministicBytes(m map[string][]byte) uint64 {
|
||||
var crc = uint64(0x1bbc7e1dde654743)
|
||||
for k, v := range m {
|
||||
crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.Hash(v) ^ 0x8da4c8da66194257)
|
||||
}
|
||||
return crc
|
||||
}
|
||||
|
||||
93
cmd/xl-storage-format-utils_test.go
Normal file
93
cmd/xl-storage-format-utils_test.go
Normal file
@ -0,0 +1,93 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
func Test_hashDeterministicString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
arg map[string]string
|
||||
}{
|
||||
{
|
||||
name: "zero",
|
||||
arg: map[string]string{},
|
||||
},
|
||||
{
|
||||
name: "nil",
|
||||
arg: nil,
|
||||
},
|
||||
{
|
||||
name: "one",
|
||||
arg: map[string]string{"key": "value"},
|
||||
},
|
||||
{
|
||||
name: "several",
|
||||
arg: map[string]string{
|
||||
xhttp.AmzRestore: "FAILED",
|
||||
xhttp.ContentMD5: mustGetUUID(),
|
||||
xhttp.AmzBucketReplicationStatus: "PENDING",
|
||||
xhttp.ContentType: "application/json",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "someempty",
|
||||
arg: map[string]string{
|
||||
xhttp.AmzRestore: "",
|
||||
xhttp.ContentMD5: mustGetUUID(),
|
||||
xhttp.AmzBucketReplicationStatus: "",
|
||||
xhttp.ContentType: "application/json",
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
const n = 100
|
||||
want := hashDeterministicString(tt.arg)
|
||||
m := tt.arg
|
||||
for i := 0; i < n; i++ {
|
||||
if got := hashDeterministicString(m); got != want {
|
||||
t.Errorf("hashDeterministicString() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
// Check casual collisions
|
||||
if m == nil {
|
||||
m = make(map[string]string)
|
||||
}
|
||||
m["12312312"] = ""
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
want = hashDeterministicString(m)
|
||||
delete(m, "12312312")
|
||||
m["another"] = ""
|
||||
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
|
||||
want = hashDeterministicString(m)
|
||||
m["another"] = "hashDeterministicString"
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
|
||||
want = hashDeterministicString(m)
|
||||
m["another"] = "hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString"
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
|
||||
// Flip key/value
|
||||
want = hashDeterministicString(m)
|
||||
delete(m, "another")
|
||||
m["hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString"] = "another"
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -18,11 +18,13 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
@ -205,6 +207,27 @@ func (m *xlMetaV1Object) ToFileInfo(volume, path string) (FileInfo, error) {
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
// Signature will return a signature that is expected to be the same across all disks.
|
||||
func (m *xlMetaV1Object) Signature() [4]byte {
|
||||
// Shallow copy
|
||||
c := *m
|
||||
// Zero unimportant fields
|
||||
c.Erasure.Index = 0
|
||||
c.Minio.Release = ""
|
||||
crc := hashDeterministicString(c.Meta)
|
||||
c.Meta = nil
|
||||
|
||||
if bts, err := c.MarshalMsg(metaDataPoolGet()); err == nil {
|
||||
crc ^= xxhash.Sum64(bts)
|
||||
metaDataPoolPut(bts)
|
||||
}
|
||||
|
||||
// Combine upper and lower part
|
||||
var tmp [4]byte
|
||||
binary.LittleEndian.PutUint32(tmp[:], uint32(crc^(crc>>32)))
|
||||
return tmp
|
||||
}
|
||||
|
||||
// XL metadata constants.
|
||||
const (
|
||||
// XL meta version.
|
||||
|
||||
89
cmd/xl-storage-format-v2-legacy.go
Normal file
89
cmd/xl-storage-format-v2-legacy.go
Normal file
@ -0,0 +1,89 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
// unmarshalV unmarshals with a specific header version.
|
||||
func (x *xlMetaV2VersionHeader) unmarshalV(v uint8, bts []byte) (o []byte, err error) {
|
||||
switch v {
|
||||
case 1:
|
||||
return x.unmarshalV1(bts)
|
||||
case xlHeaderVersion:
|
||||
return x.UnmarshalMsg(bts)
|
||||
}
|
||||
return bts, fmt.Errorf("unknown xlHeaderVersion: %d", v)
|
||||
}
|
||||
|
||||
// unmarshalV1 decodes version 1, never released.
|
||||
func (x *xlMetaV2VersionHeader) unmarshalV1(bts []byte) (o []byte, err error) {
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if zb0001 != 4 {
|
||||
err = msgp.ArrayError{Wanted: 4, Got: zb0001}
|
||||
return
|
||||
}
|
||||
bts, err = msgp.ReadExactBytes(bts, (x.VersionID)[:])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "VersionID")
|
||||
return
|
||||
}
|
||||
x.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ModTime")
|
||||
return
|
||||
}
|
||||
{
|
||||
var zb0002 uint8
|
||||
zb0002, bts, err = msgp.ReadUint8Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Type")
|
||||
return
|
||||
}
|
||||
x.Type = VersionType(zb0002)
|
||||
}
|
||||
{
|
||||
var zb0003 uint8
|
||||
zb0003, bts, err = msgp.ReadUint8Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Flags")
|
||||
return
|
||||
}
|
||||
x.Flags = xlFlags(zb0003)
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// unmarshalV unmarshals with a specific metadata version.
|
||||
func (j *xlMetaV2Version) unmarshalV(v uint8, bts []byte) (o []byte, err error) {
|
||||
switch v {
|
||||
// We accept un-set as latest version.
|
||||
case 0, xlMetaVersion:
|
||||
return j.UnmarshalMsg(bts)
|
||||
}
|
||||
return bts, fmt.Errorf("unknown xlMetaVersion: %d", v)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -9,8 +9,8 @@ import (
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
func TestMarshalUnmarshalxlMetaV2(t *testing.T) {
|
||||
v := xlMetaV2{}
|
||||
func TestMarshalUnmarshalxlMetaDataDirDecoder(t *testing.T) {
|
||||
v := xlMetaDataDirDecoder{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@ -32,8 +32,8 @@ func TestMarshalUnmarshalxlMetaV2(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgxlMetaV2(b *testing.B) {
|
||||
v := xlMetaV2{}
|
||||
func BenchmarkMarshalMsgxlMetaDataDirDecoder(b *testing.B) {
|
||||
v := xlMetaDataDirDecoder{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
@ -41,8 +41,8 @@ func BenchmarkMarshalMsgxlMetaV2(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgxlMetaV2(b *testing.B) {
|
||||
v := xlMetaV2{}
|
||||
func BenchmarkAppendMsgxlMetaDataDirDecoder(b *testing.B) {
|
||||
v := xlMetaDataDirDecoder{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
@ -53,8 +53,8 @@ func BenchmarkAppendMsgxlMetaV2(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalxlMetaV2(b *testing.B) {
|
||||
v := xlMetaV2{}
|
||||
func BenchmarkUnmarshalxlMetaDataDirDecoder(b *testing.B) {
|
||||
v := xlMetaDataDirDecoder{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
@ -67,17 +67,17 @@ func BenchmarkUnmarshalxlMetaV2(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodexlMetaV2(t *testing.T) {
|
||||
v := xlMetaV2{}
|
||||
func TestEncodeDecodexlMetaDataDirDecoder(t *testing.T) {
|
||||
v := xlMetaDataDirDecoder{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodexlMetaV2 Msgsize() is inaccurate")
|
||||
t.Log("WARNING: TestEncodeDecodexlMetaDataDirDecoder Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := xlMetaV2{}
|
||||
vn := xlMetaDataDirDecoder{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
@ -91,8 +91,8 @@ func TestEncodeDecodexlMetaV2(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodexlMetaV2(b *testing.B) {
|
||||
v := xlMetaV2{}
|
||||
func BenchmarkEncodexlMetaDataDirDecoder(b *testing.B) {
|
||||
v := xlMetaDataDirDecoder{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
@ -105,8 +105,8 @@ func BenchmarkEncodexlMetaV2(b *testing.B) {
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodexlMetaV2(b *testing.B) {
|
||||
v := xlMetaV2{}
|
||||
func BenchmarkDecodexlMetaDataDirDecoder(b *testing.B) {
|
||||
v := xlMetaDataDirDecoder{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
@ -460,3 +460,116 @@ func BenchmarkDecodexlMetaV2Version(b *testing.B) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalxlMetaV2VersionHeader(t *testing.T) {
|
||||
v := xlMetaV2VersionHeader{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgxlMetaV2VersionHeader(b *testing.B) {
|
||||
v := xlMetaV2VersionHeader{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgxlMetaV2VersionHeader(b *testing.B) {
|
||||
v := xlMetaV2VersionHeader{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalxlMetaV2VersionHeader(b *testing.B) {
|
||||
v := xlMetaV2VersionHeader{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodexlMetaV2VersionHeader(t *testing.T) {
|
||||
v := xlMetaV2VersionHeader{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodexlMetaV2VersionHeader Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := xlMetaV2VersionHeader{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodexlMetaV2VersionHeader(b *testing.B) {
|
||||
v := xlMetaV2VersionHeader{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodexlMetaV2VersionHeader(b *testing.B) {
|
||||
v := xlMetaV2VersionHeader{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
27
cmd/xl-storage-format-v2_string.go
Normal file
27
cmd/xl-storage-format-v2_string.go
Normal file
@ -0,0 +1,27 @@
|
||||
// Code generated by "stringer -type VersionType -output=xl-storage-format-v2_string.go xl-storage-format-v2.go"; DO NOT EDIT.
|
||||
|
||||
package cmd
|
||||
|
||||
import "strconv"
|
||||
|
||||
func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
// Re-run the stringer command to generate them again.
|
||||
var x [1]struct{}
|
||||
_ = x[invalidVersionType-0]
|
||||
_ = x[ObjectType-1]
|
||||
_ = x[DeleteType-2]
|
||||
_ = x[LegacyType-3]
|
||||
_ = x[lastVersionType-4]
|
||||
}
|
||||
|
||||
const _VersionType_name = "invalidVersionTypeObjectTypeDeleteTypeLegacyTypelastVersionType"
|
||||
|
||||
var _VersionType_index = [...]uint8{0, 18, 28, 38, 48, 63}
|
||||
|
||||
func (i VersionType) String() string {
|
||||
if i >= VersionType(len(_VersionType_index)-1) {
|
||||
return "VersionType(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _VersionType_name[_VersionType_index[i]:_VersionType_index[i+1]]
|
||||
}
|
||||
@ -19,12 +19,15 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
)
|
||||
|
||||
func TestXLV2FormatData(t *testing.T) {
|
||||
@ -341,15 +344,17 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
|
||||
}
|
||||
}
|
||||
fi.TransitionStatus = tc.transitionStatus
|
||||
fi.ModTime = fi.ModTime.Add(time.Duration(i) * time.Second)
|
||||
failOnErr(i+1, xl.AddVersion(fi))
|
||||
fi.ExpireRestored = tc.expireRestored
|
||||
fileInfos = append(fileInfos, fi)
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
version := xl.Versions[i]
|
||||
if actual := xl.SharedDataDirCount(version.ObjectV2.VersionID, version.ObjectV2.DataDir); actual != tc.shares {
|
||||
t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2, tc.shares, actual)
|
||||
_, version, err := xl.findVersion(uuid.MustParse(tc.versionID))
|
||||
failOnErr(i+1, err)
|
||||
if got := xl.SharedDataDirCount(version.getVersionID(), version.ObjectV2.DataDir); got != tc.shares {
|
||||
t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2.VersionID, tc.shares, got)
|
||||
}
|
||||
}
|
||||
|
||||
@ -366,3 +371,110 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
|
||||
data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
dec, _ := zstd.NewReader(nil)
|
||||
data, err = dec.DecodeAll(data, nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
b.Run("legacy", func(b *testing.B) {
|
||||
var xl xlMetaV2
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
b.SetBytes(855) // number of versions...
|
||||
for i := 0; i < b.N; i++ {
|
||||
err = xl.Load(data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
b.Run("indexed", func(b *testing.B) {
|
||||
var xl xlMetaV2
|
||||
err = xl.Load(data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
data, err := xl.AppendTo(nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
b.SetBytes(855) // number of versions...
|
||||
for i := 0; i < b.N; i++ {
|
||||
err = xl.Load(data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func Test_xlMetaV2Shallow_Load(t *testing.T) {
|
||||
// Load Legacy
|
||||
data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dec, _ := zstd.NewReader(nil)
|
||||
data, err = dec.DecodeAll(data, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
test := func(t *testing.T, xl *xlMetaV2) {
|
||||
if len(xl.versions) != 855 {
|
||||
t.Errorf("want %d versions, got %d", 855, len(xl.versions))
|
||||
}
|
||||
xl.sortByModTime()
|
||||
if !sort.SliceIsSorted(xl.versions, func(i, j int) bool {
|
||||
return xl.versions[i].header.ModTime > xl.versions[j].header.ModTime
|
||||
}) {
|
||||
t.Errorf("Contents not sorted")
|
||||
}
|
||||
for i := range xl.versions {
|
||||
hdr := xl.versions[i].header
|
||||
ver, err := xl.getIdx(i)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
continue
|
||||
}
|
||||
gotHdr := ver.header()
|
||||
if hdr != gotHdr {
|
||||
t.Errorf("Header does not match, index: %+v != meta: %+v", hdr, gotHdr)
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Run("load-legacy", func(t *testing.T) {
|
||||
var xl xlMetaV2
|
||||
err = xl.Load(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
test(t, &xl)
|
||||
})
|
||||
t.Run("roundtrip", func(t *testing.T) {
|
||||
var xl xlMetaV2
|
||||
err = xl.Load(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err = xl.AppendTo(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = xlMetaV2{}
|
||||
err = xl.Load(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
test(t, &xl)
|
||||
})
|
||||
}
|
||||
|
||||
@ -21,10 +21,14 @@ import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
func TestIsXLMetaFormatValid(t *testing.T) {
|
||||
@ -317,3 +321,221 @@ func TestGetPartSizeFromIdx(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkXlMetaV2Shallow(b *testing.B) {
|
||||
fi := FileInfo{
|
||||
Volume: "volume",
|
||||
Name: "object-name",
|
||||
VersionID: "756100c6-b393-4981-928a-d49bbc164741",
|
||||
IsLatest: true,
|
||||
Deleted: false,
|
||||
TransitionStatus: "PENDING",
|
||||
DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef",
|
||||
XLV1: false,
|
||||
ModTime: time.Now(),
|
||||
Size: 1234456,
|
||||
Mode: 0,
|
||||
Metadata: map[string]string{
|
||||
xhttp.AmzRestore: "FAILED",
|
||||
xhttp.ContentMD5: mustGetUUID(),
|
||||
xhttp.AmzBucketReplicationStatus: "PENDING",
|
||||
xhttp.ContentType: "application/json",
|
||||
},
|
||||
Parts: []ObjectPartInfo{{
|
||||
Number: 1,
|
||||
Size: 1234345,
|
||||
ActualSize: 1234345,
|
||||
},
|
||||
{
|
||||
Number: 2,
|
||||
Size: 1234345,
|
||||
ActualSize: 1234345,
|
||||
},
|
||||
},
|
||||
Erasure: ErasureInfo{
|
||||
Algorithm: ReedSolomon.String(),
|
||||
DataBlocks: 4,
|
||||
ParityBlocks: 2,
|
||||
BlockSize: 10000,
|
||||
Index: 1,
|
||||
Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8},
|
||||
Checksums: []ChecksumInfo{{
|
||||
PartNumber: 1,
|
||||
Algorithm: HighwayHash256S,
|
||||
Hash: nil,
|
||||
},
|
||||
{
|
||||
PartNumber: 2,
|
||||
Algorithm: HighwayHash256S,
|
||||
Hash: nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, size := range []int{1, 10, 1000, 100_000} {
|
||||
b.Run(fmt.Sprint(size, "-versions"), func(b *testing.B) {
|
||||
var xl xlMetaV2
|
||||
ids := make([]string, size)
|
||||
for i := 0; i < size; i++ {
|
||||
fi.VersionID = mustGetUUID()
|
||||
fi.DataDir = mustGetUUID()
|
||||
ids[i] = fi.VersionID
|
||||
fi.ModTime = fi.ModTime.Add(-time.Second)
|
||||
xl.AddVersion(fi)
|
||||
}
|
||||
// Encode all. This is used for benchmarking.
|
||||
enc, err := xl.AppendTo(nil)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.Logf("Serialized size: %d bytes", len(enc))
|
||||
rng := rand.New(rand.NewSource(0))
|
||||
var dump = make([]byte, len(enc))
|
||||
b.Run("UpdateObjectVersion", func(b *testing.B) {
|
||||
b.SetBytes(int64(size))
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Load...
|
||||
xl = xlMetaV2{}
|
||||
err := xl.Load(enc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// Update modtime for resorting...
|
||||
fi.ModTime = fi.ModTime.Add(-time.Second)
|
||||
// Update a random version.
|
||||
fi.VersionID = ids[rng.Intn(size)]
|
||||
// Update...
|
||||
err = xl.UpdateObjectVersion(fi)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// Save...
|
||||
dump, err = xl.AppendTo(dump[:0])
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
b.Run("DeleteVersion", func(b *testing.B) {
|
||||
b.SetBytes(int64(size))
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Load...
|
||||
xl = xlMetaV2{}
|
||||
err := xl.Load(enc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// Update a random version.
|
||||
fi.VersionID = ids[rng.Intn(size)]
|
||||
// Delete...
|
||||
_, _, err = xl.DeleteVersion(fi)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// Save...
|
||||
dump, err = xl.AppendTo(dump[:0])
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
b.Run("AddVersion", func(b *testing.B) {
|
||||
b.SetBytes(int64(size))
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Load...
|
||||
xl = xlMetaV2{}
|
||||
err := xl.Load(enc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// Update modtime for resorting...
|
||||
fi.ModTime = fi.ModTime.Add(-time.Second)
|
||||
// Update a random version.
|
||||
fi.VersionID = mustGetUUID()
|
||||
// Add...
|
||||
err = xl.AddVersion(fi)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// Save...
|
||||
dump, err = xl.AppendTo(dump[:0])
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
b.Run("ToFileInfo", func(b *testing.B) {
|
||||
b.SetBytes(int64(size))
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Load...
|
||||
xl = xlMetaV2{}
|
||||
err := xl.Load(enc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// List...
|
||||
_, err = xl.ToFileInfo("volume", "path", ids[rng.Intn(size)])
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
b.Run("ListVersions", func(b *testing.B) {
|
||||
b.SetBytes(int64(size))
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Load...
|
||||
xl = xlMetaV2{}
|
||||
err := xl.Load(enc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
// List...
|
||||
_, err = xl.ListVersions("volume", "path")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
b.Run("ToFileInfoNew", func(b *testing.B) {
|
||||
b.SetBytes(int64(size))
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
buf, _ := isIndexedMetaV2(enc)
|
||||
if buf == nil {
|
||||
b.Fatal("buf == nil")
|
||||
}
|
||||
_, err = buf.ToFileInfo("volume", "path", ids[rng.Intn(size)])
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
b.Run("ListVersionsNew", func(b *testing.B) {
|
||||
b.SetBytes(int64(size))
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
buf, _ := isIndexedMetaV2(enc)
|
||||
if buf == nil {
|
||||
b.Fatal("buf == nil")
|
||||
}
|
||||
_, err = buf.ListVersions("volume", "path")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -75,7 +75,7 @@ func (j xlMetaV2Version) FreeVersion() bool {
|
||||
|
||||
// AddFreeVersion adds a free-version if needed for fi.VersionID version.
|
||||
// Free-version will be added if fi.VersionID has transitioned.
|
||||
func (z *xlMetaV2) AddFreeVersion(fi FileInfo) error {
|
||||
func (x *xlMetaV2) AddFreeVersion(fi FileInfo) error {
|
||||
var uv uuid.UUID
|
||||
var err error
|
||||
switch fi.VersionID {
|
||||
@ -87,19 +87,22 @@ func (z *xlMetaV2) AddFreeVersion(fi FileInfo) error {
|
||||
}
|
||||
}
|
||||
|
||||
for _, version := range z.Versions {
|
||||
switch version.Type {
|
||||
case ObjectType:
|
||||
if version.ObjectV2.VersionID == uv {
|
||||
// if uv has tiered content we add a
|
||||
// free-version to track it for asynchronous
|
||||
// deletion via scanner.
|
||||
if freeVersion, toFree := version.ObjectV2.InitFreeVersion(fi); toFree {
|
||||
z.Versions = append(z.Versions, freeVersion)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for i, version := range x.versions {
|
||||
if version.header.VersionID != uv || version.header.Type != ObjectType {
|
||||
continue
|
||||
}
|
||||
// if uv has tiered content we add a
|
||||
// free-version to track it for asynchronous
|
||||
// deletion via scanner.
|
||||
ver, err := x.getIdx(i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if freeVersion, toFree := ver.ObjectV2.InitFreeVersion(fi); toFree {
|
||||
return x.addVersion(freeVersion)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -24,8 +24,8 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
)
|
||||
|
||||
func (z xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
|
||||
fivs, _, err := z.ListVersions(volume, path)
|
||||
func (x xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
|
||||
fivs, err := x.ListVersions(volume, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -41,8 +41,21 @@ func (z xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
|
||||
}
|
||||
|
||||
func TestFreeVersion(t *testing.T) {
|
||||
fatalErr := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Add a version with tiered content, one with local content
|
||||
xl := xlMetaV2{}
|
||||
counter := 1
|
||||
report := func() {
|
||||
t.Helper()
|
||||
// t.Logf("versions (%d): len = %d", counter, len(xl.versions))
|
||||
counter++
|
||||
}
|
||||
fi := FileInfo{
|
||||
Volume: "volume",
|
||||
Name: "object-name",
|
||||
@ -77,16 +90,21 @@ func TestFreeVersion(t *testing.T) {
|
||||
SuccessorModTime: time.Time{},
|
||||
}
|
||||
// Add a version with local content
|
||||
xl.AddVersion(fi)
|
||||
fatalErr(xl.AddVersion(fi))
|
||||
report()
|
||||
|
||||
// Add null version with tiered content
|
||||
tierfi := fi
|
||||
tierfi.VersionID = ""
|
||||
xl.AddVersion(tierfi)
|
||||
fatalErr(xl.AddVersion(tierfi))
|
||||
report()
|
||||
tierfi.TransitionStatus = lifecycle.TransitionComplete
|
||||
tierfi.TransitionedObjName = mustGetUUID()
|
||||
tierfi.TransitionTier = "MINIOTIER-1"
|
||||
xl.DeleteVersion(tierfi)
|
||||
var err error
|
||||
_, _, err = xl.DeleteVersion(tierfi)
|
||||
fatalErr(err)
|
||||
report()
|
||||
|
||||
fvIDs := []string{
|
||||
"00000000-0000-0000-0000-0000000000f1",
|
||||
@ -95,15 +113,20 @@ func TestFreeVersion(t *testing.T) {
|
||||
// Simulate overwrite of null version
|
||||
newtierfi := tierfi
|
||||
newtierfi.SetTierFreeVersionID(fvIDs[0])
|
||||
xl.AddFreeVersion(newtierfi)
|
||||
xl.AddVersion(newtierfi)
|
||||
fatalErr(xl.AddFreeVersion(newtierfi))
|
||||
report()
|
||||
fatalErr(xl.AddVersion(newtierfi))
|
||||
report()
|
||||
|
||||
// Simulate removal of null version
|
||||
newtierfi.TransitionTier = ""
|
||||
newtierfi.TransitionedObjName = ""
|
||||
newtierfi.TransitionStatus = ""
|
||||
newtierfi.SetTierFreeVersionID(fvIDs[1])
|
||||
xl.DeleteVersion(newtierfi)
|
||||
report()
|
||||
_, _, err = xl.DeleteVersion(newtierfi)
|
||||
report()
|
||||
fatalErr(err)
|
||||
|
||||
// Check number of free-versions
|
||||
freeVersions, err := xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
|
||||
@ -118,8 +141,10 @@ func TestFreeVersion(t *testing.T) {
|
||||
freefi := newtierfi
|
||||
for _, fvID := range fvIDs {
|
||||
freefi.VersionID = fvID
|
||||
xl.DeleteVersion(freefi)
|
||||
_, _, err = xl.DeleteVersion(freefi)
|
||||
fatalErr(err)
|
||||
}
|
||||
report()
|
||||
|
||||
// Check number of free-versions
|
||||
freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
|
||||
@ -129,11 +154,13 @@ func TestFreeVersion(t *testing.T) {
|
||||
if len(freeVersions) != 0 {
|
||||
t.Fatalf("Expected zero free version but got %d", len(freeVersions))
|
||||
}
|
||||
report()
|
||||
|
||||
// Adding a free version to a version with no tiered content.
|
||||
newfi := fi
|
||||
newfi.SetTierFreeVersionID("00000000-0000-0000-0000-0000000000f3")
|
||||
xl.AddFreeVersion(newfi) // this shouldn't add a free-version
|
||||
fatalErr(xl.AddFreeVersion(newfi)) // this shouldn't add a free-version
|
||||
report()
|
||||
|
||||
// Check number of free-versions
|
||||
freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
|
||||
|
||||
408
cmd/xl-storage-meta-inline.go
Normal file
408
cmd/xl-storage-meta-inline.go
Normal file
@ -0,0 +1,408 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
// xlMetaInlineData is serialized data in [string][]byte pairs.
|
||||
type xlMetaInlineData []byte
|
||||
|
||||
// xlMetaInlineDataVer indicates the version of the inline data structure.
|
||||
const xlMetaInlineDataVer = 1
|
||||
|
||||
// versionOK returns whether the version is ok.
|
||||
func (x xlMetaInlineData) versionOK() bool {
|
||||
if len(x) == 0 {
|
||||
return true
|
||||
}
|
||||
return x[0] > 0 && x[0] <= xlMetaInlineDataVer
|
||||
}
|
||||
|
||||
// afterVersion returns the payload after the version, if any.
|
||||
func (x xlMetaInlineData) afterVersion() []byte {
|
||||
if len(x) == 0 {
|
||||
return x
|
||||
}
|
||||
return x[1:]
|
||||
}
|
||||
|
||||
// find the data with key s.
|
||||
// Returns nil if not for or an error occurs.
|
||||
func (x xlMetaInlineData) find(key string) []byte {
|
||||
if len(x) == 0 || !x.versionOK() {
|
||||
return nil
|
||||
}
|
||||
sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
|
||||
if err != nil || sz == 0 {
|
||||
return nil
|
||||
}
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var found []byte
|
||||
found, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil || sz == 0 {
|
||||
return nil
|
||||
}
|
||||
if string(found) == key {
|
||||
val, _, _ := msgp.ReadBytesZC(buf)
|
||||
return val
|
||||
}
|
||||
// Skip it
|
||||
_, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validate checks if the data is valid.
|
||||
// It does not check integrity of the stored data.
|
||||
func (x xlMetaInlineData) validate() error {
|
||||
if len(x) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !x.versionOK() {
|
||||
return fmt.Errorf("xlMetaInlineData: unknown version 0x%x", x[0])
|
||||
}
|
||||
|
||||
sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
|
||||
if err != nil {
|
||||
return fmt.Errorf("xlMetaInlineData: %w", err)
|
||||
}
|
||||
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var key []byte
|
||||
key, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("xlMetaInlineData: %w", err)
|
||||
}
|
||||
if len(key) == 0 {
|
||||
return fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
|
||||
}
|
||||
_, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("xlMetaInlineData: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// repair will copy all seemingly valid data entries from a corrupted set.
|
||||
// This does not ensure that data is correct, but will allow all operations to complete.
|
||||
func (x *xlMetaInlineData) repair() {
|
||||
data := *x
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if !data.versionOK() {
|
||||
*x = nil
|
||||
return
|
||||
}
|
||||
|
||||
sz, buf, err := msgp.ReadMapHeaderBytes(data.afterVersion())
|
||||
if err != nil {
|
||||
*x = nil
|
||||
return
|
||||
}
|
||||
|
||||
// Remove all current data
|
||||
keys := make([][]byte, 0, sz)
|
||||
vals := make([][]byte, 0, sz)
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var key, val []byte
|
||||
key, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if len(key) == 0 {
|
||||
break
|
||||
}
|
||||
val, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
keys = append(keys, key)
|
||||
vals = append(vals, val)
|
||||
}
|
||||
x.serialize(-1, keys, vals)
|
||||
}
|
||||
|
||||
// validate checks if the data is valid.
|
||||
// It does not check integrity of the stored data.
|
||||
func (x xlMetaInlineData) list() ([]string, error) {
|
||||
if len(x) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if !x.versionOK() {
|
||||
return nil, errors.New("xlMetaInlineData: unknown version")
|
||||
}
|
||||
|
||||
sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, sz)
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var key []byte
|
||||
key, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
return keys, err
|
||||
}
|
||||
if len(key) == 0 {
|
||||
return keys, fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
|
||||
}
|
||||
keys = append(keys, string(key))
|
||||
// Skip data...
|
||||
_, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
return keys, err
|
||||
}
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// serialize will serialize the provided keys and values.
|
||||
// The function will panic if keys/value slices aren't of equal length.
|
||||
// Payload size can give an indication of expected payload size.
|
||||
// If plSize is <= 0 it will be calculated.
|
||||
func (x *xlMetaInlineData) serialize(plSize int, keys [][]byte, vals [][]byte) {
|
||||
if len(keys) != len(vals) {
|
||||
panic(fmt.Errorf("xlMetaInlineData.serialize: keys/value number mismatch"))
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
*x = nil
|
||||
return
|
||||
}
|
||||
if plSize <= 0 {
|
||||
plSize = 1 + msgp.MapHeaderSize
|
||||
for i := range keys {
|
||||
plSize += len(keys[i]) + len(vals[i]) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
|
||||
}
|
||||
}
|
||||
payload := make([]byte, 1, plSize)
|
||||
payload[0] = xlMetaInlineDataVer
|
||||
payload = msgp.AppendMapHeader(payload, uint32(len(keys)))
|
||||
for i := range keys {
|
||||
payload = msgp.AppendStringFromBytes(payload, keys[i])
|
||||
payload = msgp.AppendBytes(payload, vals[i])
|
||||
}
|
||||
*x = payload
|
||||
}
|
||||
|
||||
// entries returns the number of entries in the data.
|
||||
func (x xlMetaInlineData) entries() int {
|
||||
if len(x) == 0 || !x.versionOK() {
|
||||
return 0
|
||||
}
|
||||
sz, _, _ := msgp.ReadMapHeaderBytes(x.afterVersion())
|
||||
return int(sz)
|
||||
}
|
||||
|
||||
// replace will add or replace a key/value pair.
|
||||
func (x *xlMetaInlineData) replace(key string, value []byte) {
|
||||
in := x.afterVersion()
|
||||
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
|
||||
keys := make([][]byte, 0, sz+1)
|
||||
vals := make([][]byte, 0, sz+1)
|
||||
|
||||
// Version plus header...
|
||||
plSize := 1 + msgp.MapHeaderSize
|
||||
replaced := false
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var found, foundVal []byte
|
||||
var err error
|
||||
found, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
foundVal, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
plSize += len(found) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
|
||||
keys = append(keys, found)
|
||||
if string(found) == key {
|
||||
vals = append(vals, value)
|
||||
plSize += len(value)
|
||||
replaced = true
|
||||
} else {
|
||||
vals = append(vals, foundVal)
|
||||
plSize += len(foundVal)
|
||||
}
|
||||
}
|
||||
|
||||
// Add one more.
|
||||
if !replaced {
|
||||
keys = append(keys, []byte(key))
|
||||
vals = append(vals, value)
|
||||
plSize += len(key) + len(value) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
|
||||
}
|
||||
|
||||
// Reserialize...
|
||||
x.serialize(plSize, keys, vals)
|
||||
}
|
||||
|
||||
// rename will rename a key.
|
||||
// Returns whether the key was found.
|
||||
func (x *xlMetaInlineData) rename(oldKey, newKey string) bool {
|
||||
in := x.afterVersion()
|
||||
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
|
||||
keys := make([][]byte, 0, sz)
|
||||
vals := make([][]byte, 0, sz)
|
||||
|
||||
// Version plus header...
|
||||
plSize := 1 + msgp.MapHeaderSize
|
||||
found := false
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var foundKey, foundVal []byte
|
||||
var err error
|
||||
foundKey, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
foundVal, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
plSize += len(foundVal) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
|
||||
vals = append(vals, foundVal)
|
||||
if string(foundKey) != oldKey {
|
||||
keys = append(keys, foundKey)
|
||||
plSize += len(foundKey)
|
||||
} else {
|
||||
keys = append(keys, []byte(newKey))
|
||||
plSize += len(newKey)
|
||||
found = true
|
||||
}
|
||||
}
|
||||
// If not found, just return.
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
|
||||
// Reserialize...
|
||||
x.serialize(plSize, keys, vals)
|
||||
return true
|
||||
}
|
||||
|
||||
// remove will remove one or more keys.
|
||||
// Returns true if any key was found.
|
||||
func (x *xlMetaInlineData) remove(keys ...string) bool {
|
||||
in := x.afterVersion()
|
||||
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
|
||||
newKeys := make([][]byte, 0, sz)
|
||||
newVals := make([][]byte, 0, sz)
|
||||
var removeKey func(s []byte) bool
|
||||
|
||||
// Copy if big number of compares...
|
||||
if len(keys) > 5 && sz > 5 {
|
||||
mKeys := make(map[string]struct{}, len(keys))
|
||||
for _, key := range keys {
|
||||
mKeys[key] = struct{}{}
|
||||
}
|
||||
removeKey = func(s []byte) bool {
|
||||
_, ok := mKeys[string(s)]
|
||||
return ok
|
||||
}
|
||||
} else {
|
||||
removeKey = func(s []byte) bool {
|
||||
for _, key := range keys {
|
||||
if key == string(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Version plus header...
|
||||
plSize := 1 + msgp.MapHeaderSize
|
||||
found := false
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var foundKey, foundVal []byte
|
||||
var err error
|
||||
foundKey, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
foundVal, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if !removeKey(foundKey) {
|
||||
plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal)
|
||||
newKeys = append(newKeys, foundKey)
|
||||
newVals = append(newVals, foundVal)
|
||||
} else {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
// If not found, just return.
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
// If none left...
|
||||
if len(newKeys) == 0 {
|
||||
*x = nil
|
||||
return true
|
||||
}
|
||||
|
||||
// Reserialize...
|
||||
x.serialize(plSize, newKeys, newVals)
|
||||
return true
|
||||
}
|
||||
|
||||
// xlMetaV2TrimData will trim any data from the metadata without unmarshalling it.
|
||||
// If any error occurs the unmodified data is returned.
|
||||
func xlMetaV2TrimData(buf []byte) []byte {
|
||||
metaBuf, min, maj, err := checkXL2V1(buf)
|
||||
if err != nil {
|
||||
return buf
|
||||
}
|
||||
if maj == 1 && min < 1 {
|
||||
// First version to carry data.
|
||||
return buf
|
||||
}
|
||||
// Skip header
|
||||
_, metaBuf, err = msgp.ReadBytesZC(metaBuf)
|
||||
if err != nil {
|
||||
logger.LogIf(GlobalContext, err)
|
||||
return buf
|
||||
}
|
||||
// Skip CRC
|
||||
if maj > 1 || min >= 2 {
|
||||
_, metaBuf, err = msgp.ReadUint32Bytes(metaBuf)
|
||||
logger.LogIf(GlobalContext, err)
|
||||
}
|
||||
// = input - current pos
|
||||
ends := len(buf) - len(metaBuf)
|
||||
if ends > len(buf) {
|
||||
return buf
|
||||
}
|
||||
|
||||
return buf[:ends]
|
||||
}
|
||||
@ -978,7 +978,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
}
|
||||
|
||||
var xlMeta xlMetaV2
|
||||
if err = xlMeta.Load(buf); err != nil {
|
||||
if err := xlMeta.Load(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -1044,6 +1044,7 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer metaDataPoolPut(buf)
|
||||
|
||||
if !isXL2V1Format(buf) {
|
||||
return errFileVersionNotFound
|
||||
@ -1059,12 +1060,13 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi
|
||||
return err
|
||||
}
|
||||
|
||||
buf, err = xlMeta.AppendTo(nil)
|
||||
wbuf, err := xlMeta.AppendTo(metaDataPoolGet())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer metaDataPoolPut(wbuf)
|
||||
|
||||
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf)
|
||||
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), wbuf)
|
||||
}
|
||||
|
||||
// WriteMetadata - writes FileInfo metadata for path at `xl.meta`
|
||||
|
||||
@ -114,6 +114,48 @@ FLAGS:
|
||||
return nil, err
|
||||
}
|
||||
data = b
|
||||
case 3:
|
||||
v, b, err := msgp.ReadBytesZC(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil {
|
||||
// Read metadata CRC (added in v2, ignore if not found)
|
||||
b = nbuf
|
||||
}
|
||||
|
||||
nVers, v, err := decodeXLHeaders(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var versions = struct {
|
||||
Versions []json.RawMessage
|
||||
Headers []json.RawMessage
|
||||
}{
|
||||
Versions: make([]json.RawMessage, nVers),
|
||||
Headers: make([]json.RawMessage, nVers),
|
||||
}
|
||||
err = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error {
|
||||
var buf bytes.Buffer
|
||||
if _, err := msgp.UnmarshalAsJSON(&buf, hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
versions.Headers[idx] = buf.Bytes()
|
||||
buf = bytes.Buffer{}
|
||||
if _, err := msgp.UnmarshalAsJSON(&buf, meta); err != nil {
|
||||
return err
|
||||
}
|
||||
versions.Versions[idx] = buf.Bytes()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
enc := json.NewEncoder(buf)
|
||||
if err := enc.Encode(versions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data = b
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown metadata version %d", minor)
|
||||
}
|
||||
@ -416,3 +458,54 @@ func (x xlMetaInlineData) files(fn func(name string, data []byte)) error {
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
const (
|
||||
xlHeaderVersion = 2
|
||||
xlMetaVersion = 1
|
||||
)
|
||||
|
||||
func decodeXLHeaders(buf []byte) (versions int, b []byte, err error) {
|
||||
hdrVer, buf, err := msgp.ReadUintBytes(buf)
|
||||
if err != nil {
|
||||
return 0, buf, err
|
||||
}
|
||||
metaVer, buf, err := msgp.ReadUintBytes(buf)
|
||||
if err != nil {
|
||||
return 0, buf, err
|
||||
}
|
||||
if hdrVer > xlHeaderVersion {
|
||||
return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", metaVer)
|
||||
}
|
||||
if metaVer > xlMetaVersion {
|
||||
return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", metaVer)
|
||||
}
|
||||
versions, buf, err = msgp.ReadIntBytes(buf)
|
||||
if err != nil {
|
||||
return 0, buf, err
|
||||
}
|
||||
if versions < 0 {
|
||||
return 0, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", versions)
|
||||
}
|
||||
return versions, buf, nil
|
||||
}
|
||||
|
||||
// decodeVersions will decode a number of versions from a buffer
|
||||
// and perform a callback for each version in order, newest first.
|
||||
// Any non-nil error is returned.
|
||||
func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte) error) (err error) {
|
||||
var tHdr, tMeta []byte // Zero copy bytes
|
||||
for i := 0; i < versions; i++ {
|
||||
tHdr, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tMeta, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = fn(i, tHdr, tMeta); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
1
go.mod
1
go.mod
@ -81,6 +81,7 @@ require (
|
||||
github.com/valyala/bytebufferpool v1.0.0
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
|
||||
github.com/yargevad/filepathx v1.0.0
|
||||
github.com/zeebo/xxh3 v1.0.0
|
||||
go.etcd.io/etcd/api/v3 v3.5.0
|
||||
go.etcd.io/etcd/client/v3 v3.5.0
|
||||
go.uber.org/atomic v1.9.0
|
||||
|
||||
2
go.sum
2
go.sum
@ -1527,6 +1527,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/zeebo/xxh3 v1.0.0 h1:6eLPZCVXpsGnhv8RiWBEJs5kenm2W1CMwon19/l8ODc=
|
||||
github.com/zeebo/xxh3 v1.0.0/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user