MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-10-30 00:05:02 -04:00
Code Issues Packages Projects Releases Wiki Activity

Improve performance on multiple versions (#13573)

Browse Source 
Existing:

```go
type xlMetaV2 struct {
    Versions []xlMetaV2Version `json:"Versions" msg:"Versions"`
}
```

Serialized as regular MessagePack.

```go
//msgp:tuple xlMetaV2VersionHeader
type xlMetaV2VersionHeader struct {
	VersionID [16]byte
	ModTime   int64
	Type      VersionType
	Flags     xlFlags
}
```

Serialize as streaming MessagePack, format:

```
int(headerVersion)
int(xlmetaVersion)
int(nVersions)
for each version {
    binary blob, xlMetaV2VersionHeader, serialized
    binary blob, xlMetaV2Version, serialized.
}
```

xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct 
can easily be reused and does not contain pointers, so efficient as a 
slice (single allocation)

This allows quickly parsing everything as slices of bytes (no copy).

Versions are always *saved* sorted by modTime, newest *first*. 
No more need to sort on load.

* Allows checking if a version exists.
* Allows reading single version without unmarshal all.
* Allows reading latest version of type without unmarshal all.
* Allows reading latest version without unmarshal of all.
* Allows checking if the latest is deleteMarker by reading first entry.
* Allows adding/updating/deleting a version with only header deserialization.
* Reduces allocations on conversion to FileInfo(s).
This commit is contained in:
Klaus Post 2021-11-18 12:15:22 -08:00 committed by GitHub
parent 7152915318
commit faf013ec84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 3282 additions and 1463 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -20,7 +20,7 @@ help: ## print this help
getdeps: ## fetch necessary dependencies
@mkdir -p ${GOPATH}/bin
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.43.0
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@latest
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
crosscompile: ## cross compile minio

2
cmd/bucket-replication.go
View File

@ -724,7 +724,7 @@ const (
// matches k1 with all keys, returns 'true' if one of them matches
func equals(k1 string, keys ...string) bool {
for _, k2 := range keys {
if strings.EqualFold(strings.ToLower(k1), strings.ToLower(k2)) {
if strings.EqualFold(k1, k2) {
return true
}
}

7
cmd/erasure-metadata.go
View File

@ -476,9 +476,7 @@ func GetInternalReplicationState(m map[string][]byte) ReplicationState {
// getInternalReplicationState fetches internal replication state from the map m
func getInternalReplicationState(m map[string]string) ReplicationState {
d := ReplicationState{
ResetStatusesMap: make(map[string]string),
}
d := ReplicationState{}
for k, v := range m {
switch {
case equals(k, ReservedMetadataPrefixLower+ReplicationTimestamp):
@ -497,6 +495,9 @@ func getInternalReplicationState(m map[string]string) ReplicationState {
d.PurgeTargets = versionPurgeStatusesMap(v)
case strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset):
arn := strings.TrimPrefix(k, fmt.Sprintf("%s-", ReservedMetadataPrefixLower+ReplicationReset))
if d.ResetStatusesMap == nil {
d.ResetStatusesMap = make(map[string]string, 1)
}
d.ResetStatusesMap[arn] = v
}
}

8
cmd/metacache-entries.go
View File

@ -148,11 +148,15 @@ func (e *metaCacheEntry) isLatestDeletemarker() bool {
if !isXL2V1Format(e.metadata) {
return false
}
if meta, _ := isIndexedMetaV2(e.metadata); meta != nil {
return meta.IsLatestDeleteMarker()
}
// Fall back...
var xlMeta xlMetaV2
if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.Versions) == 0 {
if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.versions) == 0 {
return true
}
return xlMeta.Versions[len(xlMeta.Versions)-1].Type == DeleteType
return xlMeta.versions[0].header.Type == DeleteType
}
// fileInfo returns the decoded metadata.

BIN
cmd/testdata/xl.meta-v1.2.zst vendored Normal file
View File

Binary file not shown.

73
cmd/xl-storage-format-utils.go
View File

@ -19,27 +19,12 @@ package cmd
import (
"fmt"
"sort"
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio/internal/logger"
"github.com/zeebo/xxh3"
)
// versionsSorter sorts FileInfo slices by version.
type versionsSorter []FileInfo
func (v versionsSorter) sort() {
sort.Slice(v, func(i, j int) bool {
if v[i].IsLatest {
return true
}
if v[j].IsLatest {
return false
}
return v[i].ModTime.After(v[j].ModTime)
})
}
func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) {
fivs, err := getAllFileInfoVersions(xlMetaBuf, volume, path)
if err != nil {
@ -54,24 +39,35 @@ func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersion
}
}
fivs.Versions = fivs.Versions[:n]
// Update numversions
for i := range fivs.Versions {
fivs.Versions[i].NumVersions = n
}
return fivs, nil
}
func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) {
if isXL2V1Format(xlMetaBuf) {
var versions []FileInfo
var err error
if buf, _ := isIndexedMetaV2(xlMetaBuf); buf != nil {
versions, err = buf.ListVersions(volume, path)
} else {
var xlMeta xlMetaV2
if err := xlMeta.Load(xlMetaBuf); err != nil {
return FileInfoVersions{}, err
}
versions, latestModTime, err := xlMeta.ListVersions(volume, path)
if err != nil {
versions, err = xlMeta.ListVersions(volume, path)
}
if err != nil || len(versions) == 0 {
return FileInfoVersions{}, err
}
return FileInfoVersions{
Volume: volume,
Name: path,
Versions: versions,
LatestModTime: latestModTime,
LatestModTime: versions[0].ModTime,
}, nil
}
@ -98,11 +94,20 @@ func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVers
func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (FileInfo, error) {
if isXL2V1Format(xlMetaBuf) {
var fi FileInfo
var err error
var inData xlMetaInlineData
if buf, data := isIndexedMetaV2(xlMetaBuf); buf != nil {
inData = data
fi, err = buf.ToFileInfo(volume, path, versionID)
} else {
var xlMeta xlMetaV2
if err := xlMeta.Load(xlMetaBuf); err != nil {
return FileInfo{}, err
}
fi, err := xlMeta.ToFileInfo(volume, path, versionID)
inData = xlMeta.data
fi, err = xlMeta.ToFileInfo(volume, path, versionID)
}
if !data || err != nil {
return fi, err
}
@ -110,12 +115,12 @@ func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (F
if versionID == "" {
versionID = nullVersionID
}
fi.Data = xlMeta.data.find(versionID)
fi.Data = inData.find(versionID)
if len(fi.Data) == 0 {
// PR #11758 used DataDir, preserve it
// for users who might have used master
// branch
fi.Data = xlMeta.data.find(fi.DataDir)
fi.Data = inData.find(fi.DataDir)
}
return fi, nil
}
@ -149,3 +154,27 @@ func getXLDiskLoc(diskID string) (poolIdx, setIdx, diskIdx int) {
}
return -1, -1, -1
}
// hashDeterministicString will return a deterministic hash for the map values.
// Trivial collisions are avoided, but this is by no means a strong hash.
func hashDeterministicString(m map[string]string) uint64 {
// Seed (random)
var crc = uint64(0xc2b40bbac11a7295)
// Xor each value to make order independent
for k, v := range m {
// Separate key and value with an individual xor with a random number.
// Add values of each, so they cannot be trivially collided.
crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.HashString(v) ^ 0x8da4c8da66194257)
}
return crc
}
// hashDeterministicBytes will return a deterministic (weak) hash for the map values.
// Trivial collisions are avoided, but this is by no means a strong hash.
func hashDeterministicBytes(m map[string][]byte) uint64 {
var crc = uint64(0x1bbc7e1dde654743)
for k, v := range m {
crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.Hash(v) ^ 0x8da4c8da66194257)
}
return crc
}

93
cmd/xl-storage-format-utils_test.go Normal file
View File

@ -0,0 +1,93 @@
package cmd
import (
"testing"
xhttp "github.com/minio/minio/internal/http"
)
func Test_hashDeterministicString(t *testing.T) {
tests := []struct {
name string
arg map[string]string
}{
{
name: "zero",
arg: map[string]string{},
},
{
name: "nil",
arg: nil,
},
{
name: "one",
arg: map[string]string{"key": "value"},
},
{
name: "several",
arg: map[string]string{
xhttp.AmzRestore: "FAILED",
xhttp.ContentMD5: mustGetUUID(),
xhttp.AmzBucketReplicationStatus: "PENDING",
xhttp.ContentType: "application/json",
},
},
{
name: "someempty",
arg: map[string]string{
xhttp.AmzRestore: "",
xhttp.ContentMD5: mustGetUUID(),
xhttp.AmzBucketReplicationStatus: "",
xhttp.ContentType: "application/json",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
const n = 100
want := hashDeterministicString(tt.arg)
m := tt.arg
for i := 0; i < n; i++ {
if got := hashDeterministicString(m); got != want {
t.Errorf("hashDeterministicString() = %v, want %v", got, want)
}
}
// Check casual collisions
if m == nil {
m = make(map[string]string)
}
m["12312312"] = ""
if got := hashDeterministicString(m); got == want {
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
}
want = hashDeterministicString(m)
delete(m, "12312312")
m["another"] = ""
if got := hashDeterministicString(m); got == want {
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
}
want = hashDeterministicString(m)
m["another"] = "hashDeterministicString"
if got := hashDeterministicString(m); got == want {
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
}
want = hashDeterministicString(m)
m["another"] = "hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString"
if got := hashDeterministicString(m); got == want {
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
}
// Flip key/value
want = hashDeterministicString(m)
delete(m, "another")
m["hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString"] = "another"
if got := hashDeterministicString(m); got == want {
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
}
})
}
}

23
cmd/xl-storage-format-v1.go
View File

@ -18,11 +18,13 @@
package cmd
import (
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"time"
"github.com/cespare/xxhash/v2"
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio/internal/logger"
)
@ -205,6 +207,27 @@ func (m *xlMetaV1Object) ToFileInfo(volume, path string) (FileInfo, error) {
return fi, nil
}
// Signature will return a signature that is expected to be the same across all disks.
func (m *xlMetaV1Object) Signature() [4]byte {
// Shallow copy
c := *m
// Zero unimportant fields
c.Erasure.Index = 0
c.Minio.Release = ""
crc := hashDeterministicString(c.Meta)
c.Meta = nil
if bts, err := c.MarshalMsg(metaDataPoolGet()); err == nil {
crc ^= xxhash.Sum64(bts)
metaDataPoolPut(bts)
}
// Combine upper and lower part
var tmp [4]byte
binary.LittleEndian.PutUint32(tmp[:], uint32(crc^(crc>>32)))
return tmp
}
// XL metadata constants.
const (
// XL meta version.

89
cmd/xl-storage-format-v2-legacy.go Normal file
View File

@ -0,0 +1,89 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"github.com/tinylib/msgp/msgp"
)
// unmarshalV unmarshals with a specific header version.
func (x *xlMetaV2VersionHeader) unmarshalV(v uint8, bts []byte) (o []byte, err error) {
switch v {
case 1:
return x.unmarshalV1(bts)
case xlHeaderVersion:
return x.UnmarshalMsg(bts)
}
return bts, fmt.Errorf("unknown xlHeaderVersion: %d", v)
}
// unmarshalV1 decodes version 1, never released.
func (x *xlMetaV2VersionHeader) unmarshalV1(bts []byte) (o []byte, err error) {
var zb0001 uint32
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != 4 {
err = msgp.ArrayError{Wanted: 4, Got: zb0001}
return
}
bts, err = msgp.ReadExactBytes(bts, (x.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
x.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
{
var zb0002 uint8
zb0002, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
x.Type = VersionType(zb0002)
}
{
var zb0003 uint8
zb0003, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
x.Flags = xlFlags(zb0003)
}
o = bts
return
}
// unmarshalV unmarshals with a specific metadata version.
func (j *xlMetaV2Version) unmarshalV(v uint8, bts []byte) (o []byte, err error) {
switch v {
// We accept un-set as latest version.
case 0, xlMetaVersion:
return j.UnmarshalMsg(bts)
}
return bts, fmt.Errorf("unknown xlMetaVersion: %d", v)
}

2376
cmd/xl-storage-format-v2.go
View File

File diff suppressed because it is too large Load Diff

578
cmd/xl-storage-format-v2_gen.go
View File

@ -163,7 +163,111 @@ func (z VersionType) Msgsize() (s int) {
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaV2) DecodeMsg(dc *msgp.Reader) (err error) {
func (z *xlFlags) DecodeMsg(dc *msgp.Reader) (err error) {
{
var zb0001 uint8
zb0001, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = xlFlags(zb0001)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z xlFlags) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteUint8(uint8(z))
if err != nil {
err = msgp.WrapError(err)
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z xlFlags) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendUint8(o, uint8(z))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlFlags) UnmarshalMsg(bts []byte) (o []byte, err error) {
{
var zb0001 uint8
zb0001, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = xlFlags(zb0001)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z xlFlags) Msgsize() (s int) {
s = msgp.Uint8Size
return
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaBuf) DecodeMsg(dc *msgp.Reader) (err error) {
{
var zb0001 []byte
zb0001, err = dc.ReadBytes([]byte((*z)))
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = xlMetaBuf(zb0001)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z xlMetaBuf) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteBytes([]byte(z))
if err != nil {
err = msgp.WrapError(err)
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z xlMetaBuf) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendBytes(o, []byte(z))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaBuf) UnmarshalMsg(bts []byte) (o []byte, err error) {
{
var zb0001 []byte
zb0001, bts, err = msgp.ReadBytesBytes(bts, []byte((*z)))
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = xlMetaBuf(zb0001)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z xlMetaBuf) Msgsize() (s int) {
s = msgp.BytesPrefixSize + len([]byte(z))
return
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaDataDirDecoder) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zb0001 uint32
@ -180,24 +284,48 @@ func (z *xlMetaV2) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
switch msgp.UnsafeString(field) {
case "Versions":
var zb0002 uint32
zb0002, err = dc.ReadArrayHeader()
case "V2Obj":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "Versions")
err = msgp.WrapError(err, "ObjectV2")
return
}
if cap(z.Versions) >= int(zb0002) {
z.Versions = (z.Versions)[:zb0002]
z.ObjectV2 = nil
} else {
z.Versions = make([]xlMetaV2Version, zb0002)
if z.ObjectV2 == nil {
z.ObjectV2 = new(struct {
DataDir [16]byte `msg:"DDir"`
})
}
for za0001 := range z.Versions {
err = z.Versions[za0001].DecodeMsg(dc)
var zb0002 uint32
zb0002, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, "Versions", za0001)
err = msgp.WrapError(err, "ObjectV2")
return
}
for zb0002 > 0 {
zb0002--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
switch msgp.UnsafeString(field) {
case "DDir":
err = dc.ReadExactBytes((z.ObjectV2.DataDir)[:])
if err != nil {
err = msgp.WrapError(err, "ObjectV2", "DataDir")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
}
}
}
default:
err = dc.Skip()
@ -211,47 +339,82 @@ func (z *xlMetaV2) DecodeMsg(dc *msgp.Reader) (err error) {
}
// EncodeMsg implements msgp.Encodable
func (z *xlMetaV2) EncodeMsg(en *msgp.Writer) (err error) {
func (z *xlMetaDataDirDecoder) EncodeMsg(en *msgp.Writer) (err error) {
// omitempty: check for empty values
zb0001Len := uint32(1)
var zb0001Mask uint8 /* 1 bits */
if z.ObjectV2 == nil {
zb0001Len--
zb0001Mask |= 0x1
}
// variable map header, size zb0001Len
err = en.Append(0x80 | uint8(zb0001Len))
if err != nil {
return
}
if zb0001Len == 0 {
return
}
if (zb0001Mask & 0x1) == 0 { // if not empty
// write "V2Obj"
err = en.Append(0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a)
if err != nil {
return
}
if z.ObjectV2 == nil {
err = en.WriteNil()
if err != nil {
return
}
} else {
// map header, size 1
// write "Versions"
err = en.Append(0x81, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
// write "DDir"
err = en.Append(0x81, 0xa4, 0x44, 0x44, 0x69, 0x72)
if err != nil {
return
}
err = en.WriteArrayHeader(uint32(len(z.Versions)))
err = en.WriteBytes((z.ObjectV2.DataDir)[:])
if err != nil {
err = msgp.WrapError(err, "Versions")
err = msgp.WrapError(err, "ObjectV2", "DataDir")
return
}
for za0001 := range z.Versions {
err = z.Versions[za0001].EncodeMsg(en)
if err != nil {
err = msgp.WrapError(err, "Versions", za0001)
return
}
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *xlMetaV2) MarshalMsg(b []byte) (o []byte, err error) {
func (z *xlMetaDataDirDecoder) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 1
// string "Versions"
o = append(o, 0x81, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
o = msgp.AppendArrayHeader(o, uint32(len(z.Versions)))
for za0001 := range z.Versions {
o, err = z.Versions[za0001].MarshalMsg(o)
if err != nil {
err = msgp.WrapError(err, "Versions", za0001)
// omitempty: check for empty values
zb0001Len := uint32(1)
var zb0001Mask uint8 /* 1 bits */
if z.ObjectV2 == nil {
zb0001Len--
zb0001Mask |= 0x1
}
// variable map header, size zb0001Len
o = append(o, 0x80|uint8(zb0001Len))
if zb0001Len == 0 {
return
}
if (zb0001Mask & 0x1) == 0 { // if not empty
// string "V2Obj"
o = append(o, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a)
if z.ObjectV2 == nil {
o = msgp.AppendNil(o)
} else {
// map header, size 1
// string "DDir"
o = append(o, 0x81, 0xa4, 0x44, 0x44, 0x69, 0x72)
o = msgp.AppendBytes(o, (z.ObjectV2.DataDir)[:])
}
}
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
func (z *xlMetaDataDirDecoder) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
var zb0001 uint32
@ -268,24 +431,47 @@ func (z *xlMetaV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
switch msgp.UnsafeString(field) {
case "Versions":
var zb0002 uint32
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
case "V2Obj":
if msgp.IsNil(bts) {
bts, err = msgp.ReadNilBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Versions")
return
}
if cap(z.Versions) >= int(zb0002) {
z.Versions = (z.Versions)[:zb0002]
z.ObjectV2 = nil
} else {
z.Versions = make([]xlMetaV2Version, zb0002)
if z.ObjectV2 == nil {
z.ObjectV2 = new(struct {
DataDir [16]byte `msg:"DDir"`
})
}
for za0001 := range z.Versions {
bts, err = z.Versions[za0001].UnmarshalMsg(bts)
var zb0002 uint32
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Versions", za0001)
err = msgp.WrapError(err, "ObjectV2")
return
}
for zb0002 > 0 {
zb0002--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
switch msgp.UnsafeString(field) {
case "DDir":
bts, err = msgp.ReadExactBytes(bts, (z.ObjectV2.DataDir)[:])
if err != nil {
err = msgp.WrapError(err, "ObjectV2", "DataDir")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
}
}
}
default:
bts, err = msgp.Skip(bts)
@ -300,10 +486,12 @@ func (z *xlMetaV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *xlMetaV2) Msgsize() (s int) {
s = 1 + 9 + msgp.ArrayHeaderSize
for za0001 := range z.Versions {
s += z.Versions[za0001].Msgsize()
func (z *xlMetaDataDirDecoder) Msgsize() (s int) {
s = 1 + 6
if z.ObjectV2 == nil {
s += msgp.NilSize
} else {
s += 1 + 5 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize))
}
return
}
@ -673,6 +861,14 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
}
}
case "PartETags":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "PartETags")
return
}
z.PartETags = nil
} else {
var zb0006 uint32
zb0006, err = dc.ReadArrayHeader()
if err != nil {
@ -691,6 +887,7 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
}
}
case "PartSizes":
var zb0007 uint32
zb0007, err = dc.ReadArrayHeader()
@ -711,6 +908,14 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
}
}
case "PartASizes":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "PartActualSizes")
return
}
z.PartActualSizes = nil
} else {
var zb0008 uint32
zb0008, err = dc.ReadArrayHeader()
if err != nil {
@ -729,6 +934,7 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
}
}
case "Size":
z.Size, err = dc.ReadInt64()
if err != nil {
@ -742,6 +948,14 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
case "MetaSys":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
z.MetaSys = nil
} else {
var zb0009 uint32
zb0009, err = dc.ReadMapHeader()
if err != nil {
@ -771,7 +985,16 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
}
z.MetaSys[za0008] = za0009
}
}
case "MetaUsr":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "MetaUser")
return
}
z.MetaUser = nil
} else {
var zb0010 uint32
zb0010, err = dc.ReadMapHeader()
if err != nil {
@ -801,6 +1024,7 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
}
z.MetaUser[za0010] = za0011
}
}
default:
err = dc.Skip()
if err != nil {
@ -814,31 +1038,9 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
// omitempty: check for empty values
zb0001Len := uint32(17)
var zb0001Mask uint32 /* 17 bits */
if z.PartActualSizes == nil {
zb0001Len--
zb0001Mask |= 0x1000
}
if z.MetaSys == nil {
zb0001Len--
zb0001Mask |= 0x8000
}
if z.MetaUser == nil {
zb0001Len--
zb0001Mask |= 0x10000
}
// variable map header, size zb0001Len
err = en.WriteMapHeader(zb0001Len)
if err != nil {
return
}
if zb0001Len == 0 {
return
}
// map header, size 17
// write "ID"
err = en.Append(0xa2, 0x49, 0x44)
err = en.Append(0xde, 0x0, 0x11, 0xa2, 0x49, 0x44)
if err != nil {
return
}
@ -956,6 +1158,12 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
if err != nil {
return
}
if z.PartETags == nil { // allownil: if nil
err = en.WriteNil()
if err != nil {
return
}
} else {
err = en.WriteArrayHeader(uint32(len(z.PartETags)))
if err != nil {
err = msgp.WrapError(err, "PartETags")
@ -968,6 +1176,7 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
return
}
}
}
// write "PartSizes"
err = en.Append(0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73)
if err != nil {
@ -985,12 +1194,17 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
return
}
}
if (zb0001Mask & 0x1000) == 0 { // if not empty
// write "PartASizes"
err = en.Append(0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73)
if err != nil {
return
}
if z.PartActualSizes == nil { // allownil: if nil
err = en.WriteNil()
if err != nil {
return
}
} else {
err = en.WriteArrayHeader(uint32(len(z.PartActualSizes)))
if err != nil {
err = msgp.WrapError(err, "PartActualSizes")
@ -1024,12 +1238,17 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "ModTime")
return
}
if (zb0001Mask & 0x8000) == 0 { // if not empty
// write "MetaSys"
err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73)
if err != nil {
return
}
if z.MetaSys == nil { // allownil: if nil
err = en.WriteNil()
if err != nil {
return
}
} else {
err = en.WriteMapHeader(uint32(len(z.MetaSys)))
if err != nil {
err = msgp.WrapError(err, "MetaSys")
@ -1048,12 +1267,17 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
}
}
}
if (zb0001Mask & 0x10000) == 0 { // if not empty
// write "MetaUsr"
err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72)
if err != nil {
return
}
if z.MetaUser == nil { // allownil: if nil
err = en.WriteNil()
if err != nil {
return
}
} else {
err = en.WriteMapHeader(uint32(len(z.MetaUser)))
if err != nil {
err = msgp.WrapError(err, "MetaUser")
@ -1078,28 +1302,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
// MarshalMsg implements msgp.Marshaler
func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// omitempty: check for empty values
zb0001Len := uint32(17)
var zb0001Mask uint32 /* 17 bits */
if z.PartActualSizes == nil {
zb0001Len--
zb0001Mask |= 0x1000
}
if z.MetaSys == nil {
zb0001Len--
zb0001Mask |= 0x8000
}
if z.MetaUser == nil {
zb0001Len--
zb0001Mask |= 0x10000
}
// variable map header, size zb0001Len
o = msgp.AppendMapHeader(o, zb0001Len)
if zb0001Len == 0 {
return
}
// map header, size 17
// string "ID"
o = append(o, 0xa2, 0x49, 0x44)
o = append(o, 0xde, 0x0, 0x11, 0xa2, 0x49, 0x44)
o = msgp.AppendBytes(o, (z.VersionID)[:])
// string "DDir"
o = append(o, 0xa4, 0x44, 0x44, 0x69, 0x72)
@ -1136,19 +1341,25 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
}
// string "PartETags"
o = append(o, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73)
if z.PartETags == nil { // allownil: if nil
o = msgp.AppendNil(o)
} else {
o = msgp.AppendArrayHeader(o, uint32(len(z.PartETags)))
for za0005 := range z.PartETags {
o = msgp.AppendString(o, z.PartETags[za0005])
}
}
// string "PartSizes"
o = append(o, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73)
o = msgp.AppendArrayHeader(o, uint32(len(z.PartSizes)))
for za0006 := range z.PartSizes {
o = msgp.AppendInt64(o, z.PartSizes[za0006])
}
if (zb0001Mask & 0x1000) == 0 { // if not empty
// string "PartASizes"
o = append(o, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73)
if z.PartActualSizes == nil { // allownil: if nil
o = msgp.AppendNil(o)
} else {
o = msgp.AppendArrayHeader(o, uint32(len(z.PartActualSizes)))
for za0007 := range z.PartActualSizes {
o = msgp.AppendInt64(o, z.PartActualSizes[za0007])
@ -1160,18 +1371,22 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
// string "MTime"
o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65)
o = msgp.AppendInt64(o, z.ModTime)
if (zb0001Mask & 0x8000) == 0 { // if not empty
// string "MetaSys"
o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73)
if z.MetaSys == nil { // allownil: if nil
o = msgp.AppendNil(o)
} else {
o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys)))
for za0008, za0009 := range z.MetaSys {
o = msgp.AppendString(o, za0008)
o = msgp.AppendBytes(o, za0009)
}
}
if (zb0001Mask & 0x10000) == 0 { // if not empty
// string "MetaUsr"
o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72)
if z.MetaUser == nil { // allownil: if nil
o = msgp.AppendNil(o)
} else {
o = msgp.AppendMapHeader(o, uint32(len(z.MetaUser)))
for za0010, za0011 := range z.MetaUser {
o = msgp.AppendString(o, za0010)
@ -1294,6 +1509,10 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
}
case "PartETags":
if msgp.IsNil(bts) {
bts = bts[1:]
z.PartETags = nil
} else {
var zb0006 uint32
zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
@ -1312,6 +1531,7 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
}
}
case "PartSizes":
var zb0007 uint32
zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts)
@ -1332,6 +1552,10 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
}
case "PartASizes":
if msgp.IsNil(bts) {
bts = bts[1:]
z.PartActualSizes = nil
} else {
var zb0008 uint32
zb0008, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
@ -1350,6 +1574,7 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
}
}
case "Size":
z.Size, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
@ -1363,6 +1588,10 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
case "MetaSys":
if msgp.IsNil(bts) {
bts = bts[1:]
z.MetaSys = nil
} else {
var zb0009 uint32
zb0009, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
@ -1392,7 +1621,12 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
z.MetaSys[za0008] = za0009
}
}
case "MetaUsr":
if msgp.IsNil(bts) {
bts = bts[1:]
z.MetaUser = nil
} else {
var zb0010 uint32
zb0010, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
@ -1422,6 +1656,7 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
z.MetaUser[za0010] = za0011
}
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@ -1826,3 +2061,154 @@ func (z *xlMetaV2Version) Msgsize() (s int) {
}
return
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) {
var zb0001 uint32
zb0001, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != 5 {
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
return
}
err = dc.ReadExactBytes((z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
z.ModTime, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
err = dc.ReadExactBytes((z.Signature)[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return
}
{
var zb0002 uint8
zb0002, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
z.Type = VersionType(zb0002)
}
{
var zb0003 uint8
zb0003, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
z.Flags = xlFlags(zb0003)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) {
// array header, size 5
err = en.Append(0x95)
if err != nil {
return
}
err = en.WriteBytes((z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
err = en.WriteInt64(z.ModTime)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
err = en.WriteBytes((z.Signature)[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return
}
err = en.WriteUint8(uint8(z.Type))
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
err = en.WriteUint8(uint8(z.Flags))
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *xlMetaV2VersionHeader) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// array header, size 5
o = append(o, 0x95)
o = msgp.AppendBytes(o, (z.VersionID)[:])
o = msgp.AppendInt64(o, z.ModTime)
o = msgp.AppendBytes(o, (z.Signature)[:])
o = msgp.AppendUint8(o, uint8(z.Type))
o = msgp.AppendUint8(o, uint8(z.Flags))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) {
var zb0001 uint32
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != 5 {
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
return
}
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return
}
{
var zb0002 uint8
zb0002, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
z.Type = VersionType(zb0002)
}
{
var zb0003 uint8
zb0003, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
z.Flags = xlFlags(zb0003)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *xlMetaV2VersionHeader) Msgsize() (s int) {
s = 1 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + msgp.Int64Size + msgp.ArrayHeaderSize + (4 * (msgp.ByteSize)) + msgp.Uint8Size + msgp.Uint8Size
return
}

145
cmd/xl-storage-format-v2_gen_test.go
View File

@ -9,8 +9,8 @@ import (
"github.com/tinylib/msgp/msgp"
)
func TestMarshalUnmarshalxlMetaV2(t *testing.T) {
v := xlMetaV2{}
func TestMarshalUnmarshalxlMetaDataDirDecoder(t *testing.T) {
v := xlMetaDataDirDecoder{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
@ -32,8 +32,8 @@ func TestMarshalUnmarshalxlMetaV2(t *testing.T) {
}
}
func BenchmarkMarshalMsgxlMetaV2(b *testing.B) {
v := xlMetaV2{}
func BenchmarkMarshalMsgxlMetaDataDirDecoder(b *testing.B) {
v := xlMetaDataDirDecoder{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
@ -41,8 +41,8 @@ func BenchmarkMarshalMsgxlMetaV2(b *testing.B) {
}
}
func BenchmarkAppendMsgxlMetaV2(b *testing.B) {
v := xlMetaV2{}
func BenchmarkAppendMsgxlMetaDataDirDecoder(b *testing.B) {
v := xlMetaDataDirDecoder{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
@ -53,8 +53,8 @@ func BenchmarkAppendMsgxlMetaV2(b *testing.B) {
}
}
func BenchmarkUnmarshalxlMetaV2(b *testing.B) {
v := xlMetaV2{}
func BenchmarkUnmarshalxlMetaDataDirDecoder(b *testing.B) {
v := xlMetaDataDirDecoder{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
@ -67,17 +67,17 @@ func BenchmarkUnmarshalxlMetaV2(b *testing.B) {
}
}
func TestEncodeDecodexlMetaV2(t *testing.T) {
v := xlMetaV2{}
func TestEncodeDecodexlMetaDataDirDecoder(t *testing.T) {
v := xlMetaDataDirDecoder{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodexlMetaV2 Msgsize() is inaccurate")
t.Log("WARNING: TestEncodeDecodexlMetaDataDirDecoder Msgsize() is inaccurate")
}
vn := xlMetaV2{}
vn := xlMetaDataDirDecoder{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
@ -91,8 +91,8 @@ func TestEncodeDecodexlMetaV2(t *testing.T) {
}
}
func BenchmarkEncodexlMetaV2(b *testing.B) {
v := xlMetaV2{}
func BenchmarkEncodexlMetaDataDirDecoder(b *testing.B) {
v := xlMetaDataDirDecoder{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
@ -105,8 +105,8 @@ func BenchmarkEncodexlMetaV2(b *testing.B) {
en.Flush()
}
func BenchmarkDecodexlMetaV2(b *testing.B) {
v := xlMetaV2{}
func BenchmarkDecodexlMetaDataDirDecoder(b *testing.B) {
v := xlMetaDataDirDecoder{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
@ -460,3 +460,116 @@ func BenchmarkDecodexlMetaV2Version(b *testing.B) {
}
}
}
func TestMarshalUnmarshalxlMetaV2VersionHeader(t *testing.T) {
v := xlMetaV2VersionHeader{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgxlMetaV2VersionHeader(b *testing.B) {
v := xlMetaV2VersionHeader{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgxlMetaV2VersionHeader(b *testing.B) {
v := xlMetaV2VersionHeader{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalxlMetaV2VersionHeader(b *testing.B) {
v := xlMetaV2VersionHeader{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodexlMetaV2VersionHeader(t *testing.T) {
v := xlMetaV2VersionHeader{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodexlMetaV2VersionHeader Msgsize() is inaccurate")
}
vn := xlMetaV2VersionHeader{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodexlMetaV2VersionHeader(b *testing.B) {
v := xlMetaV2VersionHeader{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodexlMetaV2VersionHeader(b *testing.B) {
v := xlMetaV2VersionHeader{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}

27
cmd/xl-storage-format-v2_string.go Normal file
View File

@ -0,0 +1,27 @@
// Code generated by "stringer -type VersionType -output=xl-storage-format-v2_string.go xl-storage-format-v2.go"; DO NOT EDIT.
package cmd
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[invalidVersionType-0]
_ = x[ObjectType-1]
_ = x[DeleteType-2]
_ = x[LegacyType-3]
_ = x[lastVersionType-4]
}
const _VersionType_name = "invalidVersionTypeObjectTypeDeleteTypeLegacyTypelastVersionType"
var _VersionType_index = [...]uint8{0, 18, 28, 38, 48, 63}
func (i VersionType) String() string {
if i >= VersionType(len(_VersionType_index)-1) {
return "VersionType(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _VersionType_name[_VersionType_index[i]:_VersionType_index[i+1]]
}

118
cmd/xl-storage-format-v2_test.go
View File

@ -19,12 +19,15 @@ package cmd
import (
"bytes"
"sort"
"testing"
"time"
"github.com/google/uuid"
"github.com/klauspost/compress/zstd"
"github.com/minio/minio/internal/bucket/lifecycle"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/ioutil"
)
func TestXLV2FormatData(t *testing.T) {
@ -341,15 +344,17 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
}
}
fi.TransitionStatus = tc.transitionStatus
fi.ModTime = fi.ModTime.Add(time.Duration(i) * time.Second)
failOnErr(i+1, xl.AddVersion(fi))
fi.ExpireRestored = tc.expireRestored
fileInfos = append(fileInfos, fi)
}
for i, tc := range testCases {
version := xl.Versions[i]
if actual := xl.SharedDataDirCount(version.ObjectV2.VersionID, version.ObjectV2.DataDir); actual != tc.shares {
t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2, tc.shares, actual)
_, version, err := xl.findVersion(uuid.MustParse(tc.versionID))
failOnErr(i+1, err)
if got := xl.SharedDataDirCount(version.getVersionID(), version.ObjectV2.DataDir); got != tc.shares {
t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2.VersionID, tc.shares, got)
}
}
@ -366,3 +371,110 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
count++
}
}
func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
if err != nil {
b.Fatal(err)
}
dec, _ := zstd.NewReader(nil)
data, err = dec.DecodeAll(data, nil)
if err != nil {
b.Fatal(err)
}
b.Run("legacy", func(b *testing.B) {
var xl xlMetaV2
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(855) // number of versions...
for i := 0; i < b.N; i++ {
err = xl.Load(data)
if err != nil {
b.Fatal(err)
}
}
})
b.Run("indexed", func(b *testing.B) {
var xl xlMetaV2
err = xl.Load(data)
if err != nil {
b.Fatal(err)
}
data, err := xl.AppendTo(nil)
if err != nil {
b.Fatal(err)
}
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(855) // number of versions...
for i := 0; i < b.N; i++ {
err = xl.Load(data)
if err != nil {
b.Fatal(err)
}
}
})
}
func Test_xlMetaV2Shallow_Load(t *testing.T) {
// Load Legacy
data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
if err != nil {
t.Fatal(err)
}
dec, _ := zstd.NewReader(nil)
data, err = dec.DecodeAll(data, nil)
if err != nil {
t.Fatal(err)
}
test := func(t *testing.T, xl *xlMetaV2) {
if len(xl.versions) != 855 {
t.Errorf("want %d versions, got %d", 855, len(xl.versions))
}
xl.sortByModTime()
if !sort.SliceIsSorted(xl.versions, func(i, j int) bool {
return xl.versions[i].header.ModTime > xl.versions[j].header.ModTime
}) {
t.Errorf("Contents not sorted")
}
for i := range xl.versions {
hdr := xl.versions[i].header
ver, err := xl.getIdx(i)
if err != nil {
t.Error(err)
continue
}
gotHdr := ver.header()
if hdr != gotHdr {
t.Errorf("Header does not match, index: %+v != meta: %+v", hdr, gotHdr)
}
}
}
t.Run("load-legacy", func(t *testing.T) {
var xl xlMetaV2
err = xl.Load(data)
if err != nil {
t.Fatal(err)
}
test(t, &xl)
})
t.Run("roundtrip", func(t *testing.T) {
var xl xlMetaV2
err = xl.Load(data)
if err != nil {
t.Fatal(err)
}
data, err = xl.AppendTo(nil)
if err != nil {
t.Fatal(err)
}
xl = xlMetaV2{}
err = xl.Load(data)
if err != nil {
t.Fatal(err)
}
test(t, &xl)
})
}

222
cmd/xl-storage-format_test.go
View File

@ -21,10 +21,14 @@ import (
"bytes"
"encoding/hex"
"encoding/json"
"fmt"
"math/rand"
"testing"
"time"
"github.com/dustin/go-humanize"
jsoniter "github.com/json-iterator/go"
xhttp "github.com/minio/minio/internal/http"
)
func TestIsXLMetaFormatValid(t *testing.T) {
@ -317,3 +321,221 @@ func TestGetPartSizeFromIdx(t *testing.T) {
}
}
}
func BenchmarkXlMetaV2Shallow(b *testing.B) {
fi := FileInfo{
Volume: "volume",
Name: "object-name",
VersionID: "756100c6-b393-4981-928a-d49bbc164741",
IsLatest: true,
Deleted: false,
TransitionStatus: "PENDING",
DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef",
XLV1: false,
ModTime: time.Now(),
Size: 1234456,
Mode: 0,
Metadata: map[string]string{
xhttp.AmzRestore: "FAILED",
xhttp.ContentMD5: mustGetUUID(),
xhttp.AmzBucketReplicationStatus: "PENDING",
xhttp.ContentType: "application/json",
},
Parts: []ObjectPartInfo{{
Number: 1,
Size: 1234345,
ActualSize: 1234345,
},
{
Number: 2,
Size: 1234345,
ActualSize: 1234345,
},
},
Erasure: ErasureInfo{
Algorithm: ReedSolomon.String(),
DataBlocks: 4,
ParityBlocks: 2,
BlockSize: 10000,
Index: 1,
Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8},
Checksums: []ChecksumInfo{{
PartNumber: 1,
Algorithm: HighwayHash256S,
Hash: nil,
},
{
PartNumber: 2,
Algorithm: HighwayHash256S,
Hash: nil,
},
},
},
}
for _, size := range []int{1, 10, 1000, 100_000} {
b.Run(fmt.Sprint(size, "-versions"), func(b *testing.B) {
var xl xlMetaV2
ids := make([]string, size)
for i := 0; i < size; i++ {
fi.VersionID = mustGetUUID()
fi.DataDir = mustGetUUID()
ids[i] = fi.VersionID
fi.ModTime = fi.ModTime.Add(-time.Second)
xl.AddVersion(fi)
}
// Encode all. This is used for benchmarking.
enc, err := xl.AppendTo(nil)
if err != nil {
b.Fatal(err)
}
b.Logf("Serialized size: %d bytes", len(enc))
rng := rand.New(rand.NewSource(0))
var dump = make([]byte, len(enc))
b.Run("UpdateObjectVersion", func(b *testing.B) {
b.SetBytes(int64(size))
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
// Load...
xl = xlMetaV2{}
err := xl.Load(enc)
if err != nil {
b.Fatal(err)
}
// Update modtime for resorting...
fi.ModTime = fi.ModTime.Add(-time.Second)
// Update a random version.
fi.VersionID = ids[rng.Intn(size)]
// Update...
err = xl.UpdateObjectVersion(fi)
if err != nil {
b.Fatal(err)
}
// Save...
dump, err = xl.AppendTo(dump[:0])
if err != nil {
b.Fatal(err)
}
}
})
b.Run("DeleteVersion", func(b *testing.B) {
b.SetBytes(int64(size))
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
// Load...
xl = xlMetaV2{}
err := xl.Load(enc)
if err != nil {
b.Fatal(err)
}
// Update a random version.
fi.VersionID = ids[rng.Intn(size)]
// Delete...
_, _, err = xl.DeleteVersion(fi)
if err != nil {
b.Fatal(err)
}
// Save...
dump, err = xl.AppendTo(dump[:0])
if err != nil {
b.Fatal(err)
}
}
})
b.Run("AddVersion", func(b *testing.B) {
b.SetBytes(int64(size))
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
// Load...
xl = xlMetaV2{}
err := xl.Load(enc)
if err != nil {
b.Fatal(err)
}
// Update modtime for resorting...
fi.ModTime = fi.ModTime.Add(-time.Second)
// Update a random version.
fi.VersionID = mustGetUUID()
// Add...
err = xl.AddVersion(fi)
if err != nil {
b.Fatal(err)
}
// Save...
dump, err = xl.AppendTo(dump[:0])
if err != nil {
b.Fatal(err)
}
}
})
b.Run("ToFileInfo", func(b *testing.B) {
b.SetBytes(int64(size))
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
// Load...
xl = xlMetaV2{}
err := xl.Load(enc)
if err != nil {
b.Fatal(err)
}
// List...
_, err = xl.ToFileInfo("volume", "path", ids[rng.Intn(size)])
if err != nil {
b.Fatal(err)
}
}
})
b.Run("ListVersions", func(b *testing.B) {
b.SetBytes(int64(size))
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
// Load...
xl = xlMetaV2{}
err := xl.Load(enc)
if err != nil {
b.Fatal(err)
}
// List...
_, err = xl.ListVersions("volume", "path")
if err != nil {
b.Fatal(err)
}
}
})
b.Run("ToFileInfoNew", func(b *testing.B) {
b.SetBytes(int64(size))
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
buf, _ := isIndexedMetaV2(enc)
if buf == nil {
b.Fatal("buf == nil")
}
_, err = buf.ToFileInfo("volume", "path", ids[rng.Intn(size)])
if err != nil {
b.Fatal(err)
}
}
})
b.Run("ListVersionsNew", func(b *testing.B) {
b.SetBytes(int64(size))
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
buf, _ := isIndexedMetaV2(enc)
if buf == nil {
b.Fatal("buf == nil")
}
_, err = buf.ListVersions("volume", "path")
if err != nil {
b.Fatal(err)
}
}
})
})
}
}

21
cmd/xl-storage-free-version.go
View File

@ -75,7 +75,7 @@ func (j xlMetaV2Version) FreeVersion() bool {
// AddFreeVersion adds a free-version if needed for fi.VersionID version.
// Free-version will be added if fi.VersionID has transitioned.
func (z *xlMetaV2) AddFreeVersion(fi FileInfo) error {
func (x *xlMetaV2) AddFreeVersion(fi FileInfo) error {
var uv uuid.UUID
var err error
switch fi.VersionID {
@ -87,19 +87,22 @@ func (z *xlMetaV2) AddFreeVersion(fi FileInfo) error {
}
}
for _, version := range z.Versions {
switch version.Type {
case ObjectType:
if version.ObjectV2.VersionID == uv {
for i, version := range x.versions {
if version.header.VersionID != uv || version.header.Type != ObjectType {
continue
}
// if uv has tiered content we add a
// free-version to track it for asynchronous
// deletion via scanner.
if freeVersion, toFree := version.ObjectV2.InitFreeVersion(fi); toFree {
z.Versions = append(z.Versions, freeVersion)
ver, err := x.getIdx(i)
if err != nil {
return err
}
if freeVersion, toFree := ver.ObjectV2.InitFreeVersion(fi); toFree {
return x.addVersion(freeVersion)
}
return nil
}
}
}
return nil
}

47
cmd/xl-storage-free-version_test.go
View File

@ -24,8 +24,8 @@ import (
"github.com/minio/minio/internal/bucket/lifecycle"
)
func (z xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
fivs, _, err := z.ListVersions(volume, path)
func (x xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
fivs, err := x.ListVersions(volume, path)
if err != nil {
return nil, err
}
@ -41,8 +41,21 @@ func (z xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
}
func TestFreeVersion(t *testing.T) {
fatalErr := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
// Add a version with tiered content, one with local content
xl := xlMetaV2{}
counter := 1
report := func() {
t.Helper()
// t.Logf("versions (%d): len = %d", counter, len(xl.versions))
counter++
}
fi := FileInfo{
Volume: "volume",
Name: "object-name",
@ -77,16 +90,21 @@ func TestFreeVersion(t *testing.T) {
SuccessorModTime: time.Time{},
}
// Add a version with local content
xl.AddVersion(fi)
fatalErr(xl.AddVersion(fi))
report()
// Add null version with tiered content
tierfi := fi
tierfi.VersionID = ""
xl.AddVersion(tierfi)
fatalErr(xl.AddVersion(tierfi))
report()
tierfi.TransitionStatus = lifecycle.TransitionComplete
tierfi.TransitionedObjName = mustGetUUID()
tierfi.TransitionTier = "MINIOTIER-1"
xl.DeleteVersion(tierfi)
var err error
_, _, err = xl.DeleteVersion(tierfi)
fatalErr(err)
report()
fvIDs := []string{
"00000000-0000-0000-0000-0000000000f1",
@ -95,15 +113,20 @@ func TestFreeVersion(t *testing.T) {
// Simulate overwrite of null version
newtierfi := tierfi
newtierfi.SetTierFreeVersionID(fvIDs[0])
xl.AddFreeVersion(newtierfi)
xl.AddVersion(newtierfi)
fatalErr(xl.AddFreeVersion(newtierfi))
report()
fatalErr(xl.AddVersion(newtierfi))
report()
// Simulate removal of null version
newtierfi.TransitionTier = ""
newtierfi.TransitionedObjName = ""
newtierfi.TransitionStatus = ""
newtierfi.SetTierFreeVersionID(fvIDs[1])
xl.DeleteVersion(newtierfi)
report()
_, _, err = xl.DeleteVersion(newtierfi)
report()
fatalErr(err)
// Check number of free-versions
freeVersions, err := xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
@ -118,8 +141,10 @@ func TestFreeVersion(t *testing.T) {
freefi := newtierfi
for _, fvID := range fvIDs {
freefi.VersionID = fvID
xl.DeleteVersion(freefi)
_, _, err = xl.DeleteVersion(freefi)
fatalErr(err)
}
report()
// Check number of free-versions
freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
@ -129,11 +154,13 @@ func TestFreeVersion(t *testing.T) {
if len(freeVersions) != 0 {
t.Fatalf("Expected zero free version but got %d", len(freeVersions))
}
report()
// Adding a free version to a version with no tiered content.
newfi := fi
newfi.SetTierFreeVersionID("00000000-0000-0000-0000-0000000000f3")
xl.AddFreeVersion(newfi) // this shouldn't add a free-version
fatalErr(xl.AddFreeVersion(newfi)) // this shouldn't add a free-version
report()
// Check number of free-versions
freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)

408
cmd/xl-storage-meta-inline.go Normal file
View File

@ -0,0 +1,408 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"errors"
"fmt"
"github.com/minio/minio/internal/logger"
"github.com/tinylib/msgp/msgp"
)
// xlMetaInlineData is serialized data in [string][]byte pairs.
type xlMetaInlineData []byte
// xlMetaInlineDataVer indicates the version of the inline data structure.
const xlMetaInlineDataVer = 1
// versionOK returns whether the version is ok.
func (x xlMetaInlineData) versionOK() bool {
if len(x) == 0 {
return true
}
return x[0] > 0 && x[0] <= xlMetaInlineDataVer
}
// afterVersion returns the payload after the version, if any.
func (x xlMetaInlineData) afterVersion() []byte {
if len(x) == 0 {
return x
}
return x[1:]
}
// find the data with key s.
// Returns nil if not for or an error occurs.
func (x xlMetaInlineData) find(key string) []byte {
if len(x) == 0 || !x.versionOK() {
return nil
}
sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
if err != nil || sz == 0 {
return nil
}
for i := uint32(0); i < sz; i++ {
var found []byte
found, buf, err = msgp.ReadMapKeyZC(buf)
if err != nil || sz == 0 {
return nil
}
if string(found) == key {
val, _, _ := msgp.ReadBytesZC(buf)
return val
}
// Skip it
_, buf, err = msgp.ReadBytesZC(buf)
if err != nil {
return nil
}
}
return nil
}
// validate checks if the data is valid.
// It does not check integrity of the stored data.
func (x xlMetaInlineData) validate() error {
if len(x) == 0 {
return nil
}
if !x.versionOK() {
return fmt.Errorf("xlMetaInlineData: unknown version 0x%x", x[0])
}
sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
if err != nil {
return fmt.Errorf("xlMetaInlineData: %w", err)
}
for i := uint32(0); i < sz; i++ {
var key []byte
key, buf, err = msgp.ReadMapKeyZC(buf)
if err != nil {
return fmt.Errorf("xlMetaInlineData: %w", err)
}
if len(key) == 0 {
return fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
}
_, buf, err = msgp.ReadBytesZC(buf)
if err != nil {
return fmt.Errorf("xlMetaInlineData: %w", err)
}
}
return nil
}
// repair will copy all seemingly valid data entries from a corrupted set.
// This does not ensure that data is correct, but will allow all operations to complete.
func (x *xlMetaInlineData) repair() {
data := *x
if len(data) == 0 {
return
}
if !data.versionOK() {
*x = nil
return
}
sz, buf, err := msgp.ReadMapHeaderBytes(data.afterVersion())
if err != nil {
*x = nil
return
}
// Remove all current data
keys := make([][]byte, 0, sz)
vals := make([][]byte, 0, sz)
for i := uint32(0); i < sz; i++ {
var key, val []byte
key, buf, err = msgp.ReadMapKeyZC(buf)
if err != nil {
break
}
if len(key) == 0 {
break
}
val, buf, err = msgp.ReadBytesZC(buf)
if err != nil {
break
}
keys = append(keys, key)
vals = append(vals, val)
}
x.serialize(-1, keys, vals)
}
// validate checks if the data is valid.
// It does not check integrity of the stored data.
func (x xlMetaInlineData) list() ([]string, error) {
if len(x) == 0 {
return nil, nil
}
if !x.versionOK() {
return nil, errors.New("xlMetaInlineData: unknown version")
}
sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
if err != nil {
return nil, err
}
keys := make([]string, 0, sz)
for i := uint32(0); i < sz; i++ {
var key []byte
key, buf, err = msgp.ReadMapKeyZC(buf)
if err != nil {
return keys, err
}
if len(key) == 0 {
return keys, fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
}
keys = append(keys, string(key))
// Skip data...
_, buf, err = msgp.ReadBytesZC(buf)
if err != nil {
return keys, err
}
}
return keys, nil
}
// serialize will serialize the provided keys and values.
// The function will panic if keys/value slices aren't of equal length.
// Payload size can give an indication of expected payload size.
// If plSize is <= 0 it will be calculated.
func (x *xlMetaInlineData) serialize(plSize int, keys [][]byte, vals [][]byte) {
if len(keys) != len(vals) {
panic(fmt.Errorf("xlMetaInlineData.serialize: keys/value number mismatch"))
}
if len(keys) == 0 {
*x = nil
return
}
if plSize <= 0 {
plSize = 1 + msgp.MapHeaderSize
for i := range keys {
plSize += len(keys[i]) + len(vals[i]) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
}
}
payload := make([]byte, 1, plSize)
payload[0] = xlMetaInlineDataVer
payload = msgp.AppendMapHeader(payload, uint32(len(keys)))
for i := range keys {
payload = msgp.AppendStringFromBytes(payload, keys[i])
payload = msgp.AppendBytes(payload, vals[i])
}
*x = payload
}
// entries returns the number of entries in the data.
func (x xlMetaInlineData) entries() int {
if len(x) == 0 || !x.versionOK() {
return 0
}
sz, _, _ := msgp.ReadMapHeaderBytes(x.afterVersion())
return int(sz)
}
// replace will add or replace a key/value pair.
func (x *xlMetaInlineData) replace(key string, value []byte) {
in := x.afterVersion()
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
keys := make([][]byte, 0, sz+1)
vals := make([][]byte, 0, sz+1)
// Version plus header...
plSize := 1 + msgp.MapHeaderSize
replaced := false
for i := uint32(0); i < sz; i++ {
var found, foundVal []byte
var err error
found, buf, err = msgp.ReadMapKeyZC(buf)
if err != nil {
break
}
foundVal, buf, err = msgp.ReadBytesZC(buf)
if err != nil {
break
}
plSize += len(found) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
keys = append(keys, found)
if string(found) == key {
vals = append(vals, value)
plSize += len(value)
replaced = true
} else {
vals = append(vals, foundVal)
plSize += len(foundVal)
}
}
// Add one more.
if !replaced {
keys = append(keys, []byte(key))
vals = append(vals, value)
plSize += len(key) + len(value) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
}
// Reserialize...
x.serialize(plSize, keys, vals)
}
// rename will rename a key.
// Returns whether the key was found.
func (x *xlMetaInlineData) rename(oldKey, newKey string) bool {
in := x.afterVersion()
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
keys := make([][]byte, 0, sz)
vals := make([][]byte, 0, sz)
// Version plus header...
plSize := 1 + msgp.MapHeaderSize
found := false
for i := uint32(0); i < sz; i++ {
var foundKey, foundVal []byte
var err error
foundKey, buf, err = msgp.ReadMapKeyZC(buf)
if err != nil {
break
}
foundVal, buf, err = msgp.ReadBytesZC(buf)
if err != nil {
break
}
plSize += len(foundVal) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
vals = append(vals, foundVal)
if string(foundKey) != oldKey {
keys = append(keys, foundKey)
plSize += len(foundKey)
} else {
keys = append(keys, []byte(newKey))
plSize += len(newKey)
found = true
}
}
// If not found, just return.
if !found {
return false
}
// Reserialize...
x.serialize(plSize, keys, vals)
return true
}
// remove will remove one or more keys.
// Returns true if any key was found.
func (x *xlMetaInlineData) remove(keys ...string) bool {
in := x.afterVersion()
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
newKeys := make([][]byte, 0, sz)
newVals := make([][]byte, 0, sz)
var removeKey func(s []byte) bool
// Copy if big number of compares...
if len(keys) > 5 && sz > 5 {
mKeys := make(map[string]struct{}, len(keys))
for _, key := range keys {
mKeys[key] = struct{}{}
}
removeKey = func(s []byte) bool {
_, ok := mKeys[string(s)]
return ok
}
} else {
removeKey = func(s []byte) bool {
for _, key := range keys {
if key == string(s) {
return true
}
}
return false
}
}
// Version plus header...
plSize := 1 + msgp.MapHeaderSize
found := false
for i := uint32(0); i < sz; i++ {
var foundKey, foundVal []byte
var err error
foundKey, buf, err = msgp.ReadMapKeyZC(buf)
if err != nil {
break
}
foundVal, buf, err = msgp.ReadBytesZC(buf)
if err != nil {
break
}
if !removeKey(foundKey) {
plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal)
newKeys = append(newKeys, foundKey)
newVals = append(newVals, foundVal)
} else {
found = true
}
}
// If not found, just return.
if !found {
return false
}
// If none left...
if len(newKeys) == 0 {
*x = nil
return true
}
// Reserialize...
x.serialize(plSize, newKeys, newVals)
return true
}
// xlMetaV2TrimData will trim any data from the metadata without unmarshalling it.
// If any error occurs the unmodified data is returned.
func xlMetaV2TrimData(buf []byte) []byte {
metaBuf, min, maj, err := checkXL2V1(buf)
if err != nil {
return buf
}
if maj == 1 && min < 1 {
// First version to carry data.
return buf
}
// Skip header
_, metaBuf, err = msgp.ReadBytesZC(metaBuf)
if err != nil {
logger.LogIf(GlobalContext, err)
return buf
}
// Skip CRC
if maj > 1 || min >= 2 {
_, metaBuf, err = msgp.ReadUint32Bytes(metaBuf)
logger.LogIf(GlobalContext, err)
}
// = input - current pos
ends := len(buf) - len(metaBuf)
if ends > len(buf) {
return buf
}
return buf[:ends]
}

8
cmd/xl-storage.go
View File

@ -978,7 +978,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
}
var xlMeta xlMetaV2
if err = xlMeta.Load(buf); err != nil {
if err := xlMeta.Load(buf); err != nil {
return err
}
@ -1044,6 +1044,7 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi
}
return err
}
defer metaDataPoolPut(buf)
if !isXL2V1Format(buf) {
return errFileVersionNotFound
@ -1059,12 +1060,13 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi
return err
}
buf, err = xlMeta.AppendTo(nil)
wbuf, err := xlMeta.AppendTo(metaDataPoolGet())
if err != nil {
return err
}
defer metaDataPoolPut(wbuf)
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf)
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), wbuf)
}
// WriteMetadata - writes FileInfo metadata for path at `xl.meta`

93
docs/debugging/xl-meta/main.go
View File

@ -114,6 +114,48 @@ FLAGS:
return nil, err
}
data = b
case 3:
v, b, err := msgp.ReadBytesZC(b)
if err != nil {
return nil, err
}
if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil {
// Read metadata CRC (added in v2, ignore if not found)
b = nbuf
}
nVers, v, err := decodeXLHeaders(v)
if err != nil {
return nil, err
}
var versions = struct {
Versions []json.RawMessage
Headers []json.RawMessage
}{
Versions: make([]json.RawMessage, nVers),
Headers: make([]json.RawMessage, nVers),
}
err = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error {
var buf bytes.Buffer
if _, err := msgp.UnmarshalAsJSON(&buf, hdr); err != nil {
return err
}
versions.Headers[idx] = buf.Bytes()
buf = bytes.Buffer{}
if _, err := msgp.UnmarshalAsJSON(&buf, meta); err != nil {
return err
}
versions.Versions[idx] = buf.Bytes()
return nil
})
if err != nil {
return nil, err
}
enc := json.NewEncoder(buf)
if err := enc.Encode(versions); err != nil {
return nil, err
}
data = b
default:
return nil, fmt.Errorf("unknown metadata version %d", minor)
}
@ -416,3 +458,54 @@ func (x xlMetaInlineData) files(fn func(name string, data []byte)) error {
return nil
}
const (
xlHeaderVersion = 2
xlMetaVersion = 1
)
func decodeXLHeaders(buf []byte) (versions int, b []byte, err error) {
hdrVer, buf, err := msgp.ReadUintBytes(buf)
if err != nil {
return 0, buf, err
}
metaVer, buf, err := msgp.ReadUintBytes(buf)
if err != nil {
return 0, buf, err
}
if hdrVer > xlHeaderVersion {
return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", metaVer)
}
if metaVer > xlMetaVersion {
return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", metaVer)
}
versions, buf, err = msgp.ReadIntBytes(buf)
if err != nil {
return 0, buf, err
}
if versions < 0 {
return 0, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", versions)
}
return versions, buf, nil
}
// decodeVersions will decode a number of versions from a buffer
// and perform a callback for each version in order, newest first.
// Any non-nil error is returned.
func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte) error) (err error) {
var tHdr, tMeta []byte // Zero copy bytes
for i := 0; i < versions; i++ {
tHdr, buf, err = msgp.ReadBytesZC(buf)
if err != nil {
return err
}
tMeta, buf, err = msgp.ReadBytesZC(buf)
if err != nil {
return err
}
if err = fn(i, tHdr, tMeta); err != nil {
return err
}
}
return nil
}

1
go.mod
View File

@ -81,6 +81,7 @@ require (
github.com/valyala/bytebufferpool v1.0.0
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
github.com/yargevad/filepathx v1.0.0
github.com/zeebo/xxh3 v1.0.0
go.etcd.io/etcd/api/v3 v3.5.0
go.etcd.io/etcd/client/v3 v3.5.0
go.uber.org/atomic v1.9.0

2
go.sum
View File

@ -1527,6 +1527,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/zeebo/xxh3 v1.0.0 h1:6eLPZCVXpsGnhv8RiWBEJs5kenm2W1CMwon19/l8ODc=
github.com/zeebo/xxh3 v1.0.0/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=