Improve performance on multiple versions (#13573)

Existing:

```go
type xlMetaV2 struct {
    Versions []xlMetaV2Version `json:"Versions" msg:"Versions"`
}
```

Serialized as regular MessagePack.

```go
//msgp:tuple xlMetaV2VersionHeader
type xlMetaV2VersionHeader struct {
	VersionID [16]byte
	ModTime   int64
	Type      VersionType
	Flags     xlFlags
}
```

Serialize as streaming MessagePack, format:

```
int(headerVersion)
int(xlmetaVersion)
int(nVersions)
for each version {
    binary blob, xlMetaV2VersionHeader, serialized
    binary blob, xlMetaV2Version, serialized.
}
```

xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct 
can easily be reused and does not contain pointers, so efficient as a 
slice (single allocation)

This allows quickly parsing everything as slices of bytes (no copy).

Versions are always *saved* sorted by modTime, newest *first*. 
No more need to sort on load.

* Allows checking if a version exists.
* Allows reading single version without unmarshal all.
* Allows reading latest version of type without unmarshal all.
* Allows reading latest version without unmarshal of all.
* Allows checking if the latest is deleteMarker by reading first entry.
* Allows adding/updating/deleting a version with only header deserialization.
* Reduces allocations on conversion to FileInfo(s).

Makefile

@@ -20,7 +20,7 @@ help: ## print this help
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.43.0
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@latest
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio

cmd/bucket-replication.go

@@ -724,7 +724,7 @@ const (
 // matches k1 with all keys, returns 'true' if one of them matches
 func equals(k1 string, keys ...string) bool {
 	for _, k2 := range keys {
-		if strings.EqualFold(strings.ToLower(k1), strings.ToLower(k2)) {
+		if strings.EqualFold(k1, k2) {
 			return true
 		}
 	}

cmd/erasure-metadata.go

@@ -476,9 +476,7 @@ func GetInternalReplicationState(m map[string][]byte) ReplicationState {
 
 // getInternalReplicationState fetches internal replication state from the map m
 func getInternalReplicationState(m map[string]string) ReplicationState {
-	d := ReplicationState{
+	d := ReplicationState{}
-		ResetStatusesMap: make(map[string]string),
-	}
 	for k, v := range m {
 		switch {
 		case equals(k, ReservedMetadataPrefixLower+ReplicationTimestamp):
@@ -497,6 +495,9 @@ func getInternalReplicationState(m map[string]string) ReplicationState {
 			d.PurgeTargets = versionPurgeStatusesMap(v)
 		case strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset):
 			arn := strings.TrimPrefix(k, fmt.Sprintf("%s-", ReservedMetadataPrefixLower+ReplicationReset))
+			if d.ResetStatusesMap == nil {
+				d.ResetStatusesMap = make(map[string]string, 1)
+			}
 			d.ResetStatusesMap[arn] = v
 		}
 	}

cmd/metacache-entries.go

@@ -148,11 +148,15 @@ func (e *metaCacheEntry) isLatestDeletemarker() bool {
 	if !isXL2V1Format(e.metadata) {
 		return false
 	}
+	if meta, _ := isIndexedMetaV2(e.metadata); meta != nil {
+		return meta.IsLatestDeleteMarker()
+	}
+	// Fall back...
 	var xlMeta xlMetaV2
-	if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.Versions) == 0 {
+	if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.versions) == 0 {
 		return true
 	}
-	return xlMeta.Versions[len(xlMeta.Versions)-1].Type == DeleteType
+	return xlMeta.versions[0].header.Type == DeleteType
 }
 
 // fileInfo returns the decoded metadata.

cmd/xl-storage-format-utils.go

@@ -19,27 +19,12 @@ package cmd
 
 import (
 	"fmt"
-	"sort"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/logger"
+	"github.com/zeebo/xxh3"
 )
 
-// versionsSorter sorts FileInfo slices by version.
-type versionsSorter []FileInfo
-
-func (v versionsSorter) sort() {
-	sort.Slice(v, func(i, j int) bool {
-		if v[i].IsLatest {
-			return true
-		}
-		if v[j].IsLatest {
-			return false
-		}
-		return v[i].ModTime.After(v[j].ModTime)
-	})
-}
-
 func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) {
 	fivs, err := getAllFileInfoVersions(xlMetaBuf, volume, path)
 	if err != nil {
@@ -54,24 +39,35 @@ func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersion
 		}
 	}
 	fivs.Versions = fivs.Versions[:n]
+	// Update numversions
+	for i := range fivs.Versions {
+		fivs.Versions[i].NumVersions = n
+	}
 	return fivs, nil
 }
 
 func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) {
 	if isXL2V1Format(xlMetaBuf) {
-		var xlMeta xlMetaV2
+		var versions []FileInfo
-		if err := xlMeta.Load(xlMetaBuf); err != nil {
+		var err error
-			return FileInfoVersions{}, err
+		if buf, _ := isIndexedMetaV2(xlMetaBuf); buf != nil {
-		}
+			versions, err = buf.ListVersions(volume, path)
-		versions, latestModTime, err := xlMeta.ListVersions(volume, path)
+		} else {
-		if err != nil {
+			var xlMeta xlMetaV2
+			if err := xlMeta.Load(xlMetaBuf); err != nil {
+				return FileInfoVersions{}, err
+			}
+			versions, err = xlMeta.ListVersions(volume, path)
+		}
+		if err != nil || len(versions) == 0 {
 			return FileInfoVersions{}, err
 		}
+
 		return FileInfoVersions{
 			Volume:        volume,
 			Name:          path,
 			Versions:      versions,
-			LatestModTime: latestModTime,
+			LatestModTime: versions[0].ModTime,
 		}, nil
 	}
 
@@ -98,11 +94,20 @@ func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVers
 
 func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (FileInfo, error) {
 	if isXL2V1Format(xlMetaBuf) {
-		var xlMeta xlMetaV2
+		var fi FileInfo
-		if err := xlMeta.Load(xlMetaBuf); err != nil {
+		var err error
-			return FileInfo{}, err
+		var inData xlMetaInlineData
+		if buf, data := isIndexedMetaV2(xlMetaBuf); buf != nil {
+			inData = data
+			fi, err = buf.ToFileInfo(volume, path, versionID)
+		} else {
+			var xlMeta xlMetaV2
+			if err := xlMeta.Load(xlMetaBuf); err != nil {
+				return FileInfo{}, err
+			}
+			inData = xlMeta.data
+			fi, err = xlMeta.ToFileInfo(volume, path, versionID)
 		}
-		fi, err := xlMeta.ToFileInfo(volume, path, versionID)
 		if !data || err != nil {
 			return fi, err
 		}
@@ -110,12 +115,12 @@ func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (F
 		if versionID == "" {
 			versionID = nullVersionID
 		}
-		fi.Data = xlMeta.data.find(versionID)
+		fi.Data = inData.find(versionID)
 		if len(fi.Data) == 0 {
 			// PR #11758 used DataDir, preserve it
 			// for users who might have used master
 			// branch
-			fi.Data = xlMeta.data.find(fi.DataDir)
+			fi.Data = inData.find(fi.DataDir)
 		}
 		return fi, nil
 	}
@@ -149,3 +154,27 @@ func getXLDiskLoc(diskID string) (poolIdx, setIdx, diskIdx int) {
 	}
 	return -1, -1, -1
 }
+
+// hashDeterministicString will return a deterministic hash for the map values.
+// Trivial collisions are avoided, but this is by no means a strong hash.
+func hashDeterministicString(m map[string]string) uint64 {
+	// Seed (random)
+	var crc = uint64(0xc2b40bbac11a7295)
+	// Xor each value to make order independent
+	for k, v := range m {
+		// Separate key and value with an individual xor with a random number.
+		// Add values of each, so they cannot be trivially collided.
+		crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.HashString(v) ^ 0x8da4c8da66194257)
+	}
+	return crc
+}
+
+// hashDeterministicBytes will return a deterministic (weak) hash for the map values.
+// Trivial collisions are avoided, but this is by no means a strong hash.
+func hashDeterministicBytes(m map[string][]byte) uint64 {
+	var crc = uint64(0x1bbc7e1dde654743)
+	for k, v := range m {
+		crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.Hash(v) ^ 0x8da4c8da66194257)
+	}
+	return crc
+}

cmd/xl-storage-format-utils_test.go

@@ -0,0 +1,93 @@
+package cmd
+
+import (
+	"testing"
+
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+func Test_hashDeterministicString(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  map[string]string
+	}{
+		{
+			name: "zero",
+			arg:  map[string]string{},
+		},
+		{
+			name: "nil",
+			arg:  nil,
+		},
+		{
+			name: "one",
+			arg:  map[string]string{"key": "value"},
+		},
+		{
+			name: "several",
+			arg: map[string]string{
+				xhttp.AmzRestore:                 "FAILED",
+				xhttp.ContentMD5:                 mustGetUUID(),
+				xhttp.AmzBucketReplicationStatus: "PENDING",
+				xhttp.ContentType:                "application/json",
+			},
+		},
+		{
+			name: "someempty",
+			arg: map[string]string{
+				xhttp.AmzRestore:                 "",
+				xhttp.ContentMD5:                 mustGetUUID(),
+				xhttp.AmzBucketReplicationStatus: "",
+				xhttp.ContentType:                "application/json",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			const n = 100
+			want := hashDeterministicString(tt.arg)
+			m := tt.arg
+			for i := 0; i < n; i++ {
+				if got := hashDeterministicString(m); got != want {
+					t.Errorf("hashDeterministicString() = %v, want %v", got, want)
+				}
+			}
+			// Check casual collisions
+			if m == nil {
+				m = make(map[string]string)
+			}
+			m["12312312"] = ""
+			if got := hashDeterministicString(m); got == want {
+				t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
+			}
+			want = hashDeterministicString(m)
+			delete(m, "12312312")
+			m["another"] = ""
+
+			if got := hashDeterministicString(m); got == want {
+				t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
+			}
+
+			want = hashDeterministicString(m)
+			m["another"] = "hashDeterministicString"
+			if got := hashDeterministicString(m); got == want {
+				t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
+			}
+
+			want = hashDeterministicString(m)
+			m["another"] = "hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString"
+			if got := hashDeterministicString(m); got == want {
+				t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
+			}
+
+			// Flip key/value
+			want = hashDeterministicString(m)
+			delete(m, "another")
+			m["hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString"] = "another"
+			if got := hashDeterministicString(m); got == want {
+				t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
+			}
+
+		})
+	}
+}

cmd/xl-storage-format-v1.go

@@ -18,11 +18,13 @@
 package cmd
 
 import (
+	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/logger"
 )
@@ -205,6 +207,27 @@ func (m *xlMetaV1Object) ToFileInfo(volume, path string) (FileInfo, error) {
 	return fi, nil
 }
 
+// Signature will return a signature that is expected to be the same across all disks.
+func (m *xlMetaV1Object) Signature() [4]byte {
+	// Shallow copy
+	c := *m
+	// Zero unimportant fields
+	c.Erasure.Index = 0
+	c.Minio.Release = ""
+	crc := hashDeterministicString(c.Meta)
+	c.Meta = nil
+
+	if bts, err := c.MarshalMsg(metaDataPoolGet()); err == nil {
+		crc ^= xxhash.Sum64(bts)
+		metaDataPoolPut(bts)
+	}
+
+	// Combine upper and lower part
+	var tmp [4]byte
+	binary.LittleEndian.PutUint32(tmp[:], uint32(crc^(crc>>32)))
+	return tmp
+}
+
 // XL metadata constants.
 const (
 	// XL meta version.

cmd/xl-storage-format-v2-legacy.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+// unmarshalV unmarshals with a specific header version.
+func (x *xlMetaV2VersionHeader) unmarshalV(v uint8, bts []byte) (o []byte, err error) {
+	switch v {
+	case 1:
+		return x.unmarshalV1(bts)
+	case xlHeaderVersion:
+		return x.UnmarshalMsg(bts)
+	}
+	return bts, fmt.Errorf("unknown xlHeaderVersion: %d", v)
+}
+
+// unmarshalV1 decodes version 1, never released.
+func (x *xlMetaV2VersionHeader) unmarshalV1(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	bts, err = msgp.ReadExactBytes(bts, (x.VersionID)[:])
+	if err != nil {
+		err = msgp.WrapError(err, "VersionID")
+		return
+	}
+	x.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ModTime")
+		return
+	}
+	{
+		var zb0002 uint8
+		zb0002, bts, err = msgp.ReadUint8Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "Type")
+			return
+		}
+		x.Type = VersionType(zb0002)
+	}
+	{
+		var zb0003 uint8
+		zb0003, bts, err = msgp.ReadUint8Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "Flags")
+			return
+		}
+		x.Flags = xlFlags(zb0003)
+	}
+	o = bts
+	return
+}
+
+// unmarshalV unmarshals with a specific metadata version.
+func (j *xlMetaV2Version) unmarshalV(v uint8, bts []byte) (o []byte, err error) {
+	switch v {
+	// We accept un-set as latest version.
+	case 0, xlMetaVersion:
+		return j.UnmarshalMsg(bts)
+	}
+	return bts, fmt.Errorf("unknown xlMetaVersion: %d", v)
+}

cmd/xl-storage-format-v2_gen_test.go

@@ -9,8 +9,8 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
-func TestMarshalUnmarshalxlMetaV2(t *testing.T) {
+func TestMarshalUnmarshalxlMetaDataDirDecoder(t *testing.T) {
-	v := xlMetaV2{}
+	v := xlMetaDataDirDecoder{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
@@ -32,8 +32,8 @@ func TestMarshalUnmarshalxlMetaV2(t *testing.T) {
 	}
 }
 
-func BenchmarkMarshalMsgxlMetaV2(b *testing.B) {
+func BenchmarkMarshalMsgxlMetaDataDirDecoder(b *testing.B) {
-	v := xlMetaV2{}
+	v := xlMetaDataDirDecoder{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -41,8 +41,8 @@ func BenchmarkMarshalMsgxlMetaV2(b *testing.B) {
 	}
 }
 
-func BenchmarkAppendMsgxlMetaV2(b *testing.B) {
+func BenchmarkAppendMsgxlMetaDataDirDecoder(b *testing.B) {
-	v := xlMetaV2{}
+	v := xlMetaDataDirDecoder{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
@@ -53,8 +53,8 @@ func BenchmarkAppendMsgxlMetaV2(b *testing.B) {
 	}
 }
 
-func BenchmarkUnmarshalxlMetaV2(b *testing.B) {
+func BenchmarkUnmarshalxlMetaDataDirDecoder(b *testing.B) {
-	v := xlMetaV2{}
+	v := xlMetaDataDirDecoder{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
@@ -67,17 +67,17 @@ func BenchmarkUnmarshalxlMetaV2(b *testing.B) {
 	}
 }
 
-func TestEncodeDecodexlMetaV2(t *testing.T) {
+func TestEncodeDecodexlMetaDataDirDecoder(t *testing.T) {
-	v := xlMetaV2{}
+	v := xlMetaDataDirDecoder{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 
 	m := v.Msgsize()
 	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodexlMetaV2 Msgsize() is inaccurate")
+		t.Log("WARNING: TestEncodeDecodexlMetaDataDirDecoder Msgsize() is inaccurate")
 	}
 
-	vn := xlMetaV2{}
+	vn := xlMetaDataDirDecoder{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
@@ -91,8 +91,8 @@ func TestEncodeDecodexlMetaV2(t *testing.T) {
 	}
 }
 
-func BenchmarkEncodexlMetaV2(b *testing.B) {
+func BenchmarkEncodexlMetaDataDirDecoder(b *testing.B) {
-	v := xlMetaV2{}
+	v := xlMetaDataDirDecoder{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
@@ -105,8 +105,8 @@ func BenchmarkEncodexlMetaV2(b *testing.B) {
 	en.Flush()
 }
 
-func BenchmarkDecodexlMetaV2(b *testing.B) {
+func BenchmarkDecodexlMetaDataDirDecoder(b *testing.B) {
-	v := xlMetaV2{}
+	v := xlMetaDataDirDecoder{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
@@ -460,3 +460,116 @@ func BenchmarkDecodexlMetaV2Version(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalxlMetaV2VersionHeader(t *testing.T) {
+	v := xlMetaV2VersionHeader{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgxlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgxlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalxlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodexlMetaV2VersionHeader(t *testing.T) {
+	v := xlMetaV2VersionHeader{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodexlMetaV2VersionHeader Msgsize() is inaccurate")
+	}
+
+	vn := xlMetaV2VersionHeader{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodexlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodexlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/xl-storage-format-v2_string.go

@@ -0,0 +1,27 @@
+// Code generated by "stringer -type VersionType -output=xl-storage-format-v2_string.go xl-storage-format-v2.go"; DO NOT EDIT.
+
+package cmd
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[invalidVersionType-0]
+	_ = x[ObjectType-1]
+	_ = x[DeleteType-2]
+	_ = x[LegacyType-3]
+	_ = x[lastVersionType-4]
+}
+
+const _VersionType_name = "invalidVersionTypeObjectTypeDeleteTypeLegacyTypelastVersionType"
+
+var _VersionType_index = [...]uint8{0, 18, 28, 38, 48, 63}
+
+func (i VersionType) String() string {
+	if i >= VersionType(len(_VersionType_index)-1) {
+		return "VersionType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _VersionType_name[_VersionType_index[i]:_VersionType_index[i+1]]
+}

cmd/xl-storage-format-v2_test.go

@@ -19,12 +19,15 @@ package cmd
 
 import (
 	"bytes"
+	"sort"
 	"testing"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/klauspost/compress/zstd"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/ioutil"
 )
 
 func TestXLV2FormatData(t *testing.T) {
@@ -341,15 +344,17 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
 			}
 		}
 		fi.TransitionStatus = tc.transitionStatus
+		fi.ModTime = fi.ModTime.Add(time.Duration(i) * time.Second)
 		failOnErr(i+1, xl.AddVersion(fi))
 		fi.ExpireRestored = tc.expireRestored
 		fileInfos = append(fileInfos, fi)
 	}
 
 	for i, tc := range testCases {
-		version := xl.Versions[i]
+		_, version, err := xl.findVersion(uuid.MustParse(tc.versionID))
-		if actual := xl.SharedDataDirCount(version.ObjectV2.VersionID, version.ObjectV2.DataDir); actual != tc.shares {
+		failOnErr(i+1, err)
-			t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2, tc.shares, actual)
+		if got := xl.SharedDataDirCount(version.getVersionID(), version.ObjectV2.DataDir); got != tc.shares {
+			t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2.VersionID, tc.shares, got)
 		}
 	}
 
@@ -366,3 +371,110 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
 		count++
 	}
 }
+
+func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
+	data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
+	if err != nil {
+		b.Fatal(err)
+	}
+	dec, _ := zstd.NewReader(nil)
+	data, err = dec.DecodeAll(data, nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.Run("legacy", func(b *testing.B) {
+		var xl xlMetaV2
+		b.ReportAllocs()
+		b.ResetTimer()
+		b.SetBytes(855) // number of versions...
+		for i := 0; i < b.N; i++ {
+			err = xl.Load(data)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("indexed", func(b *testing.B) {
+		var xl xlMetaV2
+		err = xl.Load(data)
+		if err != nil {
+			b.Fatal(err)
+		}
+		data, err := xl.AppendTo(nil)
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.ReportAllocs()
+		b.ResetTimer()
+		b.SetBytes(855) // number of versions...
+		for i := 0; i < b.N; i++ {
+			err = xl.Load(data)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+
+}
+
+func Test_xlMetaV2Shallow_Load(t *testing.T) {
+	// Load Legacy
+	data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
+	if err != nil {
+		t.Fatal(err)
+	}
+	dec, _ := zstd.NewReader(nil)
+	data, err = dec.DecodeAll(data, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	test := func(t *testing.T, xl *xlMetaV2) {
+		if len(xl.versions) != 855 {
+			t.Errorf("want %d versions, got %d", 855, len(xl.versions))
+		}
+		xl.sortByModTime()
+		if !sort.SliceIsSorted(xl.versions, func(i, j int) bool {
+			return xl.versions[i].header.ModTime > xl.versions[j].header.ModTime
+		}) {
+			t.Errorf("Contents not sorted")
+		}
+		for i := range xl.versions {
+			hdr := xl.versions[i].header
+			ver, err := xl.getIdx(i)
+			if err != nil {
+				t.Error(err)
+				continue
+			}
+			gotHdr := ver.header()
+			if hdr != gotHdr {
+				t.Errorf("Header does not match, index: %+v != meta: %+v", hdr, gotHdr)
+			}
+		}
+	}
+	t.Run("load-legacy", func(t *testing.T) {
+		var xl xlMetaV2
+		err = xl.Load(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+		test(t, &xl)
+	})
+	t.Run("roundtrip", func(t *testing.T) {
+		var xl xlMetaV2
+		err = xl.Load(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+		data, err = xl.AppendTo(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		xl = xlMetaV2{}
+		err = xl.Load(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+		test(t, &xl)
+	})
+}

cmd/xl-storage-format_test.go

@@ -21,10 +21,14 @@ import (
 	"bytes"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
+	"math/rand"
 	"testing"
+	"time"
 
 	"github.com/dustin/go-humanize"
 	jsoniter "github.com/json-iterator/go"
+	xhttp "github.com/minio/minio/internal/http"
 )
 
 func TestIsXLMetaFormatValid(t *testing.T) {
@@ -317,3 +321,221 @@ func TestGetPartSizeFromIdx(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkXlMetaV2Shallow(b *testing.B) {
+	fi := FileInfo{
+		Volume:           "volume",
+		Name:             "object-name",
+		VersionID:        "756100c6-b393-4981-928a-d49bbc164741",
+		IsLatest:         true,
+		Deleted:          false,
+		TransitionStatus: "PENDING",
+		DataDir:          "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef",
+		XLV1:             false,
+		ModTime:          time.Now(),
+		Size:             1234456,
+		Mode:             0,
+		Metadata: map[string]string{
+			xhttp.AmzRestore:                 "FAILED",
+			xhttp.ContentMD5:                 mustGetUUID(),
+			xhttp.AmzBucketReplicationStatus: "PENDING",
+			xhttp.ContentType:                "application/json",
+		},
+		Parts: []ObjectPartInfo{{
+			Number:     1,
+			Size:       1234345,
+			ActualSize: 1234345,
+		},
+			{
+				Number:     2,
+				Size:       1234345,
+				ActualSize: 1234345,
+			},
+		},
+		Erasure: ErasureInfo{
+			Algorithm:    ReedSolomon.String(),
+			DataBlocks:   4,
+			ParityBlocks: 2,
+			BlockSize:    10000,
+			Index:        1,
+			Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8},
+			Checksums: []ChecksumInfo{{
+				PartNumber: 1,
+				Algorithm:  HighwayHash256S,
+				Hash:       nil,
+			},
+				{
+					PartNumber: 2,
+					Algorithm:  HighwayHash256S,
+					Hash:       nil,
+				},
+			},
+		},
+	}
+	for _, size := range []int{1, 10, 1000, 100_000} {
+		b.Run(fmt.Sprint(size, "-versions"), func(b *testing.B) {
+			var xl xlMetaV2
+			ids := make([]string, size)
+			for i := 0; i < size; i++ {
+				fi.VersionID = mustGetUUID()
+				fi.DataDir = mustGetUUID()
+				ids[i] = fi.VersionID
+				fi.ModTime = fi.ModTime.Add(-time.Second)
+				xl.AddVersion(fi)
+			}
+			// Encode all. This is used for benchmarking.
+			enc, err := xl.AppendTo(nil)
+			if err != nil {
+				b.Fatal(err)
+			}
+			b.Logf("Serialized size: %d bytes", len(enc))
+			rng := rand.New(rand.NewSource(0))
+			var dump = make([]byte, len(enc))
+			b.Run("UpdateObjectVersion", func(b *testing.B) {
+				b.SetBytes(int64(size))
+				b.ResetTimer()
+				b.ReportAllocs()
+				for i := 0; i < b.N; i++ {
+					// Load...
+					xl = xlMetaV2{}
+					err := xl.Load(enc)
+					if err != nil {
+						b.Fatal(err)
+					}
+					// Update modtime for resorting...
+					fi.ModTime = fi.ModTime.Add(-time.Second)
+					// Update a random version.
+					fi.VersionID = ids[rng.Intn(size)]
+					// Update...
+					err = xl.UpdateObjectVersion(fi)
+					if err != nil {
+						b.Fatal(err)
+					}
+					// Save...
+					dump, err = xl.AppendTo(dump[:0])
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+			})
+			b.Run("DeleteVersion", func(b *testing.B) {
+				b.SetBytes(int64(size))
+				b.ResetTimer()
+				b.ReportAllocs()
+				for i := 0; i < b.N; i++ {
+					// Load...
+					xl = xlMetaV2{}
+					err := xl.Load(enc)
+					if err != nil {
+						b.Fatal(err)
+					}
+					// Update a random version.
+					fi.VersionID = ids[rng.Intn(size)]
+					// Delete...
+					_, _, err = xl.DeleteVersion(fi)
+					if err != nil {
+						b.Fatal(err)
+					}
+					// Save...
+					dump, err = xl.AppendTo(dump[:0])
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+			})
+			b.Run("AddVersion", func(b *testing.B) {
+				b.SetBytes(int64(size))
+				b.ResetTimer()
+				b.ReportAllocs()
+				for i := 0; i < b.N; i++ {
+					// Load...
+					xl = xlMetaV2{}
+					err := xl.Load(enc)
+					if err != nil {
+						b.Fatal(err)
+					}
+					// Update modtime for resorting...
+					fi.ModTime = fi.ModTime.Add(-time.Second)
+					// Update a random version.
+					fi.VersionID = mustGetUUID()
+					// Add...
+					err = xl.AddVersion(fi)
+					if err != nil {
+						b.Fatal(err)
+					}
+					// Save...
+					dump, err = xl.AppendTo(dump[:0])
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+			})
+			b.Run("ToFileInfo", func(b *testing.B) {
+				b.SetBytes(int64(size))
+				b.ResetTimer()
+				b.ReportAllocs()
+				for i := 0; i < b.N; i++ {
+					// Load...
+					xl = xlMetaV2{}
+					err := xl.Load(enc)
+					if err != nil {
+						b.Fatal(err)
+					}
+					// List...
+					_, err = xl.ToFileInfo("volume", "path", ids[rng.Intn(size)])
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+			})
+			b.Run("ListVersions", func(b *testing.B) {
+				b.SetBytes(int64(size))
+				b.ResetTimer()
+				b.ReportAllocs()
+				for i := 0; i < b.N; i++ {
+					// Load...
+					xl = xlMetaV2{}
+					err := xl.Load(enc)
+					if err != nil {
+						b.Fatal(err)
+					}
+					// List...
+					_, err = xl.ListVersions("volume", "path")
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+			})
+			b.Run("ToFileInfoNew", func(b *testing.B) {
+				b.SetBytes(int64(size))
+				b.ResetTimer()
+				b.ReportAllocs()
+				for i := 0; i < b.N; i++ {
+					buf, _ := isIndexedMetaV2(enc)
+					if buf == nil {
+						b.Fatal("buf == nil")
+					}
+					_, err = buf.ToFileInfo("volume", "path", ids[rng.Intn(size)])
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+			})
+			b.Run("ListVersionsNew", func(b *testing.B) {
+				b.SetBytes(int64(size))
+				b.ResetTimer()
+				b.ReportAllocs()
+				for i := 0; i < b.N; i++ {
+					buf, _ := isIndexedMetaV2(enc)
+					if buf == nil {
+						b.Fatal("buf == nil")
+					}
+					_, err = buf.ListVersions("volume", "path")
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+			})
+		})
+	}
+}

cmd/xl-storage-free-version.go

@@ -75,7 +75,7 @@ func (j xlMetaV2Version) FreeVersion() bool {
 
 // AddFreeVersion adds a free-version if needed for fi.VersionID version.
 // Free-version will be added if fi.VersionID has transitioned.
-func (z *xlMetaV2) AddFreeVersion(fi FileInfo) error {
+func (x *xlMetaV2) AddFreeVersion(fi FileInfo) error {
 	var uv uuid.UUID
 	var err error
 	switch fi.VersionID {
@@ -87,19 +87,22 @@ func (z *xlMetaV2) AddFreeVersion(fi FileInfo) error {
 		}
 	}
 
-	for _, version := range z.Versions {
+	for i, version := range x.versions {
-		switch version.Type {
+		if version.header.VersionID != uv || version.header.Type != ObjectType {
-		case ObjectType:
+			continue
-			if version.ObjectV2.VersionID == uv {
-				// if uv has tiered content we add a
-				// free-version to track it for asynchronous
-				// deletion via scanner.
-				if freeVersion, toFree := version.ObjectV2.InitFreeVersion(fi); toFree {
-					z.Versions = append(z.Versions, freeVersion)
-				}
-				return nil
-			}
 		}
+		// if uv has tiered content we add a
+		// free-version to track it for asynchronous
+		// deletion via scanner.
+		ver, err := x.getIdx(i)
+		if err != nil {
+			return err
+		}
+
+		if freeVersion, toFree := ver.ObjectV2.InitFreeVersion(fi); toFree {
+			return x.addVersion(freeVersion)
+		}
+		return nil
 	}
 	return nil
 }

cmd/xl-storage-free-version_test.go

@@ -24,8 +24,8 @@ import (
 	"github.com/minio/minio/internal/bucket/lifecycle"
 )
 
-func (z xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
+func (x xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
-	fivs, _, err := z.ListVersions(volume, path)
+	fivs, err := x.ListVersions(volume, path)
 	if err != nil {
 		return nil, err
 	}
@@ -41,8 +41,21 @@ func (z xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) {
 }
 
 func TestFreeVersion(t *testing.T) {
+	fatalErr := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
 	// Add a version with tiered content, one with local content
 	xl := xlMetaV2{}
+	counter := 1
+	report := func() {
+		t.Helper()
+		// t.Logf("versions (%d): len = %d", counter, len(xl.versions))
+		counter++
+	}
 	fi := FileInfo{
 		Volume:           "volume",
 		Name:             "object-name",
@@ -77,16 +90,21 @@ func TestFreeVersion(t *testing.T) {
 		SuccessorModTime: time.Time{},
 	}
 	// Add a version with local content
-	xl.AddVersion(fi)
+	fatalErr(xl.AddVersion(fi))
+	report()
 
 	// Add null version with tiered content
 	tierfi := fi
 	tierfi.VersionID = ""
-	xl.AddVersion(tierfi)
+	fatalErr(xl.AddVersion(tierfi))
+	report()
 	tierfi.TransitionStatus = lifecycle.TransitionComplete
 	tierfi.TransitionedObjName = mustGetUUID()
 	tierfi.TransitionTier = "MINIOTIER-1"
-	xl.DeleteVersion(tierfi)
+	var err error
+	_, _, err = xl.DeleteVersion(tierfi)
+	fatalErr(err)
+	report()
 
 	fvIDs := []string{
 		"00000000-0000-0000-0000-0000000000f1",
@@ -95,15 +113,20 @@ func TestFreeVersion(t *testing.T) {
 	// Simulate overwrite of null version
 	newtierfi := tierfi
 	newtierfi.SetTierFreeVersionID(fvIDs[0])
-	xl.AddFreeVersion(newtierfi)
+	fatalErr(xl.AddFreeVersion(newtierfi))
-	xl.AddVersion(newtierfi)
+	report()
+	fatalErr(xl.AddVersion(newtierfi))
+	report()
 
 	// Simulate removal of null version
 	newtierfi.TransitionTier = ""
 	newtierfi.TransitionedObjName = ""
 	newtierfi.TransitionStatus = ""
 	newtierfi.SetTierFreeVersionID(fvIDs[1])
-	xl.DeleteVersion(newtierfi)
+	report()
+	_, _, err = xl.DeleteVersion(newtierfi)
+	report()
+	fatalErr(err)
 
 	// Check number of free-versions
 	freeVersions, err := xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
@@ -118,8 +141,10 @@ func TestFreeVersion(t *testing.T) {
 	freefi := newtierfi
 	for _, fvID := range fvIDs {
 		freefi.VersionID = fvID
-		xl.DeleteVersion(freefi)
+		_, _, err = xl.DeleteVersion(freefi)
+		fatalErr(err)
 	}
+	report()
 
 	// Check number of free-versions
 	freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)
@@ -129,11 +154,13 @@ func TestFreeVersion(t *testing.T) {
 	if len(freeVersions) != 0 {
 		t.Fatalf("Expected zero free version but got %d", len(freeVersions))
 	}
+	report()
 
 	// Adding a free version to a version with no tiered content.
 	newfi := fi
 	newfi.SetTierFreeVersionID("00000000-0000-0000-0000-0000000000f3")
-	xl.AddFreeVersion(newfi) // this shouldn't add a free-version
+	fatalErr(xl.AddFreeVersion(newfi)) // this shouldn't add a free-version
+	report()
 
 	// Check number of free-versions
 	freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name)

cmd/xl-storage-meta-inline.go

@@ -0,0 +1,408 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/minio/minio/internal/logger"
+	"github.com/tinylib/msgp/msgp"
+)
+
+// xlMetaInlineData is serialized data in [string][]byte pairs.
+type xlMetaInlineData []byte
+
+// xlMetaInlineDataVer indicates the version of the inline data structure.
+const xlMetaInlineDataVer = 1
+
+// versionOK returns whether the version is ok.
+func (x xlMetaInlineData) versionOK() bool {
+	if len(x) == 0 {
+		return true
+	}
+	return x[0] > 0 && x[0] <= xlMetaInlineDataVer
+}
+
+// afterVersion returns the payload after the version, if any.
+func (x xlMetaInlineData) afterVersion() []byte {
+	if len(x) == 0 {
+		return x
+	}
+	return x[1:]
+}
+
+// find the data with key s.
+// Returns nil if not for or an error occurs.
+func (x xlMetaInlineData) find(key string) []byte {
+	if len(x) == 0 || !x.versionOK() {
+		return nil
+	}
+	sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
+	if err != nil || sz == 0 {
+		return nil
+	}
+	for i := uint32(0); i < sz; i++ {
+		var found []byte
+		found, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil || sz == 0 {
+			return nil
+		}
+		if string(found) == key {
+			val, _, _ := msgp.ReadBytesZC(buf)
+			return val
+		}
+		// Skip it
+		_, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			return nil
+		}
+	}
+	return nil
+}
+
+// validate checks if the data is valid.
+// It does not check integrity of the stored data.
+func (x xlMetaInlineData) validate() error {
+	if len(x) == 0 {
+		return nil
+	}
+
+	if !x.versionOK() {
+		return fmt.Errorf("xlMetaInlineData: unknown version 0x%x", x[0])
+	}
+
+	sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
+	if err != nil {
+		return fmt.Errorf("xlMetaInlineData: %w", err)
+	}
+
+	for i := uint32(0); i < sz; i++ {
+		var key []byte
+		key, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			return fmt.Errorf("xlMetaInlineData: %w", err)
+		}
+		if len(key) == 0 {
+			return fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
+		}
+		_, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			return fmt.Errorf("xlMetaInlineData: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// repair will copy all seemingly valid data entries from a corrupted set.
+// This does not ensure that data is correct, but will allow all operations to complete.
+func (x *xlMetaInlineData) repair() {
+	data := *x
+	if len(data) == 0 {
+		return
+	}
+
+	if !data.versionOK() {
+		*x = nil
+		return
+	}
+
+	sz, buf, err := msgp.ReadMapHeaderBytes(data.afterVersion())
+	if err != nil {
+		*x = nil
+		return
+	}
+
+	// Remove all current data
+	keys := make([][]byte, 0, sz)
+	vals := make([][]byte, 0, sz)
+	for i := uint32(0); i < sz; i++ {
+		var key, val []byte
+		key, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			break
+		}
+		if len(key) == 0 {
+			break
+		}
+		val, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			break
+		}
+		keys = append(keys, key)
+		vals = append(vals, val)
+	}
+	x.serialize(-1, keys, vals)
+}
+
+// validate checks if the data is valid.
+// It does not check integrity of the stored data.
+func (x xlMetaInlineData) list() ([]string, error) {
+	if len(x) == 0 {
+		return nil, nil
+	}
+	if !x.versionOK() {
+		return nil, errors.New("xlMetaInlineData: unknown version")
+	}
+
+	sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
+	if err != nil {
+		return nil, err
+	}
+	keys := make([]string, 0, sz)
+	for i := uint32(0); i < sz; i++ {
+		var key []byte
+		key, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			return keys, err
+		}
+		if len(key) == 0 {
+			return keys, fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
+		}
+		keys = append(keys, string(key))
+		// Skip data...
+		_, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			return keys, err
+		}
+	}
+	return keys, nil
+}
+
+// serialize will serialize the provided keys and values.
+// The function will panic if keys/value slices aren't of equal length.
+// Payload size can give an indication of expected payload size.
+// If plSize is <= 0 it will be calculated.
+func (x *xlMetaInlineData) serialize(plSize int, keys [][]byte, vals [][]byte) {
+	if len(keys) != len(vals) {
+		panic(fmt.Errorf("xlMetaInlineData.serialize: keys/value number mismatch"))
+	}
+	if len(keys) == 0 {
+		*x = nil
+		return
+	}
+	if plSize <= 0 {
+		plSize = 1 + msgp.MapHeaderSize
+		for i := range keys {
+			plSize += len(keys[i]) + len(vals[i]) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
+		}
+	}
+	payload := make([]byte, 1, plSize)
+	payload[0] = xlMetaInlineDataVer
+	payload = msgp.AppendMapHeader(payload, uint32(len(keys)))
+	for i := range keys {
+		payload = msgp.AppendStringFromBytes(payload, keys[i])
+		payload = msgp.AppendBytes(payload, vals[i])
+	}
+	*x = payload
+}
+
+// entries returns the number of entries in the data.
+func (x xlMetaInlineData) entries() int {
+	if len(x) == 0 || !x.versionOK() {
+		return 0
+	}
+	sz, _, _ := msgp.ReadMapHeaderBytes(x.afterVersion())
+	return int(sz)
+}
+
+// replace will add or replace a key/value pair.
+func (x *xlMetaInlineData) replace(key string, value []byte) {
+	in := x.afterVersion()
+	sz, buf, _ := msgp.ReadMapHeaderBytes(in)
+	keys := make([][]byte, 0, sz+1)
+	vals := make([][]byte, 0, sz+1)
+
+	// Version plus header...
+	plSize := 1 + msgp.MapHeaderSize
+	replaced := false
+	for i := uint32(0); i < sz; i++ {
+		var found, foundVal []byte
+		var err error
+		found, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			break
+		}
+		foundVal, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			break
+		}
+		plSize += len(found) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
+		keys = append(keys, found)
+		if string(found) == key {
+			vals = append(vals, value)
+			plSize += len(value)
+			replaced = true
+		} else {
+			vals = append(vals, foundVal)
+			plSize += len(foundVal)
+		}
+	}
+
+	// Add one more.
+	if !replaced {
+		keys = append(keys, []byte(key))
+		vals = append(vals, value)
+		plSize += len(key) + len(value) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
+	}
+
+	// Reserialize...
+	x.serialize(plSize, keys, vals)
+}
+
+// rename will rename a key.
+// Returns whether the key was found.
+func (x *xlMetaInlineData) rename(oldKey, newKey string) bool {
+	in := x.afterVersion()
+	sz, buf, _ := msgp.ReadMapHeaderBytes(in)
+	keys := make([][]byte, 0, sz)
+	vals := make([][]byte, 0, sz)
+
+	// Version plus header...
+	plSize := 1 + msgp.MapHeaderSize
+	found := false
+	for i := uint32(0); i < sz; i++ {
+		var foundKey, foundVal []byte
+		var err error
+		foundKey, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			break
+		}
+		foundVal, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			break
+		}
+		plSize += len(foundVal) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
+		vals = append(vals, foundVal)
+		if string(foundKey) != oldKey {
+			keys = append(keys, foundKey)
+			plSize += len(foundKey)
+		} else {
+			keys = append(keys, []byte(newKey))
+			plSize += len(newKey)
+			found = true
+		}
+	}
+	// If not found, just return.
+	if !found {
+		return false
+	}
+
+	// Reserialize...
+	x.serialize(plSize, keys, vals)
+	return true
+}
+
+// remove will remove one or more keys.
+// Returns true if any key was found.
+func (x *xlMetaInlineData) remove(keys ...string) bool {
+	in := x.afterVersion()
+	sz, buf, _ := msgp.ReadMapHeaderBytes(in)
+	newKeys := make([][]byte, 0, sz)
+	newVals := make([][]byte, 0, sz)
+	var removeKey func(s []byte) bool
+
+	// Copy if big number of compares...
+	if len(keys) > 5 && sz > 5 {
+		mKeys := make(map[string]struct{}, len(keys))
+		for _, key := range keys {
+			mKeys[key] = struct{}{}
+		}
+		removeKey = func(s []byte) bool {
+			_, ok := mKeys[string(s)]
+			return ok
+		}
+	} else {
+		removeKey = func(s []byte) bool {
+			for _, key := range keys {
+				if key == string(s) {
+					return true
+				}
+			}
+			return false
+		}
+	}
+
+	// Version plus header...
+	plSize := 1 + msgp.MapHeaderSize
+	found := false
+	for i := uint32(0); i < sz; i++ {
+		var foundKey, foundVal []byte
+		var err error
+		foundKey, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			break
+		}
+		foundVal, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			break
+		}
+		if !removeKey(foundKey) {
+			plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal)
+			newKeys = append(newKeys, foundKey)
+			newVals = append(newVals, foundVal)
+		} else {
+			found = true
+		}
+	}
+	// If not found, just return.
+	if !found {
+		return false
+	}
+	// If none left...
+	if len(newKeys) == 0 {
+		*x = nil
+		return true
+	}
+
+	// Reserialize...
+	x.serialize(plSize, newKeys, newVals)
+	return true
+}
+
+// xlMetaV2TrimData will trim any data from the metadata without unmarshalling it.
+// If any error occurs the unmodified data is returned.
+func xlMetaV2TrimData(buf []byte) []byte {
+	metaBuf, min, maj, err := checkXL2V1(buf)
+	if err != nil {
+		return buf
+	}
+	if maj == 1 && min < 1 {
+		// First version to carry data.
+		return buf
+	}
+	// Skip header
+	_, metaBuf, err = msgp.ReadBytesZC(metaBuf)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return buf
+	}
+	// Skip CRC
+	if maj > 1 || min >= 2 {
+		_, metaBuf, err = msgp.ReadUint32Bytes(metaBuf)
+		logger.LogIf(GlobalContext, err)
+	}
+	//   =  input - current pos
+	ends := len(buf) - len(metaBuf)
+	if ends > len(buf) {
+		return buf
+	}
+
+	return buf[:ends]
+}

cmd/xl-storage.go

@@ -978,7 +978,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 	}
 
 	var xlMeta xlMetaV2
-	if err = xlMeta.Load(buf); err != nil {
+	if err := xlMeta.Load(buf); err != nil {
 		return err
 	}
 
@@ -1044,6 +1044,7 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi
 		}
 		return err
 	}
+	defer metaDataPoolPut(buf)
 
 	if !isXL2V1Format(buf) {
 		return errFileVersionNotFound
@@ -1059,12 +1060,13 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi
 		return err
 	}
 
-	buf, err = xlMeta.AppendTo(nil)
+	wbuf, err := xlMeta.AppendTo(metaDataPoolGet())
 	if err != nil {
 		return err
 	}
+	defer metaDataPoolPut(wbuf)
 
-	return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf)
+	return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), wbuf)
 }
 
 // WriteMetadata - writes FileInfo metadata for path at `xl.meta`

docs/debugging/xl-meta/main.go

@@ -114,6 +114,48 @@ FLAGS:
 					return nil, err
 				}
 				data = b
+			case 3:
+				v, b, err := msgp.ReadBytesZC(b)
+				if err != nil {
+					return nil, err
+				}
+				if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil {
+					// Read metadata CRC (added in v2, ignore if not found)
+					b = nbuf
+				}
+
+				nVers, v, err := decodeXLHeaders(v)
+				if err != nil {
+					return nil, err
+				}
+				var versions = struct {
+					Versions []json.RawMessage
+					Headers  []json.RawMessage
+				}{
+					Versions: make([]json.RawMessage, nVers),
+					Headers:  make([]json.RawMessage, nVers),
+				}
+				err = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error {
+					var buf bytes.Buffer
+					if _, err := msgp.UnmarshalAsJSON(&buf, hdr); err != nil {
+						return err
+					}
+					versions.Headers[idx] = buf.Bytes()
+					buf = bytes.Buffer{}
+					if _, err := msgp.UnmarshalAsJSON(&buf, meta); err != nil {
+						return err
+					}
+					versions.Versions[idx] = buf.Bytes()
+					return nil
+				})
+				if err != nil {
+					return nil, err
+				}
+				enc := json.NewEncoder(buf)
+				if err := enc.Encode(versions); err != nil {
+					return nil, err
+				}
+				data = b
 			default:
 				return nil, fmt.Errorf("unknown metadata version %d", minor)
 			}
@@ -416,3 +458,54 @@ func (x xlMetaInlineData) files(fn func(name string, data []byte)) error {
 	return nil
 
 }
+
+const (
+	xlHeaderVersion = 2
+	xlMetaVersion   = 1
+)
+
+func decodeXLHeaders(buf []byte) (versions int, b []byte, err error) {
+	hdrVer, buf, err := msgp.ReadUintBytes(buf)
+	if err != nil {
+		return 0, buf, err
+	}
+	metaVer, buf, err := msgp.ReadUintBytes(buf)
+	if err != nil {
+		return 0, buf, err
+	}
+	if hdrVer > xlHeaderVersion {
+		return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", metaVer)
+	}
+	if metaVer > xlMetaVersion {
+		return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", metaVer)
+	}
+	versions, buf, err = msgp.ReadIntBytes(buf)
+	if err != nil {
+		return 0, buf, err
+	}
+	if versions < 0 {
+		return 0, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", versions)
+	}
+	return versions, buf, nil
+}
+
+// decodeVersions will decode a number of versions from a buffer
+// and perform a callback for each version in order, newest first.
+// Any non-nil error is returned.
+func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte) error) (err error) {
+	var tHdr, tMeta []byte // Zero copy bytes
+	for i := 0; i < versions; i++ {
+		tHdr, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			return err
+		}
+		tMeta, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			return err
+		}
+		if err = fn(i, tHdr, tMeta); err != nil {
+			return err
+		}
+	}
+	return nil
+}

go.mod

@@ -81,6 +81,7 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	github.com/yargevad/filepathx v1.0.0
+	github.com/zeebo/xxh3 v1.0.0
 	go.etcd.io/etcd/api/v3 v3.5.0
 	go.etcd.io/etcd/client/v3 v3.5.0
 	go.uber.org/atomic v1.9.0

go.sum

@@ -1527,6 +1527,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/zeebo/xxh3 v1.0.0 h1:6eLPZCVXpsGnhv8RiWBEJs5kenm2W1CMwon19/l8ODc=
+github.com/zeebo/xxh3 v1.0.0/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=