Improve performance on multiple versions (#13573)

Existing:

```go
type xlMetaV2 struct {
    Versions []xlMetaV2Version `json:"Versions" msg:"Versions"`
}
```

Serialized as regular MessagePack.

```go
//msgp:tuple xlMetaV2VersionHeader
type xlMetaV2VersionHeader struct {
	VersionID [16]byte
	ModTime   int64
	Type      VersionType
	Flags     xlFlags
}
```

Serialize as streaming MessagePack, format:

```
int(headerVersion)
int(xlmetaVersion)
int(nVersions)
for each version {
    binary blob, xlMetaV2VersionHeader, serialized
    binary blob, xlMetaV2Version, serialized.
}
```

xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct 
can easily be reused and does not contain pointers, so efficient as a 
slice (single allocation)

This allows quickly parsing everything as slices of bytes (no copy).

Versions are always *saved* sorted by modTime, newest *first*. 
No more need to sort on load.

* Allows checking if a version exists.
* Allows reading single version without unmarshal all.
* Allows reading latest version of type without unmarshal all.
* Allows reading latest version without unmarshal of all.
* Allows checking if the latest is deleteMarker by reading first entry.
* Allows adding/updating/deleting a version with only header deserialization.
* Reduces allocations on conversion to FileInfo(s).

cmd/xl-storage-format-v2_gen_test.go

@@ -9,8 +9,8 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
-func TestMarshalUnmarshalxlMetaV2(t *testing.T) {
-	v := xlMetaV2{}
+func TestMarshalUnmarshalxlMetaDataDirDecoder(t *testing.T) {
+	v := xlMetaDataDirDecoder{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
@@ -32,8 +32,8 @@ func TestMarshalUnmarshalxlMetaV2(t *testing.T) {
 	}
 }
 
-func BenchmarkMarshalMsgxlMetaV2(b *testing.B) {
-	v := xlMetaV2{}
+func BenchmarkMarshalMsgxlMetaDataDirDecoder(b *testing.B) {
+	v := xlMetaDataDirDecoder{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -41,8 +41,8 @@ func BenchmarkMarshalMsgxlMetaV2(b *testing.B) {
 	}
 }
 
-func BenchmarkAppendMsgxlMetaV2(b *testing.B) {
-	v := xlMetaV2{}
+func BenchmarkAppendMsgxlMetaDataDirDecoder(b *testing.B) {
+	v := xlMetaDataDirDecoder{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
@@ -53,8 +53,8 @@ func BenchmarkAppendMsgxlMetaV2(b *testing.B) {
 	}
 }
 
-func BenchmarkUnmarshalxlMetaV2(b *testing.B) {
-	v := xlMetaV2{}
+func BenchmarkUnmarshalxlMetaDataDirDecoder(b *testing.B) {
+	v := xlMetaDataDirDecoder{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
@@ -67,17 +67,17 @@ func BenchmarkUnmarshalxlMetaV2(b *testing.B) {
 	}
 }
 
-func TestEncodeDecodexlMetaV2(t *testing.T) {
-	v := xlMetaV2{}
+func TestEncodeDecodexlMetaDataDirDecoder(t *testing.T) {
+	v := xlMetaDataDirDecoder{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 
 	m := v.Msgsize()
 	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodexlMetaV2 Msgsize() is inaccurate")
+		t.Log("WARNING: TestEncodeDecodexlMetaDataDirDecoder Msgsize() is inaccurate")
 	}
 
-	vn := xlMetaV2{}
+	vn := xlMetaDataDirDecoder{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
@@ -91,8 +91,8 @@ func TestEncodeDecodexlMetaV2(t *testing.T) {
 	}
 }
 
-func BenchmarkEncodexlMetaV2(b *testing.B) {
-	v := xlMetaV2{}
+func BenchmarkEncodexlMetaDataDirDecoder(b *testing.B) {
+	v := xlMetaDataDirDecoder{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
@@ -105,8 +105,8 @@ func BenchmarkEncodexlMetaV2(b *testing.B) {
 	en.Flush()
 }
 
-func BenchmarkDecodexlMetaV2(b *testing.B) {
-	v := xlMetaV2{}
+func BenchmarkDecodexlMetaDataDirDecoder(b *testing.B) {
+	v := xlMetaDataDirDecoder{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
@@ -460,3 +460,116 @@ func BenchmarkDecodexlMetaV2Version(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalxlMetaV2VersionHeader(t *testing.T) {
+	v := xlMetaV2VersionHeader{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgxlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgxlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalxlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodexlMetaV2VersionHeader(t *testing.T) {
+	v := xlMetaV2VersionHeader{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodexlMetaV2VersionHeader Msgsize() is inaccurate")
+	}
+
+	vn := xlMetaV2VersionHeader{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodexlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodexlMetaV2VersionHeader(b *testing.B) {
+	v := xlMetaV2VersionHeader{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}