mirror of
https://github.com/minio/minio.git
synced 2024-12-25 22:55:54 -05:00
3995355150
objects with 10,000 parts and many of them can cause a large memory spike which can potentially lead to OOM due to lack of GC. with previous PR reducing the memory usage significantly in #17963, this PR reduces this further by 80% under repeated calls. Scanner sub-system has no use for the slice of Parts(), it is better left empty. ``` benchmark old ns/op new ns/op delta BenchmarkToFileInfo/ToFileInfo-8 295658 188143 -36.36% benchmark old allocs new allocs delta BenchmarkToFileInfo/ToFileInfo-8 61 60 -1.64% benchmark old bytes new bytes delta BenchmarkToFileInfo/ToFileInfo-8 1097210 227255 -79.29% ```
1099 lines
35 KiB
Go
1099 lines
35 KiB
Go
// Copyright (c) 2015-2021 MinIO, Inc.
|
|
//
|
|
// This file is part of MinIO Object Storage stack
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"math/rand"
|
|
"os"
|
|
"reflect"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/klauspost/compress/zip"
|
|
"github.com/klauspost/compress/zstd"
|
|
"github.com/minio/minio/internal/bucket/lifecycle"
|
|
xhttp "github.com/minio/minio/internal/http"
|
|
)
|
|
|
|
func TestReadXLMetaNoData(t *testing.T) {
|
|
f, err := os.Open("testdata/xl.meta-corrupt.gz")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
gz, err := gzip.NewReader(bufio.NewReader(f))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
buf, err := io.ReadAll(gz)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, err = readXLMetaNoData(bytes.NewReader(buf), int64(len(buf)))
|
|
if err == nil {
|
|
t.Fatal("expected error but returned success")
|
|
}
|
|
}
|
|
|
|
func TestXLV2FormatData(t *testing.T) {
|
|
failOnErr := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
data := []byte("some object data")
|
|
data2 := []byte("some other object data")
|
|
|
|
xl := xlMetaV2{}
|
|
fi := FileInfo{
|
|
Volume: "volume",
|
|
Name: "object-name",
|
|
VersionID: "756100c6-b393-4981-928a-d49bbc164741",
|
|
IsLatest: true,
|
|
Deleted: false,
|
|
TransitionStatus: "",
|
|
DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef",
|
|
XLV1: false,
|
|
ModTime: time.Now(),
|
|
Size: 0,
|
|
Mode: 0,
|
|
Metadata: nil,
|
|
Parts: nil,
|
|
Erasure: ErasureInfo{
|
|
Algorithm: ReedSolomon.String(),
|
|
DataBlocks: 4,
|
|
ParityBlocks: 2,
|
|
BlockSize: 10000,
|
|
Index: 1,
|
|
Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8},
|
|
Checksums: []ChecksumInfo{{
|
|
PartNumber: 1,
|
|
Algorithm: HighwayHash256S,
|
|
Hash: nil,
|
|
}},
|
|
},
|
|
MarkDeleted: false,
|
|
Data: data,
|
|
NumVersions: 1,
|
|
SuccessorModTime: time.Time{},
|
|
}
|
|
|
|
failOnErr(xl.AddVersion(fi))
|
|
|
|
fi.VersionID = mustGetUUID()
|
|
fi.DataDir = mustGetUUID()
|
|
fi.Data = data2
|
|
failOnErr(xl.AddVersion(fi))
|
|
|
|
serialized, err := xl.AppendTo(nil)
|
|
failOnErr(err)
|
|
// Roundtrip data
|
|
var xl2 xlMetaV2
|
|
failOnErr(xl2.Load(serialized))
|
|
|
|
// We should have one data entry
|
|
list, err := xl2.data.list()
|
|
failOnErr(err)
|
|
if len(list) != 2 {
|
|
t.Fatalf("want 1 entry, got %d", len(list))
|
|
}
|
|
|
|
if !bytes.Equal(xl2.data.find("756100c6-b393-4981-928a-d49bbc164741"), data) {
|
|
t.Fatal("Find data returned", xl2.data.find("756100c6-b393-4981-928a-d49bbc164741"))
|
|
}
|
|
if !bytes.Equal(xl2.data.find(fi.VersionID), data2) {
|
|
t.Fatal("Find data returned", xl2.data.find(fi.VersionID))
|
|
}
|
|
|
|
// Remove entry
|
|
xl2.data.remove(fi.VersionID)
|
|
failOnErr(xl2.data.validate())
|
|
if xl2.data.find(fi.VersionID) != nil {
|
|
t.Fatal("Data was not removed:", xl2.data.find(fi.VersionID))
|
|
}
|
|
if xl2.data.entries() != 1 {
|
|
t.Fatal("want 1 entry, got", xl2.data.entries())
|
|
}
|
|
// Re-add
|
|
xl2.data.replace(fi.VersionID, fi.Data)
|
|
failOnErr(xl2.data.validate())
|
|
if xl2.data.entries() != 2 {
|
|
t.Fatal("want 2 entries, got", xl2.data.entries())
|
|
}
|
|
|
|
// Replace entry
|
|
xl2.data.replace("756100c6-b393-4981-928a-d49bbc164741", data2)
|
|
failOnErr(xl2.data.validate())
|
|
if xl2.data.entries() != 2 {
|
|
t.Fatal("want 2 entries, got", xl2.data.entries())
|
|
}
|
|
if !bytes.Equal(xl2.data.find("756100c6-b393-4981-928a-d49bbc164741"), data2) {
|
|
t.Fatal("Find data returned", xl2.data.find("756100c6-b393-4981-928a-d49bbc164741"))
|
|
}
|
|
|
|
if !xl2.data.rename("756100c6-b393-4981-928a-d49bbc164741", "new-key") {
|
|
t.Fatal("old key was not found")
|
|
}
|
|
failOnErr(xl2.data.validate())
|
|
if !bytes.Equal(xl2.data.find("new-key"), data2) {
|
|
t.Fatal("Find data returned", xl2.data.find("756100c6-b393-4981-928a-d49bbc164741"))
|
|
}
|
|
if xl2.data.entries() != 2 {
|
|
t.Fatal("want 2 entries, got", xl2.data.entries())
|
|
}
|
|
if !bytes.Equal(xl2.data.find(fi.VersionID), data2) {
|
|
t.Fatal("Find data returned", xl2.data.find(fi.DataDir))
|
|
}
|
|
|
|
// Test trimmed
|
|
xl2 = xlMetaV2{}
|
|
trimmed := xlMetaV2TrimData(serialized)
|
|
failOnErr(xl2.Load(trimmed))
|
|
if len(xl2.data) != 0 {
|
|
t.Fatal("data, was not trimmed, bytes left:", len(xl2.data))
|
|
}
|
|
// Corrupt metadata, last 5 bytes is the checksum, so go a bit further back.
|
|
trimmed[len(trimmed)-10] += 10
|
|
if err := xl2.Load(trimmed); err == nil {
|
|
t.Fatal("metadata corruption not detected")
|
|
}
|
|
}
|
|
|
|
// TestUsesDataDir tests xlMetaV2.UsesDataDir
|
|
func TestUsesDataDir(t *testing.T) {
|
|
vID := uuid.New()
|
|
dataDir := uuid.New()
|
|
transitioned := make(map[string][]byte)
|
|
transitioned[ReservedMetadataPrefixLower+TransitionStatus] = []byte(lifecycle.TransitionComplete)
|
|
|
|
toBeRestored := make(map[string]string)
|
|
toBeRestored[xhttp.AmzRestore] = ongoingRestoreObj().String()
|
|
|
|
restored := make(map[string]string)
|
|
restored[xhttp.AmzRestore] = completedRestoreObj(time.Now().UTC().Add(time.Hour)).String()
|
|
|
|
restoredExpired := make(map[string]string)
|
|
restoredExpired[xhttp.AmzRestore] = completedRestoreObj(time.Now().UTC().Add(-time.Hour)).String()
|
|
|
|
testCases := []struct {
|
|
xlmeta xlMetaV2Object
|
|
uses bool
|
|
}{
|
|
{ // transitioned object version
|
|
xlmeta: xlMetaV2Object{
|
|
VersionID: vID,
|
|
DataDir: dataDir,
|
|
MetaSys: transitioned,
|
|
},
|
|
uses: false,
|
|
},
|
|
{ // to be restored (requires object version to be transitioned)
|
|
xlmeta: xlMetaV2Object{
|
|
VersionID: vID,
|
|
DataDir: dataDir,
|
|
MetaSys: transitioned,
|
|
MetaUser: toBeRestored,
|
|
},
|
|
uses: false,
|
|
},
|
|
{ // restored object version (requires object version to be transitioned)
|
|
xlmeta: xlMetaV2Object{
|
|
VersionID: vID,
|
|
DataDir: dataDir,
|
|
MetaSys: transitioned,
|
|
MetaUser: restored,
|
|
},
|
|
uses: true,
|
|
},
|
|
{ // restored object version expired an hour back (requires object version to be transitioned)
|
|
xlmeta: xlMetaV2Object{
|
|
VersionID: vID,
|
|
DataDir: dataDir,
|
|
MetaSys: transitioned,
|
|
MetaUser: restoredExpired,
|
|
},
|
|
uses: false,
|
|
},
|
|
{ // object version with no ILM applied
|
|
xlmeta: xlMetaV2Object{
|
|
VersionID: vID,
|
|
DataDir: dataDir,
|
|
},
|
|
uses: true,
|
|
},
|
|
}
|
|
for i, tc := range testCases {
|
|
if got := tc.xlmeta.UsesDataDir(); got != tc.uses {
|
|
t.Fatalf("Test %d: Expected %v but got %v for %v", i+1, tc.uses, got, tc.xlmeta)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDeleteVersionWithSharedDataDir(t *testing.T) {
|
|
failOnErr := func(i int, err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatalf("Test %d: failed with %v", i, err)
|
|
}
|
|
}
|
|
|
|
data := []byte("some object data")
|
|
data2 := []byte("some other object data")
|
|
|
|
xl := xlMetaV2{}
|
|
fi := FileInfo{
|
|
Volume: "volume",
|
|
Name: "object-name",
|
|
VersionID: "756100c6-b393-4981-928a-d49bbc164741",
|
|
IsLatest: true,
|
|
Deleted: false,
|
|
TransitionStatus: "",
|
|
DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef",
|
|
XLV1: false,
|
|
ModTime: time.Now(),
|
|
Size: 0,
|
|
Mode: 0,
|
|
Metadata: nil,
|
|
Parts: nil,
|
|
Erasure: ErasureInfo{
|
|
Algorithm: ReedSolomon.String(),
|
|
DataBlocks: 4,
|
|
ParityBlocks: 2,
|
|
BlockSize: 10000,
|
|
Index: 1,
|
|
Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8},
|
|
Checksums: []ChecksumInfo{{
|
|
PartNumber: 1,
|
|
Algorithm: HighwayHash256S,
|
|
Hash: nil,
|
|
}},
|
|
},
|
|
MarkDeleted: false,
|
|
Data: data,
|
|
NumVersions: 1,
|
|
SuccessorModTime: time.Time{},
|
|
}
|
|
|
|
d0, d1, d2 := mustGetUUID(), mustGetUUID(), mustGetUUID()
|
|
testCases := []struct {
|
|
versionID string
|
|
dataDir string
|
|
data []byte
|
|
shares int
|
|
transitionStatus string
|
|
restoreObjStatus string
|
|
expireRestored bool
|
|
expectedDataDir string
|
|
}{
|
|
{ // object versions with inlined data don't count towards shared data directory
|
|
versionID: mustGetUUID(),
|
|
dataDir: d0,
|
|
data: data,
|
|
shares: 0,
|
|
},
|
|
{ // object versions with inlined data don't count towards shared data directory
|
|
versionID: mustGetUUID(),
|
|
dataDir: d1,
|
|
data: data2,
|
|
shares: 0,
|
|
},
|
|
{ // transitioned object version don't count towards shared data directory
|
|
versionID: mustGetUUID(),
|
|
dataDir: d2,
|
|
shares: 3,
|
|
transitionStatus: lifecycle.TransitionComplete,
|
|
},
|
|
{ // transitioned object version with an ongoing restore-object request.
|
|
versionID: mustGetUUID(),
|
|
dataDir: d2,
|
|
shares: 3,
|
|
transitionStatus: lifecycle.TransitionComplete,
|
|
restoreObjStatus: ongoingRestoreObj().String(),
|
|
},
|
|
// The following versions are on-disk.
|
|
{ // restored object version expiring 10 hours from now.
|
|
versionID: mustGetUUID(),
|
|
dataDir: d2,
|
|
shares: 2,
|
|
transitionStatus: lifecycle.TransitionComplete,
|
|
restoreObjStatus: completedRestoreObj(time.Now().Add(10 * time.Hour)).String(),
|
|
expireRestored: true,
|
|
},
|
|
{
|
|
versionID: mustGetUUID(),
|
|
dataDir: d2,
|
|
shares: 2,
|
|
},
|
|
{
|
|
versionID: mustGetUUID(),
|
|
dataDir: d2,
|
|
shares: 2,
|
|
expectedDataDir: d2,
|
|
},
|
|
}
|
|
|
|
var fileInfos []FileInfo
|
|
for i, tc := range testCases {
|
|
fi := fi
|
|
fi.VersionID = tc.versionID
|
|
fi.DataDir = tc.dataDir
|
|
fi.Data = tc.data
|
|
if tc.data == nil {
|
|
fi.Size = 42 // to prevent inlining of data
|
|
}
|
|
if tc.restoreObjStatus != "" {
|
|
fi.Metadata = map[string]string{
|
|
xhttp.AmzRestore: tc.restoreObjStatus,
|
|
}
|
|
}
|
|
fi.TransitionStatus = tc.transitionStatus
|
|
fi.ModTime = fi.ModTime.Add(time.Duration(i) * time.Second)
|
|
failOnErr(i+1, xl.AddVersion(fi))
|
|
fi.ExpireRestored = tc.expireRestored
|
|
fileInfos = append(fileInfos, fi)
|
|
}
|
|
|
|
for i, tc := range testCases {
|
|
_, version, err := xl.findVersion(uuid.MustParse(tc.versionID))
|
|
failOnErr(i+1, err)
|
|
if got := xl.SharedDataDirCount(version.getVersionID(), version.ObjectV2.DataDir); got != tc.shares {
|
|
t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2.VersionID, tc.shares, got)
|
|
}
|
|
}
|
|
|
|
// Deleting fileInfos[4].VersionID, fileInfos[5].VersionID should return empty data dir; there are other object version sharing the data dir.
|
|
// Subsequently deleting fileInfos[6].versionID should return fileInfos[6].dataDir since there are no other object versions sharing this data dir.
|
|
count := len(testCases)
|
|
for i := 4; i < len(testCases); i++ {
|
|
tc := testCases[i]
|
|
dataDir, err := xl.DeleteVersion(fileInfos[i])
|
|
failOnErr(count+1, err)
|
|
if dataDir != tc.expectedDataDir {
|
|
t.Fatalf("Expected %s but got %s", tc.expectedDataDir, dataDir)
|
|
}
|
|
count++
|
|
}
|
|
}
|
|
|
|
func Benchmark_mergeXLV2Versions(b *testing.B) {
|
|
data, err := os.ReadFile("testdata/xl.meta-v1.2.zst")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
dec, _ := zstd.NewReader(nil)
|
|
data, err = dec.DecodeAll(data, nil)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
var xl xlMetaV2
|
|
if err = xl.LoadOrConvert(data); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
vers := make([][]xlMetaV2ShallowVersion, 16)
|
|
for i := range vers {
|
|
vers[i] = xl.versions
|
|
}
|
|
|
|
b.Run("requested-none", func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
b.SetBytes(855) // number of versions...
|
|
for i := 0; i < b.N; i++ {
|
|
mergeXLV2Versions(8, false, 0, vers...)
|
|
}
|
|
})
|
|
|
|
b.Run("requested-v1", func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
b.SetBytes(855) // number of versions...
|
|
for i := 0; i < b.N; i++ {
|
|
mergeXLV2Versions(8, false, 1, vers...)
|
|
}
|
|
})
|
|
|
|
b.Run("requested-v2", func(b *testing.B) {
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
b.SetBytes(855) // number of versions...
|
|
for i := 0; i < b.N; i++ {
|
|
mergeXLV2Versions(8, false, 1, vers...)
|
|
}
|
|
})
|
|
}
|
|
|
|
func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
|
|
data, err := os.ReadFile("testdata/xl.meta-v1.2.zst")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
dec, _ := zstd.NewReader(nil)
|
|
data, err = dec.DecodeAll(data, nil)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
b.Run("legacy", func(b *testing.B) {
|
|
var xl xlMetaV2
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
b.SetBytes(855) // number of versions...
|
|
for i := 0; i < b.N; i++ {
|
|
err = xl.Load(data)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
|
|
b.Run("indexed", func(b *testing.B) {
|
|
var xl xlMetaV2
|
|
err = xl.Load(data)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
data, err := xl.AppendTo(nil)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
b.SetBytes(855) // number of versions...
|
|
for i := 0; i < b.N; i++ {
|
|
err = xl.Load(data)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func Test_xlMetaV2Shallow_Load(t *testing.T) {
|
|
// Load Legacy
|
|
data, err := os.ReadFile("testdata/xl.meta-v1.2.zst")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dec, _ := zstd.NewReader(nil)
|
|
data, err = dec.DecodeAll(data, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
test := func(t *testing.T, xl *xlMetaV2) {
|
|
if len(xl.versions) != 855 {
|
|
t.Errorf("want %d versions, got %d", 855, len(xl.versions))
|
|
}
|
|
xl.sortByModTime()
|
|
if !sort.SliceIsSorted(xl.versions, func(i, j int) bool {
|
|
return xl.versions[i].header.ModTime > xl.versions[j].header.ModTime
|
|
}) {
|
|
t.Errorf("Contents not sorted")
|
|
}
|
|
for i := range xl.versions {
|
|
hdr := xl.versions[i].header
|
|
ver, err := xl.getIdx(i)
|
|
if err != nil {
|
|
t.Error(err)
|
|
continue
|
|
}
|
|
gotHdr := ver.header()
|
|
if hdr != gotHdr {
|
|
t.Errorf("Header does not match, index: %+v != meta: %+v", hdr, gotHdr)
|
|
}
|
|
}
|
|
}
|
|
t.Run("load-legacy", func(t *testing.T) {
|
|
var xl xlMetaV2
|
|
err = xl.Load(data)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
test(t, &xl)
|
|
})
|
|
t.Run("roundtrip", func(t *testing.T) {
|
|
var xl xlMetaV2
|
|
err = xl.Load(data)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
data, err = xl.AppendTo(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
xl = xlMetaV2{}
|
|
err = xl.Load(data)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
test(t, &xl)
|
|
})
|
|
t.Run("write-timestamp", func(t *testing.T) {
|
|
var xl xlMetaV2
|
|
err = xl.Load(data)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ventry := xlMetaV2Version{
|
|
Type: DeleteType,
|
|
DeleteMarker: &xlMetaV2DeleteMarker{
|
|
VersionID: uuid.New(),
|
|
ModTime: time.Now().UnixNano(),
|
|
MetaSys: map[string][]byte{ReservedMetadataPrefixLower + ReplicationTimestamp: []byte("2022-10-27T15:40:53.195813291+08:00"), ReservedMetadataPrefixLower + ReplicaTimestamp: []byte("2022-10-27T15:40:53.195813291+08:00")},
|
|
},
|
|
WrittenByVersion: globalVersionUnix,
|
|
}
|
|
xl.data = nil
|
|
xl.versions = xl.versions[:2]
|
|
xl.addVersion(ventry)
|
|
|
|
data, err = xl.AppendTo(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// t.Logf("data := %#v\n", data)
|
|
})
|
|
}
|
|
|
|
func Test_xlMetaV2Shallow_LoadTimeStamp(t *testing.T) {
|
|
// v0 Saved with
|
|
// ReservedMetadataPrefixLower + ReplicationTimestamp: []byte("2022-10-27T15:40:53.195813291+08:00")
|
|
// ReservedMetadataPrefixLower + ReplicaTimestamp: []byte("2022-10-27T15:40:53.195813291+08:00")
|
|
data := []byte{0x58, 0x4c, 0x32, 0x20, 0x1, 0x0, 0x3, 0x0, 0xc6, 0x0, 0x0, 0x3, 0xe8, 0x2, 0x1, 0x3, 0xc4, 0x24, 0x95, 0xc4, 0x10, 0x6a, 0x33, 0x51, 0xe3, 0x48, 0x0, 0x4e, 0xed, 0xab, 0x9, 0x21, 0xeb, 0x83, 0x5a, 0xc6, 0x45, 0xd3, 0x17, 0x21, 0xe7, 0xa1, 0x98, 0x52, 0x88, 0xc, 0xc4, 0x4, 0xfe, 0x54, 0xbc, 0x2f, 0x2, 0x0, 0xc4, 0xd5, 0x83, 0xa4, 0x54, 0x79, 0x70, 0x65, 0x2, 0xa6, 0x44, 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x83, 0xa2, 0x49, 0x44, 0xc4, 0x10, 0x6a, 0x33, 0x51, 0xe3, 0x48, 0x0, 0x4e, 0xed, 0xab, 0x9, 0x21, 0xeb, 0x83, 0x5a, 0xc6, 0x45, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65, 0xd3, 0x17, 0x21, 0xe7, 0xa1, 0x98, 0x52, 0x88, 0xc, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73, 0x82, 0xd9, 0x26, 0x78, 0x2d, 0x6d, 0x69, 0x6e, 0x69, 0x6f, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0xc4, 0x23, 0x32, 0x30, 0x32, 0x32, 0x2d, 0x31, 0x30, 0x2d, 0x32, 0x37, 0x54, 0x31, 0x35, 0x3a, 0x34, 0x30, 0x3a, 0x35, 0x33, 0x2e, 0x31, 0x39, 0x35, 0x38, 0x31, 0x33, 0x32, 0x39, 0x31, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0xd9, 0x22, 0x78, 0x2d, 0x6d, 0x69, 0x6e, 0x69, 0x6f, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x2d, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0xc4, 0x23, 0x32, 0x30, 0x32, 0x32, 0x2d, 0x31, 0x30, 0x2d, 0x32, 0x37, 0x54, 0x31, 0x35, 0x3a, 0x34, 0x30, 0x3a, 0x35, 0x33, 0x2e, 0x31, 0x39, 0x35, 0x38, 0x31, 0x33, 0x32, 0x39, 0x31, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0xa1, 0x76, 0x0, 0xc4, 0x24, 0x95, 0xc4, 0x10, 0xdb, 0xaf, 0x9a, 0xe8, 0xda, 0xe0, 0x40, 0xa6, 0x80, 0xf2, 0x1c, 0x39, 0xe8, 0x7, 0x38, 0x2c, 0xd3, 0x16, 0xb2, 0x7f, 0xfe, 0x45, 0x1c, 0xf, 0x98, 0xc4, 0x4, 0xea, 0x49, 0x78, 0x27, 0x1, 0x6, 0xc5, 0x1, 0x4b, 0x82, 0xa4, 0x54, 0x79, 0x70, 0x65, 0x1, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a, 0xde, 0x0, 0x11, 0xa2, 0x49, 0x44, 0xc4, 0x10, 0xdb, 0xaf, 0x9a, 0xe8, 0xda, 0xe0, 0x40, 0xa6, 0x80, 0xf2, 0x1c, 0x39, 0xe8, 0x7, 0x38, 0x2c, 0xa4, 0x44, 0x44, 0x69, 0x72, 0xc4, 0x10, 0x4f, 0xe7, 0xd7, 0xf3, 0x15, 0x2d, 0x4c, 0xcd, 0x96, 0x65, 0x24, 0x86, 0xdc, 0x2, 0x1b, 0xed, 0xa6, 0x45, 0x63, 0x41, 0x6c, 0x67, 0x6f, 0x1, 0xa3, 0x45, 0x63, 0x4d, 0x4, 0xa3, 0x45, 0x63, 0x4e, 0x4, 0xa7, 0x45, 0x63, 0x42, 0x53, 0x69, 0x7a, 0x65, 0xd2, 0x0, 0x10, 0x0, 0x0, 0xa7, 0x45, 0x63, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x7, 0xa6, 0x45, 0x63, 0x44, 0x69, 0x73, 0x74, 0x98, 0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0xa8, 0x43, 0x53, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f, 0x1, 0xa8, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x73, 0x91, 0x1, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73, 0x91, 0xa0, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x91, 0xd1, 0x1, 0x0, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x91, 0xd1, 0x1, 0x0, 0xa4, 0x53, 0x69, 0x7a, 0x65, 0xd1, 0x1, 0x0, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65, 0xd3, 0x16, 0xb2, 0x7f, 0xfe, 0x45, 0x1c, 0xf, 0x98, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73, 0x81, 0xbc, 0x78, 0x2d, 0x6d, 0x69, 0x6e, 0x69, 0x6f, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2d, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x2d, 0x64, 0x61, 0x74, 0x61, 0xc4, 0x4, 0x74, 0x72, 0x75, 0x65, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72, 0x82, 0xac, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x74, 0x79, 0x70, 0x65, 0xb8, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6f, 0x63, 0x74, 0x65, 0x74, 0x2d, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0xa4, 0x65, 0x74, 0x61, 0x67, 0xd9, 0x20, 0x62, 0x65, 0x65, 0x62, 0x32, 0x32, 0x30, 0x66, 0x61, 0x39, 0x64, 0x65, 0x36, 0x31, 0x36, 0x36, 0x62, 0x64, 0x31, 0x66, 0x66, 0x37, 0x32, 0x33, 0x66, 0x35, 0x62, 0x31, 0x66, 0x31, 0x39, 0x66, 0xc4, 0x24, 0x95, 0xc4, 0x10, 0xe6, 0xb4, 0xa8, 0xe0, 0xc6, 0x3d, 0x40, 0x7a, 0x87, 0x61, 0xf2, 0x43, 0x41, 0xdf, 0xcb, 0x93, 0xd3, 0x16, 0xb2, 0x7f, 0xfe, 0x3e, 0x43, 0xf2, 0xf8, 0xc4, 0x4, 0x91, 0xd9, 0xee, 0x2e, 0x1, 0x6, 0xc5, 0x1, 0x4b, 0x82, 0xa4, 0x54, 0x79, 0x70, 0x65, 0x1, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a, 0xde, 0x0, 0x11, 0xa2, 0x49, 0x44, 0xc4, 0x10, 0xe6, 0xb4, 0xa8, 0xe0, 0xc6, 0x3d, 0x40, 0x7a, 0x87, 0x61, 0xf2, 0x43, 0x41, 0xdf, 0xcb, 0x93, 0xa4, 0x44, 0x44, 0x69, 0x72, 0xc4, 0x10, 0xfd, 0x76, 0xe, 0x16, 0xfc, 0x17, 0x4d, 0x38, 0xbc, 0xac, 0x18, 0xde, 0x36, 0xae, 0xd7, 0x48, 0xa6, 0x45, 0x63, 0x41, 0x6c, 0x67, 0x6f, 0x1, 0xa3, 0x45, 0x63, 0x4d, 0x4, 0xa3, 0x45, 0x63, 0x4e, 0x4, 0xa7, 0x45, 0x63, 0x42, 0x53, 0x69, 0x7a, 0x65, 0xd2, 0x0, 0x10, 0x0, 0x0, 0xa7, 0x45, 0x63, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x7, 0xa6, 0x45, 0x63, 0x44, 0x69, 0x73, 0x74, 0x98, 0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0xa8, 0x43, 0x53, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f, 0x1, 0xa8, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x73, 0x91, 0x1, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73, 0x91, 0xa0, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x91, 0xd1, 0x1, 0x0, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x91, 0xd1, 0x1, 0x0, 0xa4, 0x53, 0x69, 0x7a, 0x65, 0xd1, 0x1, 0x0, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65, 0xd3, 0x16, 0xb2, 0x7f, 0xfe, 0x3e, 0x43, 0xf2, 0xf8, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73, 0x81, 0xbc, 0x78, 0x2d, 0x6d, 0x69, 0x6e, 0x69, 0x6f, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2d, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x2d, 0x64, 0x61, 0x74, 0x61, 0xc4, 0x4, 0x74, 0x72, 0x75, 0x65, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72, 0x82, 0xac, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x74, 0x79, 0x70, 0x65, 0xb8, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6f, 0x63, 0x74, 0x65, 0x74, 0x2d, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0xa4, 0x65, 0x74, 0x61, 0x67, 0xd9, 0x20, 0x36, 0x64, 0x31, 0x31, 0x61, 0x34, 0x35, 0x31, 0x62, 0x30, 0x62, 0x32, 0x34, 0x35, 0x65, 0x32, 0x61, 0x34, 0x35, 0x65, 0x34, 0x64, 0x31, 0x36, 0x66, 0x63, 0x33, 0x39, 0x62, 0x62, 0x61, 0x61, 0xce, 0x78, 0x57, 0x89, 0xc4}
|
|
var xl xlMetaV2
|
|
err := xl.Load(data)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v0 := xl.versions[0]
|
|
|
|
// Saved with signature 0xfe, 0x54, 0xbc, 0x2f
|
|
// Signature must be converted after load.
|
|
wantSig := [4]byte{0x1e, 0x5f, 0xba, 0x4a}
|
|
if v0.header.Signature != wantSig {
|
|
t.Errorf("Wrong signature, want %#v, got %#v", wantSig, v0.header.Signature)
|
|
}
|
|
v, err := xl.getIdx(0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantTimeStamp := "2022-10-27T07:40:53.195813291Z"
|
|
got := string(v.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp])
|
|
if wantTimeStamp != got {
|
|
t.Errorf("Wrong timestamp, want %v, got %v", wantTimeStamp, got)
|
|
}
|
|
got = string(v.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp])
|
|
if wantTimeStamp != got {
|
|
t.Errorf("Wrong timestamp, want %v, got %v", wantTimeStamp, got)
|
|
}
|
|
}
|
|
|
|
func Test_mergeXLV2Versions(t *testing.T) {
|
|
dataZ, err := os.ReadFile("testdata/xl-meta-consist.zip")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var vers [][]xlMetaV2ShallowVersion
|
|
zr, err := zip.NewReader(bytes.NewReader(dataZ), int64(len(dataZ)))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, file := range zr.File {
|
|
if file.UncompressedSize64 == 0 {
|
|
continue
|
|
}
|
|
in, err := file.Open()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer in.Close()
|
|
buf, err := io.ReadAll(in)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var xl xlMetaV2
|
|
err = xl.LoadOrConvert(buf)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
vers = append(vers, xl.versions)
|
|
}
|
|
for _, v2 := range vers {
|
|
for _, ver := range v2 {
|
|
b, _ := json.Marshal(ver.header)
|
|
t.Log(string(b))
|
|
var x xlMetaV2Version
|
|
_, _ = x.unmarshalV(0, ver.meta)
|
|
b, _ = json.Marshal(x)
|
|
t.Log(string(b), x.getSignature())
|
|
}
|
|
}
|
|
|
|
for i := range vers {
|
|
t.Run(fmt.Sprintf("non-strict-q%d", i), func(t *testing.T) {
|
|
merged := mergeXLV2Versions(i, false, 0, vers...)
|
|
if len(merged) == 0 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
t.Run(fmt.Sprintf("strict-q%d", i), func(t *testing.T) {
|
|
merged := mergeXLV2Versions(i, true, 0, vers...)
|
|
if len(merged) == 0 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
t.Run(fmt.Sprintf("signature-q%d", i), func(t *testing.T) {
|
|
// Mutate signature, non strict
|
|
vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
|
|
for i, ver := range vers {
|
|
newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
|
|
for _, v := range ver {
|
|
v.header.Signature = [4]byte{byte(i + 10), 0, 0, 0}
|
|
newVers = append(newVers, v)
|
|
}
|
|
vMod = append(vMod, newVers)
|
|
}
|
|
merged := mergeXLV2Versions(i, false, 0, vMod...)
|
|
if len(merged) == 0 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
t.Run(fmt.Sprintf("modtime-q%d", i), func(t *testing.T) {
|
|
// Mutate modtime, but rest is consistent.
|
|
vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
|
|
for i, ver := range vers {
|
|
newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
|
|
for _, v := range ver {
|
|
v.header.ModTime += int64(i)
|
|
newVers = append(newVers, v)
|
|
}
|
|
vMod = append(vMod, newVers)
|
|
}
|
|
merged := mergeXLV2Versions(i, false, 0, vMod...)
|
|
if len(merged) == 0 && i < 2 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
if len(merged) > 0 && i >= 2 {
|
|
t.Error("Got unexpected results")
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
t.Run(fmt.Sprintf("flags-q%d", i), func(t *testing.T) {
|
|
// Mutate signature, non strict
|
|
vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
|
|
for i, ver := range vers {
|
|
newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
|
|
for _, v := range ver {
|
|
v.header.Flags += xlFlags(i)
|
|
newVers = append(newVers, v)
|
|
}
|
|
vMod = append(vMod, newVers)
|
|
}
|
|
merged := mergeXLV2Versions(i, false, 0, vMod...)
|
|
if len(merged) == 0 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
t.Run(fmt.Sprintf("versionid-q%d", i), func(t *testing.T) {
|
|
// Mutate signature, non strict
|
|
vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
|
|
for i, ver := range vers {
|
|
newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
|
|
for _, v := range ver {
|
|
v.header.VersionID[0] += byte(i)
|
|
newVers = append(newVers, v)
|
|
}
|
|
vMod = append(vMod, newVers)
|
|
}
|
|
merged := mergeXLV2Versions(i, false, 0, vMod...)
|
|
if len(merged) == 0 && i < 2 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
if len(merged) > 0 && i >= 2 {
|
|
t.Error("Got unexpected results")
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
t.Run(fmt.Sprintf("strict-signature-q%d", i), func(t *testing.T) {
|
|
// Mutate signature, non strict
|
|
vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
|
|
for i, ver := range vers {
|
|
newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
|
|
for _, v := range ver {
|
|
v.header.Signature = [4]byte{byte(i + 10), 0, 0, 0}
|
|
newVers = append(newVers, v)
|
|
}
|
|
vMod = append(vMod, newVers)
|
|
}
|
|
merged := mergeXLV2Versions(i, true, 0, vMod...)
|
|
if len(merged) == 0 && i < 2 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
if len(merged) > 0 && i >= 2 {
|
|
t.Error("Got unexpected results")
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
t.Run(fmt.Sprintf("strict-modtime-q%d", i), func(t *testing.T) {
|
|
// Mutate signature, non strict
|
|
vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
|
|
for i, ver := range vers {
|
|
newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
|
|
for _, v := range ver {
|
|
v.header.ModTime += int64(i + 10)
|
|
newVers = append(newVers, v)
|
|
}
|
|
vMod = append(vMod, newVers)
|
|
}
|
|
merged := mergeXLV2Versions(i, true, 0, vMod...)
|
|
if len(merged) == 0 && i < 2 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
if len(merged) > 0 && i >= 2 {
|
|
t.Error("Got unexpected results", len(merged), merged[0].header)
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
t.Run(fmt.Sprintf("strict-flags-q%d", i), func(t *testing.T) {
|
|
// Mutate signature, non strict
|
|
vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
|
|
for i, ver := range vers {
|
|
newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
|
|
for _, v := range ver {
|
|
v.header.Flags += xlFlags(i + 10)
|
|
newVers = append(newVers, v)
|
|
}
|
|
vMod = append(vMod, newVers)
|
|
}
|
|
merged := mergeXLV2Versions(i, true, 0, vMod...)
|
|
if len(merged) == 0 && i < 2 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
if len(merged) > 0 && i >= 2 {
|
|
t.Error("Got unexpected results", len(merged))
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
t.Run(fmt.Sprintf("strict-type-q%d", i), func(t *testing.T) {
|
|
// Mutate signature, non strict
|
|
vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
|
|
for i, ver := range vers {
|
|
newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
|
|
for _, v := range ver {
|
|
v.header.Type += VersionType(i + 10)
|
|
newVers = append(newVers, v)
|
|
}
|
|
vMod = append(vMod, newVers)
|
|
}
|
|
merged := mergeXLV2Versions(i, true, 0, vMod...)
|
|
if len(merged) == 0 && i < 2 {
|
|
t.Error("Did not get any results")
|
|
return
|
|
}
|
|
if len(merged) > 0 && i >= 2 {
|
|
t.Error("Got unexpected results", len(merged))
|
|
return
|
|
}
|
|
for _, ver := range merged {
|
|
if ver.header.Type == invalidVersionType {
|
|
t.Errorf("Invalid result returned: %v", ver.header)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_mergeXLV2Versions2(t *testing.T) {
|
|
vDelMarker := xlMetaV2ShallowVersion{header: xlMetaV2VersionHeader{
|
|
VersionID: [16]byte{2},
|
|
ModTime: 1500,
|
|
Signature: [4]byte{5, 6, 7, 8},
|
|
Type: DeleteType,
|
|
Flags: 0,
|
|
}}
|
|
vDelMarker.meta, _ = base64.StdEncoding.DecodeString("gqRUeXBlAqZEZWxPYmqDoklExBCvwGEaY+BAO4B4vyG5ERorpU1UaW1l0xbgJlsWE9IHp01ldGFTeXOA")
|
|
|
|
vObj := xlMetaV2ShallowVersion{header: xlMetaV2VersionHeader{
|
|
VersionID: [16]byte{1},
|
|
ModTime: 1000,
|
|
Signature: [4]byte{1, 2, 3, 4},
|
|
Type: ObjectType,
|
|
Flags: xlFlagUsesDataDir | xlFlagInlineData,
|
|
}}
|
|
vObj.meta, _ = base64.StdEncoding.DecodeString("gqRUeXBlAaVWMk9iat4AEaJJRMQQEkaOteYCSrWB3nqppSIKTqRERGlyxBAO8fXSJ5RI+YEtsp8KneVVpkVjQWxnbwGjRWNNDKNFY04Ep0VjQlNpemXSABAAAKdFY0luZGV4BaZFY0Rpc3TcABAFBgcICQoLDA0ODxABAgMEqENTdW1BbGdvAahQYXJ0TnVtc5EBqVBhcnRFVGFnc8CpUGFydFNpemVzkdEBL6pQYXJ0QVNpemVzkdEBL6RTaXpl0QEvpU1UaW1l0xbgJhIa6ABvp01ldGFTeXOBvHgtbWluaW8taW50ZXJuYWwtaW5saW5lLWRhdGHEBHRydWWnTWV0YVVzcoKsY29udGVudC10eXBluGFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbaRldGFn2SBlYTIxMDE2MmVlYjRhZGMzMWZmOTg0Y2I3NDRkNmFmNg==")
|
|
|
|
testCases := []struct {
|
|
name string
|
|
input [][]xlMetaV2ShallowVersion
|
|
quorum int
|
|
reqVersions int
|
|
want []xlMetaV2ShallowVersion
|
|
}{
|
|
{
|
|
name: "obj-on-one",
|
|
input: [][]xlMetaV2ShallowVersion{
|
|
0: {vDelMarker, vObj}, // disk 0
|
|
1: {vDelMarker}, // disk 1
|
|
2: {vDelMarker}, // disk 2
|
|
},
|
|
quorum: 2,
|
|
reqVersions: 0,
|
|
want: []xlMetaV2ShallowVersion{vDelMarker},
|
|
},
|
|
{
|
|
name: "obj-on-two",
|
|
input: [][]xlMetaV2ShallowVersion{
|
|
0: {vDelMarker, vObj}, // disk 0
|
|
1: {vDelMarker, vObj}, // disk 1
|
|
2: {vDelMarker}, // disk 2
|
|
},
|
|
quorum: 2,
|
|
reqVersions: 0,
|
|
want: []xlMetaV2ShallowVersion{vDelMarker, vObj},
|
|
},
|
|
{
|
|
name: "obj-on-all",
|
|
input: [][]xlMetaV2ShallowVersion{
|
|
0: {vDelMarker, vObj}, // disk 0
|
|
1: {vDelMarker, vObj}, // disk 1
|
|
2: {vDelMarker, vObj}, // disk 2
|
|
},
|
|
quorum: 2,
|
|
reqVersions: 0,
|
|
want: []xlMetaV2ShallowVersion{vDelMarker, vObj},
|
|
},
|
|
{
|
|
name: "del-on-one",
|
|
input: [][]xlMetaV2ShallowVersion{
|
|
0: {vDelMarker, vObj}, // disk 0
|
|
1: {vObj}, // disk 1
|
|
2: {vObj}, // disk 2
|
|
},
|
|
quorum: 2,
|
|
reqVersions: 0,
|
|
want: []xlMetaV2ShallowVersion{vObj},
|
|
},
|
|
{
|
|
name: "del-on-two",
|
|
input: [][]xlMetaV2ShallowVersion{
|
|
0: {vDelMarker, vObj}, // disk 0
|
|
1: {vDelMarker, vObj}, // disk 1
|
|
2: {vObj}, // disk 2
|
|
},
|
|
quorum: 2,
|
|
reqVersions: 0,
|
|
want: []xlMetaV2ShallowVersion{vDelMarker, vObj},
|
|
},
|
|
{
|
|
name: "del-on-two-16stripe",
|
|
input: [][]xlMetaV2ShallowVersion{
|
|
0: {vObj}, // disk 0
|
|
1: {vDelMarker, vObj}, // disk 1
|
|
2: {vDelMarker, vObj}, // disk 2
|
|
3: {vDelMarker, vObj}, // disk 3
|
|
4: {vDelMarker, vObj}, // disk 4
|
|
5: {vDelMarker, vObj}, // disk 5
|
|
6: {vDelMarker, vObj}, // disk 6
|
|
7: {vDelMarker, vObj}, // disk 7
|
|
8: {vDelMarker, vObj}, // disk 8
|
|
9: {vDelMarker, vObj}, // disk 9
|
|
10: {vObj}, // disk 10
|
|
11: {vDelMarker, vObj}, // disk 11
|
|
12: {vDelMarker, vObj}, // disk 12
|
|
13: {vDelMarker, vObj}, // disk 13
|
|
14: {vDelMarker, vObj}, // disk 14
|
|
15: {vDelMarker, vObj}, // disk 15
|
|
},
|
|
quorum: 7,
|
|
reqVersions: 0,
|
|
want: []xlMetaV2ShallowVersion{vDelMarker, vObj},
|
|
},
|
|
}
|
|
for _, test := range testCases {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
// Run multiple times, shuffling the input order.
|
|
for i := int64(0); i < 50; i++ {
|
|
t.Run(fmt.Sprint(i), func(t *testing.T) {
|
|
rng := rand.New(rand.NewSource(i))
|
|
rng.Shuffle(len(test.input), func(i, j int) {
|
|
test.input[i], test.input[j] = test.input[j], test.input[i]
|
|
})
|
|
got := mergeXLV2Versions(test.quorum, true, 0, test.input...)
|
|
if !reflect.DeepEqual(test.want, got) {
|
|
t.Errorf("want %v != got %v", test.want, got)
|
|
}
|
|
})
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestXMinIOHealingSkip(t *testing.T) {
|
|
xl := xlMetaV2{}
|
|
failOnErr := func(err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatalf("Test failed with %v", err)
|
|
}
|
|
}
|
|
|
|
fi := FileInfo{
|
|
Volume: "volume",
|
|
Name: "object-name",
|
|
VersionID: "756100c6-b393-4981-928a-d49bbc164741",
|
|
IsLatest: true,
|
|
Deleted: false,
|
|
ModTime: time.Now(),
|
|
Size: 1 << 10,
|
|
Mode: 0,
|
|
Erasure: ErasureInfo{
|
|
Algorithm: ReedSolomon.String(),
|
|
DataBlocks: 4,
|
|
ParityBlocks: 2,
|
|
BlockSize: 10000,
|
|
Index: 1,
|
|
Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8},
|
|
Checksums: []ChecksumInfo{{
|
|
PartNumber: 1,
|
|
Algorithm: HighwayHash256S,
|
|
Hash: nil,
|
|
}},
|
|
},
|
|
NumVersions: 1,
|
|
}
|
|
|
|
fi.SetHealing()
|
|
failOnErr(xl.AddVersion(fi))
|
|
|
|
var err error
|
|
fi, err = xl.ToFileInfo(fi.Volume, fi.Name, fi.VersionID, false, true)
|
|
if err != nil {
|
|
t.Fatalf("xl.ToFileInfo failed with %v", err)
|
|
}
|
|
|
|
if fi.Healing() {
|
|
t.Fatal("Expected fi.Healing to be false")
|
|
}
|
|
}
|
|
|
|
func benchmarkManyPartsOptionally(b *testing.B, allParts bool) {
|
|
f, err := os.Open("testdata/xl-many-parts.meta")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
data, err := io.ReadAll(f)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
buf, _, _ := isIndexedMetaV2(data)
|
|
if buf == nil {
|
|
b.Fatal("buf == nil")
|
|
}
|
|
|
|
b.Run("ToFileInfo", func(b *testing.B) {
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
_, err = buf.ToFileInfo("volume", "path", "", allParts)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func BenchmarkToFileInfoNoParts(b *testing.B) {
|
|
benchmarkManyPartsOptionally(b, false)
|
|
}
|
|
|
|
func BenchmarkToFileInfoWithParts(b *testing.B) {
|
|
benchmarkManyPartsOptionally(b, true)
|
|
}
|