mirror of
https://github.com/minio/minio.git
synced 2025-11-08 21:24:55 -05:00
Improve listing consistency with version merging (#13723)
This commit is contained in:
Klaus Post
GitHub
@@ -18,9 +18,12 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Test_metaCacheEntries_sort(t *testing.T) {
|
||||
@@ -219,3 +222,438 @@ func Test_metaCacheEntry_isInDir(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_metaCacheEntries_resolve(t *testing.T) {
|
||||
baseTime, err := time.Parse("2006/01/02", "2015/02/25")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var inputs = []xlMetaV2{
|
||||
0: {
|
||||
versions: []xlMetaV2ShallowVersion{
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
ModTime: baseTime.Add(30 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{1, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
// Mismatches Modtime+Signature and older...
|
||||
1: {
|
||||
versions: []xlMetaV2ShallowVersion{
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
ModTime: baseTime.Add(15 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{2, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
// Has another version prior to the one we want.
|
||||
2: {
|
||||
versions: []xlMetaV2ShallowVersion{
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
ModTime: baseTime.Add(60 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{2, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
ModTime: baseTime.Add(30 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{1, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
// Has a completely different version id
|
||||
3: {
|
||||
versions: []xlMetaV2ShallowVersion{
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
ModTime: baseTime.Add(60 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{1, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
4: {
|
||||
versions: []xlMetaV2ShallowVersion{},
|
||||
},
|
||||
// Has a zero version id
|
||||
5: {
|
||||
versions: []xlMetaV2ShallowVersion{
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{},
|
||||
ModTime: baseTime.Add(60 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{5, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
// Zero version, modtime newer..
|
||||
6: {
|
||||
versions: []xlMetaV2ShallowVersion{
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{},
|
||||
ModTime: baseTime.Add(90 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{6, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
7: {
|
||||
versions: []xlMetaV2ShallowVersion{
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{},
|
||||
ModTime: baseTime.Add(90 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{6, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
ModTime: baseTime.Add(60 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{2, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
ModTime: baseTime.Add(60 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{1, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
ModTime: baseTime.Add(30 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{1, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
// Delete marker.
|
||||
8: {
|
||||
versions: []xlMetaV2ShallowVersion{
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7},
|
||||
ModTime: baseTime.Add(90 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{6, 1, 1, 1},
|
||||
Type: DeleteType,
|
||||
Flags: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
// Delete marker and version from 1
|
||||
9: {
|
||||
versions: []xlMetaV2ShallowVersion{
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7},
|
||||
ModTime: baseTime.Add(90 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{6, 1, 1, 1},
|
||||
Type: DeleteType,
|
||||
Flags: 0,
|
||||
}},
|
||||
{header: xlMetaV2VersionHeader{
|
||||
VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
ModTime: baseTime.Add(15 * time.Minute).UnixNano(),
|
||||
Signature: [4]byte{2, 1, 1, 1},
|
||||
Type: ObjectType,
|
||||
Flags: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
inputSerialized := make([]metaCacheEntry, len(inputs))
|
||||
for i, xl := range inputs {
|
||||
xl.sortByModTime()
|
||||
var err error
|
||||
var entry = metaCacheEntry{
|
||||
name: "testobject",
|
||||
}
|
||||
entry.metadata, err = xl.AppendTo(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputSerialized[i] = entry
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
m metaCacheEntries
|
||||
r metadataResolutionParams
|
||||
wantSelected *metaCacheEntry
|
||||
wantOk bool
|
||||
}{
|
||||
{
|
||||
name: "consistent",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[0]},
|
||||
r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "consistent-strict",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[0]},
|
||||
r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "one zero, below quorum",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}},
|
||||
r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
|
||||
wantSelected: nil,
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "one zero, below quorum, strict",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}},
|
||||
r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: true},
|
||||
wantSelected: nil,
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "one zero, at quorum",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}},
|
||||
r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "one zero, at quorum, strict",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}},
|
||||
r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: true},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "modtime, signature mismatch",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]},
|
||||
r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "modtime,signature mismatch, strict",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]},
|
||||
r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: true},
|
||||
wantSelected: nil,
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "modtime, signature mismatch, at quorum",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]},
|
||||
r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "modtime,signature mismatch, at quorum, strict",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]},
|
||||
r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: true},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "additional version",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]},
|
||||
r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
// Since we have the same version in all inputs, that is strictly ok.
|
||||
name: "additional version, strict",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]},
|
||||
r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: true},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
// Since we have the same version in all inputs, that is strictly ok.
|
||||
name: "additional version, quorum one",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]},
|
||||
r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: true},
|
||||
// We get the both versions, since we only request quorum 1
|
||||
wantSelected: &inputSerialized[2],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "additional version, quorum two",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]},
|
||||
r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "2 additional versions, quorum two",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[2], inputSerialized[2]},
|
||||
r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true},
|
||||
wantSelected: &inputSerialized[2],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
// inputSerialized[1] have older versions of the second in inputSerialized[2]
|
||||
name: "modtimemismatch",
|
||||
m: metaCacheEntries{inputSerialized[1], inputSerialized[1], inputSerialized[2], inputSerialized[2]},
|
||||
r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
|
||||
wantSelected: &inputSerialized[2],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
// inputSerialized[1] have older versions of the second in inputSerialized[2]
|
||||
name: "modtimemismatch,strict",
|
||||
m: metaCacheEntries{inputSerialized[1], inputSerialized[1], inputSerialized[2], inputSerialized[2]},
|
||||
r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true},
|
||||
wantSelected: &inputSerialized[2],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
// inputSerialized[1] have older versions of the second in inputSerialized[2], but
|
||||
// since it is not strict, we should get it that one (with latest modtime)
|
||||
name: "modtimemismatch,not strict",
|
||||
m: metaCacheEntries{inputSerialized[1], inputSerialized[1], inputSerialized[2], inputSerialized[2]},
|
||||
r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "one-q1",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
|
||||
r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: false},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "one-q1-strict",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
|
||||
r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: true},
|
||||
wantSelected: &inputSerialized[0],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "one-q2",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
|
||||
r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
|
||||
wantSelected: nil,
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "one-q2-strict",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
|
||||
r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true},
|
||||
wantSelected: nil,
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "two-diff-q2",
|
||||
m: metaCacheEntries{inputSerialized[0], inputSerialized[3], inputSerialized[4], inputSerialized[4]},
|
||||
r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
|
||||
wantSelected: nil,
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "zeroid",
|
||||
m: metaCacheEntries{inputSerialized[5], inputSerialized[5], inputSerialized[6], inputSerialized[6]},
|
||||
r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
|
||||
wantSelected: &inputSerialized[6],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
// When ID is zero, do not allow non-strict matches to reach quorum.
|
||||
name: "zeroid-belowq",
|
||||
m: metaCacheEntries{inputSerialized[5], inputSerialized[5], inputSerialized[6], inputSerialized[6]},
|
||||
r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
|
||||
wantSelected: nil,
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "merge4",
|
||||
m: metaCacheEntries{inputSerialized[2], inputSerialized[3], inputSerialized[5], inputSerialized[6]},
|
||||
r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: false},
|
||||
wantSelected: &inputSerialized[7],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "deletemarker",
|
||||
m: metaCacheEntries{inputSerialized[8], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
|
||||
r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: false},
|
||||
wantSelected: &inputSerialized[8],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "deletemarker-nonq",
|
||||
m: metaCacheEntries{inputSerialized[8], inputSerialized[8], inputSerialized[4], inputSerialized[4]},
|
||||
r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
|
||||
wantSelected: nil,
|
||||
wantOk: false,
|
||||
},
|
||||
{
|
||||
name: "deletemarker-nonq",
|
||||
m: metaCacheEntries{inputSerialized[8], inputSerialized[8], inputSerialized[8], inputSerialized[1]},
|
||||
r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
|
||||
wantSelected: &inputSerialized[8],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "deletemarker-mixed",
|
||||
m: metaCacheEntries{inputSerialized[8], inputSerialized[8], inputSerialized[1], inputSerialized[1]},
|
||||
r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
|
||||
wantSelected: &inputSerialized[9],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "deletemarker-q3",
|
||||
m: metaCacheEntries{inputSerialized[8], inputSerialized[9], inputSerialized[9], inputSerialized[1]},
|
||||
r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
|
||||
wantSelected: &inputSerialized[9],
|
||||
wantOk: true,
|
||||
},
|
||||
{
|
||||
name: "deletemarker-q3-strict",
|
||||
m: metaCacheEntries{inputSerialized[8], inputSerialized[9], inputSerialized[9], inputSerialized[1]},
|
||||
r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: true},
|
||||
wantSelected: &inputSerialized[9],
|
||||
wantOk: true,
|
||||
},
|
||||
}
|
||||
|
||||
for testID, tt := range tests {
|
||||
rng := rand.New(rand.NewSource(0))
|
||||
// Run for a number of times, shuffling the input to ensure that output is consistent.
|
||||
for i := 0; i < 10; i++ {
|
||||
t.Run(fmt.Sprintf("test-%d-%s-run-%d", testID, tt.name, i), func(t *testing.T) {
|
||||
if i > 0 {
|
||||
rng.Shuffle(len(tt.m), func(i, j int) {
|
||||
tt.m[i], tt.m[j] = tt.m[j], tt.m[i]
|
||||
})
|
||||
}
|
||||
gotSelected, gotOk := tt.m.resolve(&tt.r)
|
||||
if gotOk != tt.wantOk {
|
||||
t.Errorf("resolve() gotOk = %v, want %v", gotOk, tt.wantOk)
|
||||
}
|
||||
if gotSelected != nil {
|
||||
gotSelected.cached = nil
|
||||
gotSelected.reusable = false
|
||||
}
|
||||
if !reflect.DeepEqual(gotSelected, tt.wantSelected) {
|
||||
wantM, _ := tt.wantSelected.xlmeta()
|
||||
gotM, _ := gotSelected.xlmeta()
|
||||
t.Errorf("resolve() gotSelected = \n%#v, want \n%#v", *gotM, *wantM)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user