Fix listing in objects split across pools (#19227)

Merging same-object - multiple versions from different pools would not always result in correct ordering.

When merging keep inputs separate.

```
λ mc ls --versions local/testbucket
------ before ------

[2024-03-05 20:17:19 CET]   228B STANDARD 1f163718-9bc5-4b01-bff7-5d8cf09caf10 v3 PUT hosts
[2024-03-05 20:19:56 CET]  19KiB STANDARD null v2 PUT hosts
[2024-03-05 20:17:15 CET]   228B STANDARD 73c9f651-f023-4566-b012-cc537fdb7ce2 v1 PUT hosts

------ after ------
λ mc ls --versions local/testbucket
[2024-03-05 20:19:56 CET]  19KiB STANDARD null v3 PUT hosts
[2024-03-05 20:17:19 CET]   228B STANDARD 1f163718-9bc5-4b01-bff7-5d8cf09caf10 v2 PUT hosts
[2024-03-05 20:17:15 CET]   228B STANDARD 73c9f651-f023-4566-b012-cc537fdb7ce2 v1 PUT hosts
```
This commit is contained in:
Klaus Post
2024-03-08 18:50:48 +01:00
committed by GitHub
parent 1787bcfc91
commit 650efc2e96
4 changed files with 78 additions and 4 deletions

View File

@@ -21,6 +21,7 @@ import (
"bufio"
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"encoding/json"
"fmt"
@@ -1011,6 +1012,77 @@ func Test_mergeXLV2Versions2(t *testing.T) {
}
}
func Test_mergeEntryChannels(t *testing.T) {
dataZ, err := os.ReadFile("testdata/xl-meta-merge.zip")
if err != nil {
t.Fatal(err)
}
var vers []metaCacheEntry
zr, err := zip.NewReader(bytes.NewReader(dataZ), int64(len(dataZ)))
if err != nil {
t.Fatal(err)
}
for _, file := range zr.File {
if file.UncompressedSize64 == 0 {
continue
}
in, err := file.Open()
if err != nil {
t.Fatal(err)
}
defer in.Close()
buf, err := io.ReadAll(in)
if err != nil {
t.Fatal(err)
}
buf = xlMetaV2TrimData(buf)
vers = append(vers, metaCacheEntry{
name: "a",
metadata: buf,
})
}
// Shuffle...
for i := 0; i < 100; i++ {
rng := rand.New(rand.NewSource(int64(i)))
rng.Shuffle(len(vers), func(i, j int) {
vers[i], vers[j] = vers[j], vers[i]
})
var entries []chan metaCacheEntry
for _, v := range vers {
v.cached = nil
ch := make(chan metaCacheEntry, 1)
ch <- v
close(ch)
entries = append(entries, ch)
}
out := make(chan metaCacheEntry, 1)
err := mergeEntryChannels(context.Background(), entries, out, 1)
if err != nil {
t.Fatal(err)
}
got, ok := <-out
if !ok {
t.Fatal("Got no result")
}
xl, err := got.xlmeta()
if err != nil {
t.Fatal(err)
}
if len(xl.versions) != 3 {
t.Fatal("Got wrong number of versions, want 3, got", len(xl.versions))
}
if !sort.SliceIsSorted(xl.versions, func(i, j int) bool {
return xl.versions[i].header.sortsBefore(xl.versions[j].header)
}) {
t.Errorf("Got unsorted result")
}
}
}
func TestXMinIOHealingSkip(t *testing.T) {
xl := xlMetaV2{}
failOnErr := func(err error) {