mirror of
https://github.com/minio/minio.git
synced 2025-11-07 21:02:58 -05:00
Improve performance on multiple versions (#13573)
Existing:
```go
type xlMetaV2 struct {
Versions []xlMetaV2Version `json:"Versions" msg:"Versions"`
}
```
Serialized as regular MessagePack.
```go
//msgp:tuple xlMetaV2VersionHeader
type xlMetaV2VersionHeader struct {
VersionID [16]byte
ModTime int64
Type VersionType
Flags xlFlags
}
```
Serialize as streaming MessagePack, format:
```
int(headerVersion)
int(xlmetaVersion)
int(nVersions)
for each version {
binary blob, xlMetaV2VersionHeader, serialized
binary blob, xlMetaV2Version, serialized.
}
```
xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct
can easily be reused and does not contain pointers, so efficient as a
slice (single allocation)
This allows quickly parsing everything as slices of bytes (no copy).
Versions are always *saved* sorted by modTime, newest *first*.
No more need to sort on load.
* Allows checking if a version exists.
* Allows reading single version without unmarshal all.
* Allows reading latest version of type without unmarshal all.
* Allows reading latest version without unmarshal of all.
* Allows checking if the latest is deleteMarker by reading first entry.
* Allows adding/updating/deleting a version with only header deserialization.
* Reduces allocations on conversion to FileInfo(s).
This commit is contained in:
93
cmd/xl-storage-format-utils_test.go
Normal file
93
cmd/xl-storage-format-utils_test.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
func Test_hashDeterministicString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
arg map[string]string
|
||||
}{
|
||||
{
|
||||
name: "zero",
|
||||
arg: map[string]string{},
|
||||
},
|
||||
{
|
||||
name: "nil",
|
||||
arg: nil,
|
||||
},
|
||||
{
|
||||
name: "one",
|
||||
arg: map[string]string{"key": "value"},
|
||||
},
|
||||
{
|
||||
name: "several",
|
||||
arg: map[string]string{
|
||||
xhttp.AmzRestore: "FAILED",
|
||||
xhttp.ContentMD5: mustGetUUID(),
|
||||
xhttp.AmzBucketReplicationStatus: "PENDING",
|
||||
xhttp.ContentType: "application/json",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "someempty",
|
||||
arg: map[string]string{
|
||||
xhttp.AmzRestore: "",
|
||||
xhttp.ContentMD5: mustGetUUID(),
|
||||
xhttp.AmzBucketReplicationStatus: "",
|
||||
xhttp.ContentType: "application/json",
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
const n = 100
|
||||
want := hashDeterministicString(tt.arg)
|
||||
m := tt.arg
|
||||
for i := 0; i < n; i++ {
|
||||
if got := hashDeterministicString(m); got != want {
|
||||
t.Errorf("hashDeterministicString() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
// Check casual collisions
|
||||
if m == nil {
|
||||
m = make(map[string]string)
|
||||
}
|
||||
m["12312312"] = ""
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
want = hashDeterministicString(m)
|
||||
delete(m, "12312312")
|
||||
m["another"] = ""
|
||||
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
|
||||
want = hashDeterministicString(m)
|
||||
m["another"] = "hashDeterministicString"
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
|
||||
want = hashDeterministicString(m)
|
||||
m["another"] = "hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString"
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
|
||||
// Flip key/value
|
||||
want = hashDeterministicString(m)
|
||||
delete(m, "another")
|
||||
m["hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString"] = "another"
|
||||
if got := hashDeterministicString(m); got == want {
|
||||
t.Errorf("hashDeterministicString() = %v, does not want %v", got, want)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user