mirror of
https://github.com/minio/minio.git
synced 2025-11-07 12:52:58 -05:00
Improve performance on multiple versions (#13573)
Existing:
```go
type xlMetaV2 struct {
Versions []xlMetaV2Version `json:"Versions" msg:"Versions"`
}
```
Serialized as regular MessagePack.
```go
//msgp:tuple xlMetaV2VersionHeader
type xlMetaV2VersionHeader struct {
VersionID [16]byte
ModTime int64
Type VersionType
Flags xlFlags
}
```
Serialize as streaming MessagePack, format:
```
int(headerVersion)
int(xlmetaVersion)
int(nVersions)
for each version {
binary blob, xlMetaV2VersionHeader, serialized
binary blob, xlMetaV2Version, serialized.
}
```
xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct
can easily be reused and does not contain pointers, so efficient as a
slice (single allocation)
This allows quickly parsing everything as slices of bytes (no copy).
Versions are always *saved* sorted by modTime, newest *first*.
No more need to sort on load.
* Allows checking if a version exists.
* Allows reading single version without unmarshal all.
* Allows reading latest version of type without unmarshal all.
* Allows reading latest version without unmarshal of all.
* Allows checking if the latest is deleteMarker by reading first entry.
* Allows adding/updating/deleting a version with only header deserialization.
* Reduces allocations on conversion to FileInfo(s).
This commit is contained in:
Klaus Post
GitHub
408
cmd/xl-storage-meta-inline.go
Normal file
408
cmd/xl-storage-meta-inline.go
Normal file
@@ -0,0 +1,408 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
// xlMetaInlineData is serialized data in [string][]byte pairs.
|
||||
type xlMetaInlineData []byte
|
||||
|
||||
// xlMetaInlineDataVer indicates the version of the inline data structure.
|
||||
const xlMetaInlineDataVer = 1
|
||||
|
||||
// versionOK returns whether the version is ok.
|
||||
func (x xlMetaInlineData) versionOK() bool {
|
||||
if len(x) == 0 {
|
||||
return true
|
||||
}
|
||||
return x[0] > 0 && x[0] <= xlMetaInlineDataVer
|
||||
}
|
||||
|
||||
// afterVersion returns the payload after the version, if any.
|
||||
func (x xlMetaInlineData) afterVersion() []byte {
|
||||
if len(x) == 0 {
|
||||
return x
|
||||
}
|
||||
return x[1:]
|
||||
}
|
||||
|
||||
// find the data with key s.
|
||||
// Returns nil if not for or an error occurs.
|
||||
func (x xlMetaInlineData) find(key string) []byte {
|
||||
if len(x) == 0 || !x.versionOK() {
|
||||
return nil
|
||||
}
|
||||
sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
|
||||
if err != nil || sz == 0 {
|
||||
return nil
|
||||
}
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var found []byte
|
||||
found, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil || sz == 0 {
|
||||
return nil
|
||||
}
|
||||
if string(found) == key {
|
||||
val, _, _ := msgp.ReadBytesZC(buf)
|
||||
return val
|
||||
}
|
||||
// Skip it
|
||||
_, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validate checks if the data is valid.
|
||||
// It does not check integrity of the stored data.
|
||||
func (x xlMetaInlineData) validate() error {
|
||||
if len(x) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !x.versionOK() {
|
||||
return fmt.Errorf("xlMetaInlineData: unknown version 0x%x", x[0])
|
||||
}
|
||||
|
||||
sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
|
||||
if err != nil {
|
||||
return fmt.Errorf("xlMetaInlineData: %w", err)
|
||||
}
|
||||
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var key []byte
|
||||
key, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("xlMetaInlineData: %w", err)
|
||||
}
|
||||
if len(key) == 0 {
|
||||
return fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
|
||||
}
|
||||
_, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("xlMetaInlineData: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// repair will copy all seemingly valid data entries from a corrupted set.
|
||||
// This does not ensure that data is correct, but will allow all operations to complete.
|
||||
func (x *xlMetaInlineData) repair() {
|
||||
data := *x
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if !data.versionOK() {
|
||||
*x = nil
|
||||
return
|
||||
}
|
||||
|
||||
sz, buf, err := msgp.ReadMapHeaderBytes(data.afterVersion())
|
||||
if err != nil {
|
||||
*x = nil
|
||||
return
|
||||
}
|
||||
|
||||
// Remove all current data
|
||||
keys := make([][]byte, 0, sz)
|
||||
vals := make([][]byte, 0, sz)
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var key, val []byte
|
||||
key, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if len(key) == 0 {
|
||||
break
|
||||
}
|
||||
val, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
keys = append(keys, key)
|
||||
vals = append(vals, val)
|
||||
}
|
||||
x.serialize(-1, keys, vals)
|
||||
}
|
||||
|
||||
// validate checks if the data is valid.
|
||||
// It does not check integrity of the stored data.
|
||||
func (x xlMetaInlineData) list() ([]string, error) {
|
||||
if len(x) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if !x.versionOK() {
|
||||
return nil, errors.New("xlMetaInlineData: unknown version")
|
||||
}
|
||||
|
||||
sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, sz)
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var key []byte
|
||||
key, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
return keys, err
|
||||
}
|
||||
if len(key) == 0 {
|
||||
return keys, fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
|
||||
}
|
||||
keys = append(keys, string(key))
|
||||
// Skip data...
|
||||
_, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
return keys, err
|
||||
}
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// serialize will serialize the provided keys and values.
|
||||
// The function will panic if keys/value slices aren't of equal length.
|
||||
// Payload size can give an indication of expected payload size.
|
||||
// If plSize is <= 0 it will be calculated.
|
||||
func (x *xlMetaInlineData) serialize(plSize int, keys [][]byte, vals [][]byte) {
|
||||
if len(keys) != len(vals) {
|
||||
panic(fmt.Errorf("xlMetaInlineData.serialize: keys/value number mismatch"))
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
*x = nil
|
||||
return
|
||||
}
|
||||
if plSize <= 0 {
|
||||
plSize = 1 + msgp.MapHeaderSize
|
||||
for i := range keys {
|
||||
plSize += len(keys[i]) + len(vals[i]) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
|
||||
}
|
||||
}
|
||||
payload := make([]byte, 1, plSize)
|
||||
payload[0] = xlMetaInlineDataVer
|
||||
payload = msgp.AppendMapHeader(payload, uint32(len(keys)))
|
||||
for i := range keys {
|
||||
payload = msgp.AppendStringFromBytes(payload, keys[i])
|
||||
payload = msgp.AppendBytes(payload, vals[i])
|
||||
}
|
||||
*x = payload
|
||||
}
|
||||
|
||||
// entries returns the number of entries in the data.
|
||||
func (x xlMetaInlineData) entries() int {
|
||||
if len(x) == 0 || !x.versionOK() {
|
||||
return 0
|
||||
}
|
||||
sz, _, _ := msgp.ReadMapHeaderBytes(x.afterVersion())
|
||||
return int(sz)
|
||||
}
|
||||
|
||||
// replace will add or replace a key/value pair.
|
||||
func (x *xlMetaInlineData) replace(key string, value []byte) {
|
||||
in := x.afterVersion()
|
||||
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
|
||||
keys := make([][]byte, 0, sz+1)
|
||||
vals := make([][]byte, 0, sz+1)
|
||||
|
||||
// Version plus header...
|
||||
plSize := 1 + msgp.MapHeaderSize
|
||||
replaced := false
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var found, foundVal []byte
|
||||
var err error
|
||||
found, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
foundVal, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
plSize += len(found) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
|
||||
keys = append(keys, found)
|
||||
if string(found) == key {
|
||||
vals = append(vals, value)
|
||||
plSize += len(value)
|
||||
replaced = true
|
||||
} else {
|
||||
vals = append(vals, foundVal)
|
||||
plSize += len(foundVal)
|
||||
}
|
||||
}
|
||||
|
||||
// Add one more.
|
||||
if !replaced {
|
||||
keys = append(keys, []byte(key))
|
||||
vals = append(vals, value)
|
||||
plSize += len(key) + len(value) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
|
||||
}
|
||||
|
||||
// Reserialize...
|
||||
x.serialize(plSize, keys, vals)
|
||||
}
|
||||
|
||||
// rename will rename a key.
|
||||
// Returns whether the key was found.
|
||||
func (x *xlMetaInlineData) rename(oldKey, newKey string) bool {
|
||||
in := x.afterVersion()
|
||||
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
|
||||
keys := make([][]byte, 0, sz)
|
||||
vals := make([][]byte, 0, sz)
|
||||
|
||||
// Version plus header...
|
||||
plSize := 1 + msgp.MapHeaderSize
|
||||
found := false
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var foundKey, foundVal []byte
|
||||
var err error
|
||||
foundKey, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
foundVal, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
plSize += len(foundVal) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
|
||||
vals = append(vals, foundVal)
|
||||
if string(foundKey) != oldKey {
|
||||
keys = append(keys, foundKey)
|
||||
plSize += len(foundKey)
|
||||
} else {
|
||||
keys = append(keys, []byte(newKey))
|
||||
plSize += len(newKey)
|
||||
found = true
|
||||
}
|
||||
}
|
||||
// If not found, just return.
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
|
||||
// Reserialize...
|
||||
x.serialize(plSize, keys, vals)
|
||||
return true
|
||||
}
|
||||
|
||||
// remove will remove one or more keys.
|
||||
// Returns true if any key was found.
|
||||
func (x *xlMetaInlineData) remove(keys ...string) bool {
|
||||
in := x.afterVersion()
|
||||
sz, buf, _ := msgp.ReadMapHeaderBytes(in)
|
||||
newKeys := make([][]byte, 0, sz)
|
||||
newVals := make([][]byte, 0, sz)
|
||||
var removeKey func(s []byte) bool
|
||||
|
||||
// Copy if big number of compares...
|
||||
if len(keys) > 5 && sz > 5 {
|
||||
mKeys := make(map[string]struct{}, len(keys))
|
||||
for _, key := range keys {
|
||||
mKeys[key] = struct{}{}
|
||||
}
|
||||
removeKey = func(s []byte) bool {
|
||||
_, ok := mKeys[string(s)]
|
||||
return ok
|
||||
}
|
||||
} else {
|
||||
removeKey = func(s []byte) bool {
|
||||
for _, key := range keys {
|
||||
if key == string(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Version plus header...
|
||||
plSize := 1 + msgp.MapHeaderSize
|
||||
found := false
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
var foundKey, foundVal []byte
|
||||
var err error
|
||||
foundKey, buf, err = msgp.ReadMapKeyZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
foundVal, buf, err = msgp.ReadBytesZC(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if !removeKey(foundKey) {
|
||||
plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal)
|
||||
newKeys = append(newKeys, foundKey)
|
||||
newVals = append(newVals, foundVal)
|
||||
} else {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
// If not found, just return.
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
// If none left...
|
||||
if len(newKeys) == 0 {
|
||||
*x = nil
|
||||
return true
|
||||
}
|
||||
|
||||
// Reserialize...
|
||||
x.serialize(plSize, newKeys, newVals)
|
||||
return true
|
||||
}
|
||||
|
||||
// xlMetaV2TrimData will trim any data from the metadata without unmarshalling it.
|
||||
// If any error occurs the unmodified data is returned.
|
||||
func xlMetaV2TrimData(buf []byte) []byte {
|
||||
metaBuf, min, maj, err := checkXL2V1(buf)
|
||||
if err != nil {
|
||||
return buf
|
||||
}
|
||||
if maj == 1 && min < 1 {
|
||||
// First version to carry data.
|
||||
return buf
|
||||
}
|
||||
// Skip header
|
||||
_, metaBuf, err = msgp.ReadBytesZC(metaBuf)
|
||||
if err != nil {
|
||||
logger.LogIf(GlobalContext, err)
|
||||
return buf
|
||||
}
|
||||
// Skip CRC
|
||||
if maj > 1 || min >= 2 {
|
||||
_, metaBuf, err = msgp.ReadUint32Bytes(metaBuf)
|
||||
logger.LogIf(GlobalContext, err)
|
||||
}
|
||||
// = input - current pos
|
||||
ends := len(buf) - len(metaBuf)
|
||||
if ends > len(buf) {
|
||||
return buf
|
||||
}
|
||||
|
||||
return buf[:ends]
|
||||
}
|
||||
Reference in New Issue
Block a user