MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-27 15:45:55 -05:00
Code Issues Packages Projects Releases Wiki Activity
2eee744e34
minio/cmd/xl-storage-format-v2_gen.go

2343 lines
53 KiB
Go
Raw Normal View History

Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
package cmd
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
import (
"github.com/tinylib/msgp/msgp"
)
// DecodeMsg implements msgp.Decodable
func (z *ChecksumAlgo) DecodeMsg(dc *msgp.Reader) (err error) {
{
var zb0001 uint8
zb0001, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = ChecksumAlgo(zb0001)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z ChecksumAlgo) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteUint8(uint8(z))
if err != nil {
err = msgp.WrapError(err)
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z ChecksumAlgo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendUint8(o, uint8(z))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *ChecksumAlgo) UnmarshalMsg(bts []byte) (o []byte, err error) {
{
var zb0001 uint8
zb0001, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = ChecksumAlgo(zb0001)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z ChecksumAlgo) Msgsize() (s int) {
s = msgp.Uint8Size
return
}
// DecodeMsg implements msgp.Decodable
func (z *ErasureAlgo) DecodeMsg(dc *msgp.Reader) (err error) {
{
var zb0001 uint8
zb0001, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = ErasureAlgo(zb0001)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z ErasureAlgo) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteUint8(uint8(z))
if err != nil {
err = msgp.WrapError(err)
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z ErasureAlgo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendUint8(o, uint8(z))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *ErasureAlgo) UnmarshalMsg(bts []byte) (o []byte, err error) {
{
var zb0001 uint8
zb0001, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = ErasureAlgo(zb0001)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z ErasureAlgo) Msgsize() (s int) {
s = msgp.Uint8Size
return
}
// DecodeMsg implements msgp.Decodable
func (z *VersionType) DecodeMsg(dc *msgp.Reader) (err error) {
{
var zb0001 uint8
zb0001, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = VersionType(zb0001)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z VersionType) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteUint8(uint8(z))
if err != nil {
err = msgp.WrapError(err)
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z VersionType) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendUint8(o, uint8(z))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *VersionType) UnmarshalMsg(bts []byte) (o []byte, err error) {
{
var zb0001 uint8
zb0001, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = VersionType(zb0001)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z VersionType) Msgsize() (s int) {
s = msgp.Uint8Size
return
}
// DecodeMsg implements msgp.Decodable
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
func (z *xlFlags) DecodeMsg(dc *msgp.Reader) (err error) {
{
var zb0001 uint8
zb0001, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = xlFlags(zb0001)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z xlFlags) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteUint8(uint8(z))
if err != nil {
err = msgp.WrapError(err)
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z xlFlags) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendUint8(o, uint8(z))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlFlags) UnmarshalMsg(bts []byte) (o []byte, err error) {
{
var zb0001 uint8
zb0001, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = xlFlags(zb0001)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z xlFlags) Msgsize() (s int) {
s = msgp.Uint8Size
return
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaBuf) DecodeMsg(dc *msgp.Reader) (err error) {
{
var zb0001 []byte
zb0001, err = dc.ReadBytes([]byte((*z)))
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = xlMetaBuf(zb0001)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z xlMetaBuf) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteBytes([]byte(z))
if err != nil {
err = msgp.WrapError(err)
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z xlMetaBuf) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendBytes(o, []byte(z))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaBuf) UnmarshalMsg(bts []byte) (o []byte, err error) {
{
var zb0001 []byte
zb0001, bts, err = msgp.ReadBytesBytes(bts, []byte((*z)))
if err != nil {
err = msgp.WrapError(err)
return
}
(*z) = xlMetaBuf(zb0001)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z xlMetaBuf) Msgsize() (s int) {
s = msgp.BytesPrefixSize + len([]byte(z))
return
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaDataDirDecoder) DecodeMsg(dc *msgp.Reader) (err error) {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
case "V2Obj":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
z.ObjectV2 = nil
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
} else {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.ObjectV2 == nil {
z.ObjectV2 = new(struct {
DataDir [16]byte `msg:"DDir"`
})
}
var zb0002 uint32
zb0002, err = dc.ReadMapHeader()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
err = msgp.WrapError(err, "ObjectV2")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
for zb0002 > 0 {
zb0002--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
switch msgp.UnsafeString(field) {
case "DDir":
err = dc.ReadExactBytes((z.ObjectV2.DataDir)[:])
if err != nil {
err = msgp.WrapError(err, "ObjectV2", "DataDir")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
return
}
// EncodeMsg implements msgp.Encodable
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
func (z *xlMetaDataDirDecoder) EncodeMsg(en *msgp.Writer) (err error) {
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
// check for omitted fields
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
zb0001Len := uint32(1)
var zb0001Mask uint8 /* 1 bits */
fix: add healing metric type for heal tracing (#15631) changes the `heal.checkBucket` to `heal.Bucket` instead since the latter is more meaningful.
2022-08-31 15:28:03 -04:00
_ = zb0001Mask
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.ObjectV2 == nil {
zb0001Len--
zb0001Mask |= 0x1
}
// variable map header, size zb0001Len
err = en.Append(0x80 | uint8(zb0001Len))
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if zb0001Len == 0 {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x1) == 0 { // if not omitted
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
// write "V2Obj"
err = en.Append(0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.ObjectV2 == nil {
err = en.WriteNil()
if err != nil {
return
}
} else {
// map header, size 1
// write "DDir"
err = en.Append(0x81, 0xa4, 0x44, 0x44, 0x69, 0x72)
if err != nil {
return
}
err = en.WriteBytes((z.ObjectV2.DataDir)[:])
if err != nil {
err = msgp.WrapError(err, "ObjectV2", "DataDir")
return
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
return
}
// MarshalMsg implements msgp.Marshaler
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
func (z *xlMetaDataDirDecoder) MarshalMsg(b []byte) (o []byte, err error) {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
o = msgp.Require(b, z.Msgsize())
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
// check for omitted fields
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
zb0001Len := uint32(1)
var zb0001Mask uint8 /* 1 bits */
fix: add healing metric type for heal tracing (#15631) changes the `heal.checkBucket` to `heal.Bucket` instead since the latter is more meaningful.
2022-08-31 15:28:03 -04:00
_ = zb0001Mask
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.ObjectV2 == nil {
zb0001Len--
zb0001Mask |= 0x1
}
// variable map header, size zb0001Len
o = append(o, 0x80|uint8(zb0001Len))
if zb0001Len == 0 {
return
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x1) == 0 { // if not omitted
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
// string "V2Obj"
o = append(o, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a)
if z.ObjectV2 == nil {
o = msgp.AppendNil(o)
} else {
// map header, size 1
// string "DDir"
o = append(o, 0x81, 0xa4, 0x44, 0x44, 0x69, 0x72)
o = msgp.AppendBytes(o, (z.ObjectV2.DataDir)[:])
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
}
return
}
// UnmarshalMsg implements msgp.Unmarshaler
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
func (z *xlMetaDataDirDecoder) UnmarshalMsg(bts []byte) (o []byte, err error) {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
case "V2Obj":
if msgp.IsNil(bts) {
bts, err = msgp.ReadNilBytes(bts)
if err != nil {
return
}
z.ObjectV2 = nil
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
} else {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.ObjectV2 == nil {
z.ObjectV2 = new(struct {
DataDir [16]byte `msg:"DDir"`
})
}
var zb0002 uint32
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
err = msgp.WrapError(err, "ObjectV2")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
for zb0002 > 0 {
zb0002--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
switch msgp.UnsafeString(field) {
case "DDir":
bts, err = msgp.ReadExactBytes(bts, (z.ObjectV2.DataDir)[:])
if err != nil {
err = msgp.WrapError(err, "ObjectV2", "DataDir")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
func (z *xlMetaDataDirDecoder) Msgsize() (s int) {
s = 1 + 6
if z.ObjectV2 == nil {
s += msgp.NilSize
} else {
s += 1 + 5 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize))
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
return
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaV2DeleteMarker) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "ID":
err = dc.ReadExactBytes((z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
case "MTime":
z.ModTime, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
case "MetaSys":
var zb0002 uint32
zb0002, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
if z.MetaSys == nil {
z.MetaSys = make(map[string][]byte, zb0002)
} else if len(z.MetaSys) > 0 {
for key := range z.MetaSys {
delete(z.MetaSys, key)
}
}
for zb0002 > 0 {
zb0002--
var za0002 string
var za0003 []byte
za0002, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
za0003, err = dc.ReadBytes(za0003)
if err != nil {
err = msgp.WrapError(err, "MetaSys", za0002)
return
}
z.MetaSys[za0002] = za0003
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *xlMetaV2DeleteMarker) EncodeMsg(en *msgp.Writer) (err error) {
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
// check for omitted fields
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
zb0001Len := uint32(3)
var zb0001Mask uint8 /* 3 bits */
fix: add healing metric type for heal tracing (#15631) changes the `heal.checkBucket` to `heal.Bucket` instead since the latter is more meaningful.
2022-08-31 15:28:03 -04:00
_ = zb0001Mask
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
if z.MetaSys == nil {
zb0001Len--
zb0001Mask |= 0x4
}
// variable map header, size zb0001Len
err = en.Append(0x80 | uint8(zb0001Len))
if err != nil {
return
}
if zb0001Len == 0 {
return
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// write "ID"
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
err = en.Append(0xa2, 0x49, 0x44)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
return
}
err = en.WriteBytes((z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
// write "MTime"
err = en.Append(0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65)
if err != nil {
return
}
err = en.WriteInt64(z.ModTime)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x4) == 0 { // if not omitted
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
// write "MetaSys"
err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73)
if err != nil {
return
}
err = en.WriteMapHeader(uint32(len(z.MetaSys)))
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
for za0002, za0003 := range z.MetaSys {
err = en.WriteString(za0002)
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
err = en.WriteBytes(za0003)
if err != nil {
err = msgp.WrapError(err, "MetaSys", za0002)
return
}
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
// MarshalMsg implements msgp.Marshaler
func (z *xlMetaV2DeleteMarker) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
// check for omitted fields
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
zb0001Len := uint32(3)
var zb0001Mask uint8 /* 3 bits */
fix: add healing metric type for heal tracing (#15631) changes the `heal.checkBucket` to `heal.Bucket` instead since the latter is more meaningful.
2022-08-31 15:28:03 -04:00
_ = zb0001Mask
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
if z.MetaSys == nil {
zb0001Len--
zb0001Mask |= 0x4
}
// variable map header, size zb0001Len
o = append(o, 0x80|uint8(zb0001Len))
if zb0001Len == 0 {
return
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// string "ID"
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
o = append(o, 0xa2, 0x49, 0x44)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
o = msgp.AppendBytes(o, (z.VersionID)[:])
// string "MTime"
o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65)
o = msgp.AppendInt64(o, z.ModTime)
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x4) == 0 { // if not omitted
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
// string "MetaSys"
o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73)
o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys)))
for za0002, za0003 := range z.MetaSys {
o = msgp.AppendString(o, za0002)
o = msgp.AppendBytes(o, za0003)
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaV2DeleteMarker) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "ID":
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
case "MTime":
z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
case "MetaSys":
var zb0002 uint32
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
if z.MetaSys == nil {
z.MetaSys = make(map[string][]byte, zb0002)
} else if len(z.MetaSys) > 0 {
for key := range z.MetaSys {
delete(z.MetaSys, key)
}
}
for zb0002 > 0 {
var za0002 string
var za0003 []byte
zb0002--
za0002, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
za0003, bts, err = msgp.ReadBytesBytes(bts, za0003)
if err != nil {
err = msgp.WrapError(err, "MetaSys", za0002)
return
}
z.MetaSys[za0002] = za0003
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *xlMetaV2DeleteMarker) Msgsize() (s int) {
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
s = 1 + 3 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 6 + msgp.Int64Size + 8 + msgp.MapHeaderSize
if z.MetaSys != nil {
for za0002, za0003 := range z.MetaSys {
_ = za0003
s += msgp.StringPrefixSize + len(za0002) + msgp.BytesPrefixSize + len(za0003)
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "ID":
err = dc.ReadExactBytes((z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
case "DDir":
err = dc.ReadExactBytes((z.DataDir)[:])
if err != nil {
err = msgp.WrapError(err, "DataDir")
return
}
case "EcAlgo":
{
var zb0002 uint8
zb0002, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "ErasureAlgorithm")
return
}
z.ErasureAlgorithm = ErasureAlgo(zb0002)
}
case "EcM":
z.ErasureM, err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "ErasureM")
return
}
case "EcN":
z.ErasureN, err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "ErasureN")
return
}
case "EcBSize":
z.ErasureBlockSize, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "ErasureBlockSize")
return
}
case "EcIndex":
z.ErasureIndex, err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "ErasureIndex")
return
}
case "EcDist":
var zb0003 uint32
zb0003, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "ErasureDist")
return
}
if cap(z.ErasureDist) >= int(zb0003) {
z.ErasureDist = (z.ErasureDist)[:zb0003]
} else {
z.ErasureDist = make([]uint8, zb0003)
}
for za0003 := range z.ErasureDist {
z.ErasureDist[za0003], err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "ErasureDist", za0003)
return
}
}
case "CSumAlgo":
{
var zb0004 uint8
zb0004, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "BitrotChecksumAlgo")
return
}
z.BitrotChecksumAlgo = ChecksumAlgo(zb0004)
}
case "PartNums":
var zb0005 uint32
zb0005, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "PartNumbers")
return
}
if cap(z.PartNumbers) >= int(zb0005) {
z.PartNumbers = (z.PartNumbers)[:zb0005]
} else {
z.PartNumbers = make([]int, zb0005)
}
for za0004 := range z.PartNumbers {
z.PartNumbers[za0004], err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "PartNumbers", za0004)
return
}
}
case "PartETags":
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "PartETags")
return
}
z.PartETags = nil
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
} else {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
var zb0006 uint32
zb0006, err = dc.ReadArrayHeader()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
err = msgp.WrapError(err, "PartETags")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if z.PartETags != nil && cap(z.PartETags) >= int(zb0006) {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
z.PartETags = (z.PartETags)[:zb0006]
} else {
z.PartETags = make([]string, zb0006)
}
for za0005 := range z.PartETags {
z.PartETags[za0005], err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "PartETags", za0005)
return
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
case "PartSizes":
var zb0007 uint32
zb0007, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "PartSizes")
return
}
if cap(z.PartSizes) >= int(zb0007) {
z.PartSizes = (z.PartSizes)[:zb0007]
} else {
z.PartSizes = make([]int64, zb0007)
}
for za0006 := range z.PartSizes {
z.PartSizes[za0006], err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "PartSizes", za0006)
return
}
}
case "PartASizes":
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "PartActualSizes")
return
}
z.PartActualSizes = nil
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
} else {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
var zb0008 uint32
zb0008, err = dc.ReadArrayHeader()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
err = msgp.WrapError(err, "PartActualSizes")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if z.PartActualSizes != nil && cap(z.PartActualSizes) >= int(zb0008) {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
z.PartActualSizes = (z.PartActualSizes)[:zb0008]
} else {
z.PartActualSizes = make([]int64, zb0008)
}
for za0007 := range z.PartActualSizes {
z.PartActualSizes[za0007], err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "PartActualSizes", za0007)
return
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
case "PartIdx":
var zb0009 uint32
zb0009, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err, "PartIndices")
return
}
if cap(z.PartIndices) >= int(zb0009) {
z.PartIndices = (z.PartIndices)[:zb0009]
} else {
z.PartIndices = make([][]byte, zb0009)
}
for za0008 := range z.PartIndices {
z.PartIndices[za0008], err = dc.ReadBytes(z.PartIndices[za0008])
if err != nil {
err = msgp.WrapError(err, "PartIndices", za0008)
return
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
case "Size":
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
z.Size, err = dc.ReadInt64()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
err = msgp.WrapError(err, "Size")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
case "MTime":
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
z.ModTime, err = dc.ReadInt64()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
err = msgp.WrapError(err, "ModTime")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
case "MetaSys":
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if dc.IsNil() {
err = dc.ReadNil()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
z.MetaSys = nil
} else {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
var zb0010 uint32
zb0010, err = dc.ReadMapHeader()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
err = msgp.WrapError(err, "MetaSys")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.MetaSys == nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
z.MetaSys = make(map[string][]byte, zb0010)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
} else if len(z.MetaSys) > 0 {
for key := range z.MetaSys {
delete(z.MetaSys, key)
}
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for zb0010 > 0 {
zb0010--
var za0009 string
var za0010 []byte
za0009, err = dc.ReadString()
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
za0010, err = dc.ReadBytes(za0010)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if err != nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
err = msgp.WrapError(err, "MetaSys", za0009)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
z.MetaSys[za0009] = za0010
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
case "MetaUsr":
if dc.IsNil() {
err = dc.ReadNil()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
err = msgp.WrapError(err, "MetaUser")
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
z.MetaUser = nil
} else {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
var zb0011 uint32
zb0011, err = dc.ReadMapHeader()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
err = msgp.WrapError(err, "MetaUser")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.MetaUser == nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
z.MetaUser = make(map[string]string, zb0011)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
} else if len(z.MetaUser) > 0 {
for key := range z.MetaUser {
delete(z.MetaUser, key)
}
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for zb0011 > 0 {
zb0011--
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
var za0011 string
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
var za0012 string
za0011, err = dc.ReadString()
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if err != nil {
err = msgp.WrapError(err, "MetaUser")
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
za0012, err = dc.ReadString()
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if err != nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
err = msgp.WrapError(err, "MetaUser", za0011)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
z.MetaUser[za0011] = za0012
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
// check for omitted fields
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
zb0001Len := uint32(18)
var zb0001Mask uint32 /* 18 bits */
fix: add healing metric type for heal tracing (#15631) changes the `heal.checkBucket` to `heal.Bucket` instead since the latter is more meaningful.
2022-08-31 15:28:03 -04:00
_ = zb0001Mask
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
if z.PartIndices == nil {
zb0001Len--
zb0001Mask |= 0x2000
}
// variable map header, size zb0001Len
err = en.WriteMapHeader(zb0001Len)
if err != nil {
return
}
if zb0001Len == 0 {
return
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// write "ID"
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
err = en.Append(0xa2, 0x49, 0x44)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
return
}
err = en.WriteBytes((z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
// write "DDir"
err = en.Append(0xa4, 0x44, 0x44, 0x69, 0x72)
if err != nil {
return
}
err = en.WriteBytes((z.DataDir)[:])
if err != nil {
err = msgp.WrapError(err, "DataDir")
return
}
// write "EcAlgo"
err = en.Append(0xa6, 0x45, 0x63, 0x41, 0x6c, 0x67, 0x6f)
if err != nil {
return
}
err = en.WriteUint8(uint8(z.ErasureAlgorithm))
if err != nil {
err = msgp.WrapError(err, "ErasureAlgorithm")
return
}
// write "EcM"
err = en.Append(0xa3, 0x45, 0x63, 0x4d)
if err != nil {
return
}
err = en.WriteInt(z.ErasureM)
if err != nil {
err = msgp.WrapError(err, "ErasureM")
return
}
// write "EcN"
err = en.Append(0xa3, 0x45, 0x63, 0x4e)
if err != nil {
return
}
err = en.WriteInt(z.ErasureN)
if err != nil {
err = msgp.WrapError(err, "ErasureN")
return
}
// write "EcBSize"
err = en.Append(0xa7, 0x45, 0x63, 0x42, 0x53, 0x69, 0x7a, 0x65)
if err != nil {
return
}
err = en.WriteInt64(z.ErasureBlockSize)
if err != nil {
err = msgp.WrapError(err, "ErasureBlockSize")
return
}
// write "EcIndex"
err = en.Append(0xa7, 0x45, 0x63, 0x49, 0x6e, 0x64, 0x65, 0x78)
if err != nil {
return
}
err = en.WriteInt(z.ErasureIndex)
if err != nil {
err = msgp.WrapError(err, "ErasureIndex")
return
}
// write "EcDist"
err = en.Append(0xa6, 0x45, 0x63, 0x44, 0x69, 0x73, 0x74)
if err != nil {
return
}
err = en.WriteArrayHeader(uint32(len(z.ErasureDist)))
if err != nil {
err = msgp.WrapError(err, "ErasureDist")
return
}
for za0003 := range z.ErasureDist {
err = en.WriteUint8(z.ErasureDist[za0003])
if err != nil {
err = msgp.WrapError(err, "ErasureDist", za0003)
return
}
}
// write "CSumAlgo"
err = en.Append(0xa8, 0x43, 0x53, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f)
if err != nil {
return
}
err = en.WriteUint8(uint8(z.BitrotChecksumAlgo))
if err != nil {
err = msgp.WrapError(err, "BitrotChecksumAlgo")
return
}
// write "PartNums"
err = en.Append(0xa8, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x73)
if err != nil {
return
}
err = en.WriteArrayHeader(uint32(len(z.PartNumbers)))
if err != nil {
err = msgp.WrapError(err, "PartNumbers")
return
}
for za0004 := range z.PartNumbers {
err = en.WriteInt(z.PartNumbers[za0004])
if err != nil {
err = msgp.WrapError(err, "PartNumbers", za0004)
return
}
}
// write "PartETags"
err = en.Append(0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73)
if err != nil {
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.PartETags == nil { // allownil: if nil
err = en.WriteNil()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
} else {
err = en.WriteArrayHeader(uint32(len(z.PartETags)))
if err != nil {
err = msgp.WrapError(err, "PartETags")
return
}
for za0005 := range z.PartETags {
err = en.WriteString(z.PartETags[za0005])
if err != nil {
err = msgp.WrapError(err, "PartETags", za0005)
return
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
// write "PartSizes"
err = en.Append(0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73)
if err != nil {
return
}
err = en.WriteArrayHeader(uint32(len(z.PartSizes)))
if err != nil {
err = msgp.WrapError(err, "PartSizes")
return
}
for za0006 := range z.PartSizes {
err = en.WriteInt64(z.PartSizes[za0006])
if err != nil {
err = msgp.WrapError(err, "PartSizes", za0006)
return
}
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
// write "PartASizes"
err = en.Append(0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73)
if err != nil {
return
}
if z.PartActualSizes == nil { // allownil: if nil
err = en.WriteNil()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
} else {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
err = en.WriteArrayHeader(uint32(len(z.PartActualSizes)))
if err != nil {
err = msgp.WrapError(err, "PartActualSizes")
return
}
for za0007 := range z.PartActualSizes {
err = en.WriteInt64(z.PartActualSizes[za0007])
if err != nil {
err = msgp.WrapError(err, "PartActualSizes", za0007)
return
}
}
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x2000) == 0 { // if not omitted
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
// write "PartIdx"
err = en.Append(0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78)
if err != nil {
return
}
err = en.WriteArrayHeader(uint32(len(z.PartIndices)))
if err != nil {
err = msgp.WrapError(err, "PartIndices")
return
}
for za0008 := range z.PartIndices {
err = en.WriteBytes(z.PartIndices[za0008])
if err != nil {
err = msgp.WrapError(err, "PartIndices", za0008)
return
}
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// write "Size"
err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
if err != nil {
return
}
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
err = en.WriteInt64(z.Size)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
err = msgp.WrapError(err, "Size")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
// write "MTime"
err = en.Append(0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65)
if err != nil {
return
}
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
err = en.WriteInt64(z.ModTime)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
err = msgp.WrapError(err, "ModTime")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
// write "MetaSys"
err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73)
if err != nil {
return
}
if z.MetaSys == nil { // allownil: if nil
err = en.WriteNil()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
} else {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
err = en.WriteMapHeader(uint32(len(z.MetaSys)))
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for za0009, za0010 := range z.MetaSys {
err = en.WriteString(za0009)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
err = en.WriteBytes(za0010)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
err = msgp.WrapError(err, "MetaSys", za0009)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
}
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
// write "MetaUsr"
err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72)
if err != nil {
return
}
if z.MetaUser == nil { // allownil: if nil
err = en.WriteNil()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
} else {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
err = en.WriteMapHeader(uint32(len(z.MetaUser)))
if err != nil {
err = msgp.WrapError(err, "MetaUser")
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for za0011, za0012 := range z.MetaUser {
err = en.WriteString(za0011)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
err = msgp.WrapError(err, "MetaUser")
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
err = en.WriteString(za0012)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
err = msgp.WrapError(err, "MetaUser", za0011)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
}
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
// check for omitted fields
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
zb0001Len := uint32(18)
var zb0001Mask uint32 /* 18 bits */
fix: add healing metric type for heal tracing (#15631) changes the `heal.checkBucket` to `heal.Bucket` instead since the latter is more meaningful.
2022-08-31 15:28:03 -04:00
_ = zb0001Mask
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
if z.PartIndices == nil {
zb0001Len--
zb0001Mask |= 0x2000
}
// variable map header, size zb0001Len
o = msgp.AppendMapHeader(o, zb0001Len)
if zb0001Len == 0 {
return
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// string "ID"
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
o = append(o, 0xa2, 0x49, 0x44)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
o = msgp.AppendBytes(o, (z.VersionID)[:])
// string "DDir"
o = append(o, 0xa4, 0x44, 0x44, 0x69, 0x72)
o = msgp.AppendBytes(o, (z.DataDir)[:])
// string "EcAlgo"
o = append(o, 0xa6, 0x45, 0x63, 0x41, 0x6c, 0x67, 0x6f)
o = msgp.AppendUint8(o, uint8(z.ErasureAlgorithm))
// string "EcM"
o = append(o, 0xa3, 0x45, 0x63, 0x4d)
o = msgp.AppendInt(o, z.ErasureM)
// string "EcN"
o = append(o, 0xa3, 0x45, 0x63, 0x4e)
o = msgp.AppendInt(o, z.ErasureN)
// string "EcBSize"
o = append(o, 0xa7, 0x45, 0x63, 0x42, 0x53, 0x69, 0x7a, 0x65)
o = msgp.AppendInt64(o, z.ErasureBlockSize)
// string "EcIndex"
o = append(o, 0xa7, 0x45, 0x63, 0x49, 0x6e, 0x64, 0x65, 0x78)
o = msgp.AppendInt(o, z.ErasureIndex)
// string "EcDist"
o = append(o, 0xa6, 0x45, 0x63, 0x44, 0x69, 0x73, 0x74)
o = msgp.AppendArrayHeader(o, uint32(len(z.ErasureDist)))
for za0003 := range z.ErasureDist {
o = msgp.AppendUint8(o, z.ErasureDist[za0003])
}
// string "CSumAlgo"
o = append(o, 0xa8, 0x43, 0x53, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f)
o = msgp.AppendUint8(o, uint8(z.BitrotChecksumAlgo))
// string "PartNums"
o = append(o, 0xa8, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x73)
o = msgp.AppendArrayHeader(o, uint32(len(z.PartNumbers)))
for za0004 := range z.PartNumbers {
o = msgp.AppendInt(o, z.PartNumbers[za0004])
}
// string "PartETags"
o = append(o, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.PartETags == nil { // allownil: if nil
o = msgp.AppendNil(o)
} else {
o = msgp.AppendArrayHeader(o, uint32(len(z.PartETags)))
for za0005 := range z.PartETags {
o = msgp.AppendString(o, z.PartETags[za0005])
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
// string "PartSizes"
o = append(o, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73)
o = msgp.AppendArrayHeader(o, uint32(len(z.PartSizes)))
for za0006 := range z.PartSizes {
o = msgp.AppendInt64(o, z.PartSizes[za0006])
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
// string "PartASizes"
o = append(o, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73)
if z.PartActualSizes == nil { // allownil: if nil
o = msgp.AppendNil(o)
} else {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
o = msgp.AppendArrayHeader(o, uint32(len(z.PartActualSizes)))
for za0007 := range z.PartActualSizes {
o = msgp.AppendInt64(o, z.PartActualSizes[za0007])
}
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x2000) == 0 { // if not omitted
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
// string "PartIdx"
o = append(o, 0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78)
o = msgp.AppendArrayHeader(o, uint32(len(z.PartIndices)))
for za0008 := range z.PartIndices {
o = msgp.AppendBytes(o, z.PartIndices[za0008])
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// string "Size"
o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
o = msgp.AppendInt64(o, z.Size)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// string "MTime"
o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65)
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
o = msgp.AppendInt64(o, z.ModTime)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
// string "MetaSys"
o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73)
if z.MetaSys == nil { // allownil: if nil
o = msgp.AppendNil(o)
} else {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys)))
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for za0009, za0010 := range z.MetaSys {
o = msgp.AppendString(o, za0009)
o = msgp.AppendBytes(o, za0010)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
// string "MetaUsr"
o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72)
if z.MetaUser == nil { // allownil: if nil
o = msgp.AppendNil(o)
} else {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
o = msgp.AppendMapHeader(o, uint32(len(z.MetaUser)))
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for za0011, za0012 := range z.MetaUser {
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
o = msgp.AppendString(o, za0011)
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
o = msgp.AppendString(o, za0012)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
}
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "ID":
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
case "DDir":
bts, err = msgp.ReadExactBytes(bts, (z.DataDir)[:])
if err != nil {
err = msgp.WrapError(err, "DataDir")
return
}
case "EcAlgo":
{
var zb0002 uint8
zb0002, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ErasureAlgorithm")
return
}
z.ErasureAlgorithm = ErasureAlgo(zb0002)
}
case "EcM":
z.ErasureM, bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ErasureM")
return
}
case "EcN":
z.ErasureN, bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ErasureN")
return
}
case "EcBSize":
z.ErasureBlockSize, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ErasureBlockSize")
return
}
case "EcIndex":
z.ErasureIndex, bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ErasureIndex")
return
}
case "EcDist":
var zb0003 uint32
zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ErasureDist")
return
}
if cap(z.ErasureDist) >= int(zb0003) {
z.ErasureDist = (z.ErasureDist)[:zb0003]
} else {
z.ErasureDist = make([]uint8, zb0003)
}
for za0003 := range z.ErasureDist {
z.ErasureDist[za0003], bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ErasureDist", za0003)
return
}
}
case "CSumAlgo":
{
var zb0004 uint8
zb0004, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "BitrotChecksumAlgo")
return
}
z.BitrotChecksumAlgo = ChecksumAlgo(zb0004)
}
case "PartNums":
var zb0005 uint32
zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "PartNumbers")
return
}
if cap(z.PartNumbers) >= int(zb0005) {
z.PartNumbers = (z.PartNumbers)[:zb0005]
} else {
z.PartNumbers = make([]int, zb0005)
}
for za0004 := range z.PartNumbers {
z.PartNumbers[za0004], bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "PartNumbers", za0004)
return
}
}
case "PartETags":
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if msgp.IsNil(bts) {
bts = bts[1:]
z.PartETags = nil
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
} else {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
var zb0006 uint32
zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
err = msgp.WrapError(err, "PartETags")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if z.PartETags != nil && cap(z.PartETags) >= int(zb0006) {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
z.PartETags = (z.PartETags)[:zb0006]
} else {
z.PartETags = make([]string, zb0006)
}
for za0005 := range z.PartETags {
z.PartETags[za0005], bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "PartETags", za0005)
return
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
case "PartSizes":
var zb0007 uint32
zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "PartSizes")
return
}
if cap(z.PartSizes) >= int(zb0007) {
z.PartSizes = (z.PartSizes)[:zb0007]
} else {
z.PartSizes = make([]int64, zb0007)
}
for za0006 := range z.PartSizes {
z.PartSizes[za0006], bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "PartSizes", za0006)
return
}
}
case "PartASizes":
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if msgp.IsNil(bts) {
bts = bts[1:]
z.PartActualSizes = nil
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
} else {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
var zb0008 uint32
zb0008, bts, err = msgp.ReadArrayHeaderBytes(bts)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
err = msgp.WrapError(err, "PartActualSizes")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if z.PartActualSizes != nil && cap(z.PartActualSizes) >= int(zb0008) {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
z.PartActualSizes = (z.PartActualSizes)[:zb0008]
} else {
z.PartActualSizes = make([]int64, zb0008)
}
for za0007 := range z.PartActualSizes {
z.PartActualSizes[za0007], bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "PartActualSizes", za0007)
return
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
case "PartIdx":
var zb0009 uint32
zb0009, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "PartIndices")
return
}
if cap(z.PartIndices) >= int(zb0009) {
z.PartIndices = (z.PartIndices)[:zb0009]
} else {
z.PartIndices = make([][]byte, zb0009)
}
for za0008 := range z.PartIndices {
z.PartIndices[za0008], bts, err = msgp.ReadBytesBytes(bts, z.PartIndices[za0008])
if err != nil {
err = msgp.WrapError(err, "PartIndices", za0008)
return
}
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
case "Size":
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
z.Size, bts, err = msgp.ReadInt64Bytes(bts)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
err = msgp.WrapError(err, "Size")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
case "MTime":
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
fix: simplify data structure before release (#9968) - additionally upgrade to msgp@v1.1.2 - change StatModTime,StatSize fields as simple Size/ModTime - reduce 50000 entries per List batch to 10000 as client needs to wait too long to see the first batch some times which is not desired and it is worth we write the data as soon as we have it.
2020-07-04 15:25:53 -04:00
err = msgp.WrapError(err, "ModTime")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
case "MetaSys":
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if msgp.IsNil(bts) {
bts = bts[1:]
z.MetaSys = nil
} else {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
var zb0010 uint32
zb0010, bts, err = msgp.ReadMapHeaderBytes(bts)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.MetaSys == nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
z.MetaSys = make(map[string][]byte, zb0010)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
} else if len(z.MetaSys) > 0 {
for key := range z.MetaSys {
delete(z.MetaSys, key)
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for zb0010 > 0 {
var za0009 string
var za0010 []byte
zb0010--
za0009, bts, err = msgp.ReadStringBytes(bts)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if err != nil {
err = msgp.WrapError(err, "MetaSys")
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
za0010, bts, err = msgp.ReadBytesBytes(bts, za0010)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if err != nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
err = msgp.WrapError(err, "MetaSys", za0009)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
z.MetaSys[za0009] = za0010
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
case "MetaUsr":
if msgp.IsNil(bts) {
bts = bts[1:]
z.MetaUser = nil
} else {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
var zb0011 uint32
zb0011, bts, err = msgp.ReadMapHeaderBytes(bts)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
err = msgp.WrapError(err, "MetaUser")
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if z.MetaUser == nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
z.MetaUser = make(map[string]string, zb0011)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
} else if len(z.MetaUser) > 0 {
for key := range z.MetaUser {
delete(z.MetaUser, key)
}
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for zb0011 > 0 {
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
var za0011 string
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
var za0012 string
zb0011--
za0011, bts, err = msgp.ReadStringBytes(bts)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if err != nil {
err = msgp.WrapError(err, "MetaUser")
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
za0012, bts, err = msgp.ReadStringBytes(bts)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
if err != nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
err = msgp.WrapError(err, "MetaUser", za0011)
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
return
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
z.MetaUser[za0011] = za0012
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *xlMetaV2Object) Msgsize() (s int) {
s = 3 + 3 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 5 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 7 + msgp.Uint8Size + 4 + msgp.IntSize + 4 + msgp.IntSize + 8 + msgp.Int64Size + 8 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.ErasureDist) * (msgp.Uint8Size)) + 9 + msgp.Uint8Size + 9 + msgp.ArrayHeaderSize + (len(z.PartNumbers) * (msgp.IntSize)) + 10 + msgp.ArrayHeaderSize
for za0005 := range z.PartETags {
s += msgp.StringPrefixSize + len(z.PartETags[za0005])
}
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
s += 10 + msgp.ArrayHeaderSize + (len(z.PartSizes) * (msgp.Int64Size)) + 11 + msgp.ArrayHeaderSize + (len(z.PartActualSizes) * (msgp.Int64Size)) + 8 + msgp.ArrayHeaderSize
for za0008 := range z.PartIndices {
s += msgp.BytesPrefixSize + len(z.PartIndices[za0008])
}
s += 5 + msgp.Int64Size + 6 + msgp.Int64Size + 8 + msgp.MapHeaderSize
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if z.MetaSys != nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for za0009, za0010 := range z.MetaSys {
_ = za0010
s += msgp.StringPrefixSize + len(za0009) + msgp.BytesPrefixSize + len(za0010)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
}
s += 8 + msgp.MapHeaderSize
if z.MetaUser != nil {
Add compressed file index (#15247)
2022-07-11 20:30:56 -04:00
for za0011, za0012 := range z.MetaUser {
_ = za0012
s += msgp.StringPrefixSize + len(za0011) + msgp.StringPrefixSize + len(za0012)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
}
return
}
// DecodeMsg implements msgp.Decodable
func (z *xlMetaV2Version) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "Type":
{
var zb0002 uint8
zb0002, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
z.Type = VersionType(zb0002)
}
case "V1Obj":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "ObjectV1")
return
}
z.ObjectV1 = nil
} else {
if z.ObjectV1 == nil {
z.ObjectV1 = new(xlMetaV1Object)
}
err = z.ObjectV1.DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err, "ObjectV1")
return
}
}
case "V2Obj":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
z.ObjectV2 = nil
} else {
if z.ObjectV2 == nil {
z.ObjectV2 = new(xlMetaV2Object)
}
err = z.ObjectV2.DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
}
case "DelObj":
if dc.IsNil() {
err = dc.ReadNil()
if err != nil {
err = msgp.WrapError(err, "DeleteMarker")
return
}
z.DeleteMarker = nil
} else {
if z.DeleteMarker == nil {
z.DeleteMarker = new(xlMetaV2DeleteMarker)
}
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
err = z.DeleteMarker.DecodeMsg(dc)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
err = msgp.WrapError(err, "DeleteMarker")
return
}
}
save MinIO version with each version (8-bytes extra) (#15170) store MinIO version along with each version in 'xl.meta' for future purposes, can be used as ways to add specific code for bug fixes if any.
2022-06-27 06:59:41 -04:00
case "v":
z.WrittenByVersion, err = dc.ReadUint64()
if err != nil {
err = msgp.WrapError(err, "WrittenByVersion")
return
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) {
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
// check for omitted fields
save MinIO version with each version (8-bytes extra) (#15170) store MinIO version along with each version in 'xl.meta' for future purposes, can be used as ways to add specific code for bug fixes if any.
2022-06-27 06:59:41 -04:00
zb0001Len := uint32(5)
var zb0001Mask uint8 /* 5 bits */
fix: add healing metric type for heal tracing (#15631) changes the `heal.checkBucket` to `heal.Bucket` instead since the latter is more meaningful.
2022-08-31 15:28:03 -04:00
_ = zb0001Mask
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if z.ObjectV1 == nil {
zb0001Len--
zb0001Mask |= 0x2
}
if z.ObjectV2 == nil {
zb0001Len--
zb0001Mask |= 0x4
}
if z.DeleteMarker == nil {
zb0001Len--
zb0001Mask |= 0x8
}
// variable map header, size zb0001Len
err = en.Append(0x80 | uint8(zb0001Len))
if err != nil {
return
}
if zb0001Len == 0 {
return
}
// write "Type"
err = en.Append(0xa4, 0x54, 0x79, 0x70, 0x65)
if err != nil {
return
}
err = en.WriteUint8(uint8(z.Type))
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x2) == 0 { // if not omitted
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// write "V1Obj"
err = en.Append(0xa5, 0x56, 0x31, 0x4f, 0x62, 0x6a)
if err != nil {
return
}
if z.ObjectV1 == nil {
err = en.WriteNil()
if err != nil {
return
}
} else {
err = z.ObjectV1.EncodeMsg(en)
if err != nil {
err = msgp.WrapError(err, "ObjectV1")
return
}
}
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x4) == 0 { // if not omitted
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// write "V2Obj"
err = en.Append(0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a)
if err != nil {
return
}
if z.ObjectV2 == nil {
err = en.WriteNil()
if err != nil {
return
}
} else {
err = z.ObjectV2.EncodeMsg(en)
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
}
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x8) == 0 { // if not omitted
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// write "DelObj"
err = en.Append(0xa6, 0x44, 0x65, 0x6c, 0x4f, 0x62, 0x6a)
if err != nil {
return
}
if z.DeleteMarker == nil {
err = en.WriteNil()
if err != nil {
return
}
} else {
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
err = z.DeleteMarker.EncodeMsg(en)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
err = msgp.WrapError(err, "DeleteMarker")
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
}
}
save MinIO version with each version (8-bytes extra) (#15170) store MinIO version along with each version in 'xl.meta' for future purposes, can be used as ways to add specific code for bug fixes if any.
2022-06-27 06:59:41 -04:00
// write "v"
err = en.Append(0xa1, 0x76)
if err != nil {
return
}
err = en.WriteUint64(z.WrittenByVersion)
if err != nil {
err = msgp.WrapError(err, "WrittenByVersion")
return
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
// MarshalMsg implements msgp.Marshaler
func (z *xlMetaV2Version) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
// check for omitted fields
save MinIO version with each version (8-bytes extra) (#15170) store MinIO version along with each version in 'xl.meta' for future purposes, can be used as ways to add specific code for bug fixes if any.
2022-06-27 06:59:41 -04:00
zb0001Len := uint32(5)
var zb0001Mask uint8 /* 5 bits */
fix: add healing metric type for heal tracing (#15631) changes the `heal.checkBucket` to `heal.Bucket` instead since the latter is more meaningful.
2022-08-31 15:28:03 -04:00
_ = zb0001Mask
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if z.ObjectV1 == nil {
zb0001Len--
zb0001Mask |= 0x2
}
if z.ObjectV2 == nil {
zb0001Len--
zb0001Mask |= 0x4
}
if z.DeleteMarker == nil {
zb0001Len--
zb0001Mask |= 0x8
}
// variable map header, size zb0001Len
o = append(o, 0x80|uint8(zb0001Len))
if zb0001Len == 0 {
return
}
// string "Type"
o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65)
o = msgp.AppendUint8(o, uint8(z.Type))
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x2) == 0 { // if not omitted
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// string "V1Obj"
o = append(o, 0xa5, 0x56, 0x31, 0x4f, 0x62, 0x6a)
if z.ObjectV1 == nil {
o = msgp.AppendNil(o)
} else {
o, err = z.ObjectV1.MarshalMsg(o)
if err != nil {
err = msgp.WrapError(err, "ObjectV1")
return
}
}
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x4) == 0 { // if not omitted
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// string "V2Obj"
o = append(o, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a)
if z.ObjectV2 == nil {
o = msgp.AppendNil(o)
} else {
o, err = z.ObjectV2.MarshalMsg(o)
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
}
}
Encode dir obj names before expiration (#19305) Object names of directory objects qualified for ExpiredObjectAllVersions must be encoded appropriately before calling on deletePrefix on their erasure set. e.g., a directory object and regular objects with overlapping prefixes could lead to the expiration of regular objects, which is not the intention of ILM. ``` bucket/dir/ ---> directory object bucket/dir/obj-1 ``` When `bucket/dir/` qualifies for expiration, the current implementation would remove regular objects under the prefix `bucket/dir/`, in this case, `bucket/dir/obj-1`.
2024-03-21 13:21:35 -04:00
if (zb0001Mask & 0x8) == 0 { // if not omitted
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
// string "DelObj"
o = append(o, 0xa6, 0x44, 0x65, 0x6c, 0x4f, 0x62, 0x6a)
if z.DeleteMarker == nil {
o = msgp.AppendNil(o)
} else {
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
o, err = z.DeleteMarker.MarshalMsg(o)
if err != nil {
err = msgp.WrapError(err, "DeleteMarker")
return
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
}
save MinIO version with each version (8-bytes extra) (#15170) store MinIO version along with each version in 'xl.meta' for future purposes, can be used as ways to add specific code for bug fixes if any.
2022-06-27 06:59:41 -04:00
// string "v"
o = append(o, 0xa1, 0x76)
o = msgp.AppendUint64(o, z.WrittenByVersion)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaV2Version) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "Type":
{
var zb0002 uint8
zb0002, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
z.Type = VersionType(zb0002)
}
case "V1Obj":
if msgp.IsNil(bts) {
bts, err = msgp.ReadNilBytes(bts)
if err != nil {
return
}
z.ObjectV1 = nil
} else {
if z.ObjectV1 == nil {
z.ObjectV1 = new(xlMetaV1Object)
}
bts, err = z.ObjectV1.UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err, "ObjectV1")
return
}
}
case "V2Obj":
if msgp.IsNil(bts) {
bts, err = msgp.ReadNilBytes(bts)
if err != nil {
return
}
z.ObjectV2 = nil
} else {
if z.ObjectV2 == nil {
z.ObjectV2 = new(xlMetaV2Object)
}
bts, err = z.ObjectV2.UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err, "ObjectV2")
return
}
}
case "DelObj":
if msgp.IsNil(bts) {
bts, err = msgp.ReadNilBytes(bts)
if err != nil {
return
}
z.DeleteMarker = nil
} else {
if z.DeleteMarker == nil {
z.DeleteMarker = new(xlMetaV2DeleteMarker)
}
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
bts, err = z.DeleteMarker.UnmarshalMsg(bts)
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
if err != nil {
err = msgp.WrapError(err, "DeleteMarker")
return
}
}
save MinIO version with each version (8-bytes extra) (#15170) store MinIO version along with each version in 'xl.meta' for future purposes, can be used as ways to add specific code for bug fixes if any.
2022-06-27 06:59:41 -04:00
case "v":
z.WrittenByVersion, bts, err = msgp.ReadUint64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "WrittenByVersion")
return
}
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *xlMetaV2Version) Msgsize() (s int) {
s = 1 + 5 + msgp.Uint8Size + 6
if z.ObjectV1 == nil {
s += msgp.NilSize
} else {
s += z.ObjectV1.Msgsize()
}
s += 6
if z.ObjectV2 == nil {
s += msgp.NilSize
} else {
s += z.ObjectV2.Msgsize()
}
s += 7
if z.DeleteMarker == nil {
s += msgp.NilSize
} else {
Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012.
2020-11-19 21:43:58 -05:00
s += z.DeleteMarker.Msgsize()
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
}
save MinIO version with each version (8-bytes extra) (#15170) store MinIO version along with each version in 'xl.meta' for future purposes, can be used as ways to add specific code for bug fixes if any.
2022-06-27 06:59:41 -04:00
s += 2 + msgp.Uint64Size
Support bucket versioning (#9377) - Implement a new xl.json 2.0.0 format to support, this moves the entire marshaling logic to POSIX layer, top layer always consumes a common FileInfo construct which simplifies the metadata reads. - Implement list object versions - Migrate to siphash from crchash for new deployments for object placements. Fixes #2111
2020-06-12 23:04:01 -04:00
return
}
Improve performance on multiple versions (#13573) Existing: ```go type xlMetaV2 struct { Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` } ``` Serialized as regular MessagePack. ```go //msgp:tuple xlMetaV2VersionHeader type xlMetaV2VersionHeader struct { VersionID [16]byte ModTime int64 Type VersionType Flags xlFlags } ``` Serialize as streaming MessagePack, format: ``` int(headerVersion) int(xlmetaVersion) int(nVersions) for each version { binary blob, xlMetaV2VersionHeader, serialized binary blob, xlMetaV2Version, serialized. } ``` xlMetaV2VersionHeader is <= 30 bytes serialized. Deserialized struct can easily be reused and does not contain pointers, so efficient as a slice (single allocation) This allows quickly parsing everything as slices of bytes (no copy). Versions are always *saved* sorted by modTime, newest *first*. No more need to sort on load. * Allows checking if a version exists. * Allows reading single version without unmarshal all. * Allows reading latest version of type without unmarshal all. * Allows reading latest version without unmarshal of all. * Allows checking if the latest is deleteMarker by reading first entry. * Allows adding/updating/deleting a version with only header deserialization. * Reduces allocations on conversion to FileInfo(s).
2021-11-18 15:15:22 -05:00
// DecodeMsg implements msgp.Decodable
func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) {
var zb0001 uint32
zb0001, err = dc.ReadArrayHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != 5 {
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
return
}
err = dc.ReadExactBytes((z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
z.ModTime, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
err = dc.ReadExactBytes((z.Signature)[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return
}
{
var zb0002 uint8
zb0002, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
z.Type = VersionType(zb0002)
}
{
var zb0003 uint8
zb0003, err = dc.ReadUint8()
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
z.Flags = xlFlags(zb0003)
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) {
// array header, size 5
err = en.Append(0x95)
if err != nil {
return
}
err = en.WriteBytes((z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
err = en.WriteInt64(z.ModTime)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
err = en.WriteBytes((z.Signature)[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return
}
err = en.WriteUint8(uint8(z.Type))
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
err = en.WriteUint8(uint8(z.Flags))
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *xlMetaV2VersionHeader) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// array header, size 5
o = append(o, 0x95)
o = msgp.AppendBytes(o, (z.VersionID)[:])
o = msgp.AppendInt64(o, z.ModTime)
o = msgp.AppendBytes(o, (z.Signature)[:])
o = msgp.AppendUint8(o, uint8(z.Type))
o = msgp.AppendUint8(o, uint8(z.Flags))
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) {
var zb0001 uint32
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != 5 {
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
return
}
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return
}
{
var zb0002 uint8
zb0002, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
z.Type = VersionType(zb0002)
}
{
var zb0003 uint8
zb0003, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
z.Flags = xlFlags(zb0003)
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *xlMetaV2VersionHeader) Msgsize() (s int) {
s = 1 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + msgp.Int64Size + msgp.ArrayHeaderSize + (4 * (msgp.ByteSize)) + msgp.Uint8Size + msgp.Uint8Size
return
}
Reference in New Issue Copy Permalink