MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

implement a safer completeMultipart implementation (#20227)

Browse Source 
- optimize writing part.N.meta by writing both part.N
  and its meta in sequence without network component.

- remove part.N.meta, part.N which were partially success
  ful, in quorum loss situations during renamePart()

- allow for strict read quorum check arbitrated via ETag
  for the given part number, this makes it double safer
  upon final commit.

- return an appropriate error when read quorum is missing,
  instead of returning InvalidPart{}, which is non-retryable
  error. This kind of situation can happen when many
  nodes are going offline in rotation, an example of such
  a restart() behavior is statefulset updates in k8s.

fixes #20091
This commit is contained in:
Harshavardhana
2024-08-12 01:38:15 -07:00
committed by GitHub
parent 909b169593
commit 2e0fd2cba9
19 changed files with 1487 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
cmd/xl-storage-format-v1_gen.go
View File

@@ -569,37 +569,37 @@ func (z *ObjectPartInfo) DecodeMsg(dc *msgp.Reader) (err error) {
return
}
switch msgp.UnsafeString(field) {
case "ETag":
case "e":
z.ETag, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "ETag")
return
}
case "Number":
case "n":
z.Number, err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "Number")
return
}
case "Size":
case "s":
z.Size, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "Size")
return
}
case "ActualSize":
case "as":
z.ActualSize, err = dc.ReadInt64()
if err != nil {
err = msgp.WrapError(err, "ActualSize")
return
}
case "ModTime":
case "mt":
z.ModTime, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
case "index":
case "i":
z.Index, err = dc.ReadBytes(z.Index)
if err != nil {
err = msgp.WrapError(err, "Index")
@@ -635,6 +635,12 @@ func (z *ObjectPartInfo) DecodeMsg(dc *msgp.Reader) (err error) {
}
z.Checksums[za0001] = za0002
}
case "err":
z.Error, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Error")
return
}
default:
err = dc.Skip()
if err != nil {
@@ -649,8 +655,8 @@ func (z *ObjectPartInfo) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
// check for omitted fields
zb0001Len := uint32(7)
var zb0001Mask uint8 /* 7 bits */
zb0001Len := uint32(8)
var zb0001Mask uint8 /* 8 bits */
_ = zb0001Mask
if z.Index == nil {
zb0001Len--
@@ -660,6 +666,10 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
zb0001Len--
zb0001Mask |= 0x40
}
if z.Error == "" {
zb0001Len--
zb0001Mask |= 0x80
}
// variable map header, size zb0001Len
err = en.Append(0x80 | uint8(zb0001Len))
if err != nil {
@@ -668,8 +678,8 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
if zb0001Len == 0 {
return
}
// write "ETag"
err = en.Append(0xa4, 0x45, 0x54, 0x61, 0x67)
// write "e"
err = en.Append(0xa1, 0x65)
if err != nil {
return
}
@@ -678,8 +688,8 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "ETag")
return
}
// write "Number"
err = en.Append(0xa6, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
// write "n"
err = en.Append(0xa1, 0x6e)
if err != nil {
return
}
@@ -688,8 +698,8 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Number")
return
}
// write "Size"
err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
// write "s"
err = en.Append(0xa1, 0x73)
if err != nil {
return
}
@@ -698,8 +708,8 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Size")
return
}
// write "ActualSize"
err = en.Append(0xaa, 0x41, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65)
// write "as"
err = en.Append(0xa2, 0x61, 0x73)
if err != nil {
return
}
@@ -708,8 +718,8 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "ActualSize")
return
}
// write "ModTime"
err = en.Append(0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
// write "mt"
err = en.Append(0xa2, 0x6d, 0x74)
if err != nil {
return
}
@@ -719,8 +729,8 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
return
}
if (zb0001Mask & 0x20) == 0 { // if not omitted
// write "index"
err = en.Append(0xa5, 0x69, 0x6e, 0x64, 0x65, 0x78)
// write "i"
err = en.Append(0xa1, 0x69)
if err != nil {
return
}
@@ -754,6 +764,18 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
}
}
}
if (zb0001Mask & 0x80) == 0 { // if not omitted
// write "err"
err = en.Append(0xa3, 0x65, 0x72, 0x72)
if err != nil {
return
}
err = en.WriteString(z.Error)
if err != nil {
err = msgp.WrapError(err, "Error")
return
}
}
return
}
@@ -761,8 +783,8 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// check for omitted fields
zb0001Len := uint32(7)
var zb0001Mask uint8 /* 7 bits */
zb0001Len := uint32(8)
var zb0001Mask uint8 /* 8 bits */
_ = zb0001Mask
if z.Index == nil {
zb0001Len--
@@ -772,29 +794,33 @@ func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) {
zb0001Len--
zb0001Mask |= 0x40
}
if z.Error == "" {
zb0001Len--
zb0001Mask |= 0x80
}
// variable map header, size zb0001Len
o = append(o, 0x80|uint8(zb0001Len))
if zb0001Len == 0 {
return
}
// string "ETag"
o = append(o, 0xa4, 0x45, 0x54, 0x61, 0x67)
// string "e"
o = append(o, 0xa1, 0x65)
o = msgp.AppendString(o, z.ETag)
// string "Number"
o = append(o, 0xa6, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
// string "n"
o = append(o, 0xa1, 0x6e)
o = msgp.AppendInt(o, z.Number)
// string "Size"
o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
// string "s"
o = append(o, 0xa1, 0x73)
o = msgp.AppendInt64(o, z.Size)
// string "ActualSize"
o = append(o, 0xaa, 0x41, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65)
// string "as"
o = append(o, 0xa2, 0x61, 0x73)
o = msgp.AppendInt64(o, z.ActualSize)
// string "ModTime"
o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
// string "mt"
o = append(o, 0xa2, 0x6d, 0x74)
o = msgp.AppendTime(o, z.ModTime)
if (zb0001Mask & 0x20) == 0 { // if not omitted
// string "index"
o = append(o, 0xa5, 0x69, 0x6e, 0x64, 0x65, 0x78)
// string "i"
o = append(o, 0xa1, 0x69)
o = msgp.AppendBytes(o, z.Index)
}
if (zb0001Mask & 0x40) == 0 { // if not omitted
@@ -806,6 +832,11 @@ func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.AppendString(o, za0002)
}
}
if (zb0001Mask & 0x80) == 0 { // if not omitted
// string "err"
o = append(o, 0xa3, 0x65, 0x72, 0x72)
o = msgp.AppendString(o, z.Error)
}
return
}
@@ -827,37 +858,37 @@ func (z *ObjectPartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
return
}
switch msgp.UnsafeString(field) {
case "ETag":
case "e":
z.ETag, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ETag")
return
}
case "Number":
case "n":
z.Number, bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Number")
return
}
case "Size":
case "s":
z.Size, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Size")
return
}
case "ActualSize":
case "as":
z.ActualSize, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ActualSize")
return
}
case "ModTime":
case "mt":
z.ModTime, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
case "index":
case "i":
z.Index, bts, err = msgp.ReadBytesBytes(bts, z.Index)
if err != nil {
err = msgp.WrapError(err, "Index")
@@ -893,6 +924,12 @@ func (z *ObjectPartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
}
z.Checksums[za0001] = za0002
}
case "err":
z.Error, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Error")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -907,13 +944,14 @@ func (z *ObjectPartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *ObjectPartInfo) Msgsize() (s int) {
s = 1 + 5 + msgp.StringPrefixSize + len(z.ETag) + 7 + msgp.IntSize + 5 + msgp.Int64Size + 11 + msgp.Int64Size + 8 + msgp.TimeSize + 6 + msgp.BytesPrefixSize + len(z.Index) + 4 + msgp.MapHeaderSize
s = 1 + 2 + msgp.StringPrefixSize + len(z.ETag) + 2 + msgp.IntSize + 2 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.TimeSize + 2 + msgp.BytesPrefixSize + len(z.Index) + 4 + msgp.MapHeaderSize
if z.Checksums != nil {
for za0001, za0002 := range z.Checksums {
_ = za0002
s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002)
}
}
s += 4 + msgp.StringPrefixSize + len(z.Error)
return
}