MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 12:52:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

replication: Simplify mrf requeueing and add backlog handler (#17171)

Browse Source 
Simplify MRF queueing and add backlog handler

- Limit re-tries to 3 to avoid repeated re-queueing. Fall offs
to be re-tried when the scanner revisits this object or upon access.

- Change MRF to have each node process only its MRF entries.

- Collect MRF backlog by the node to allow for current backlog visibility
This commit is contained in:
Poorna
2023-07-13 02:51:33 -04:00
committed by GitHub
parent 9b9871cfbb
commit 5e2f8d7a42
12 changed files with 389 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
cmd/bucket-replication-utils_gen.go
View File

@@ -327,6 +327,12 @@ func (z *MRFReplicateEntries) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "Entries", za0001, "Object")
return
}
case "rc":
za0002.RetryCount, err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "Entries", za0001, "RetryCount")
return
}
default:
err = dc.Skip()
if err != nil {
@@ -373,9 +379,9 @@ func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Entries")
return
}
// map header, size 2
// map header, size 3
// write "b"
err = en.Append(0x82, 0xa1, 0x62)
err = en.Append(0x83, 0xa1, 0x62)
if err != nil {
return
}
@@ -394,6 +400,16 @@ func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Entries", za0001, "Object")
return
}
// write "rc"
err = en.Append(0xa2, 0x72, 0x63)
if err != nil {
return
}
err = en.WriteInt(za0002.RetryCount)
if err != nil {
err = msgp.WrapError(err, "Entries", za0001, "RetryCount")
return
}
}
// write "v"
err = en.Append(0xa1, 0x76)
@@ -417,13 +433,16 @@ func (z *MRFReplicateEntries) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
for za0001, za0002 := range z.Entries {
o = msgp.AppendString(o, za0001)
// map header, size 2
// map header, size 3
// string "b"
o = append(o, 0x82, 0xa1, 0x62)
o = append(o, 0x83, 0xa1, 0x62)
o = msgp.AppendString(o, za0002.Bucket)
// string "o"
o = append(o, 0xa1, 0x6f)
o = msgp.AppendString(o, za0002.Object)
// string "rc"
o = append(o, 0xa2, 0x72, 0x63)
o = msgp.AppendInt(o, za0002.RetryCount)
}
// string "v"
o = append(o, 0xa1, 0x76)
@@ -498,6 +517,12 @@ func (z *MRFReplicateEntries) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "Entries", za0001, "Object")
return
}
case "rc":
za0002.RetryCount, bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Entries", za0001, "RetryCount")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -532,7 +557,7 @@ func (z *MRFReplicateEntries) Msgsize() (s int) {
if z.Entries != nil {
for za0001, za0002 := range z.Entries {
_ = za0002
s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object)
s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object) + 3 + msgp.IntSize
}
}
s += 2 + msgp.IntSize
@@ -569,6 +594,12 @@ func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "Object")
return
}
case "rc":
z.RetryCount, err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "RetryCount")
return
}
default:
err = dc.Skip()
if err != nil {
@@ -582,9 +613,9 @@ func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
// map header, size 3
// write "b"
err = en.Append(0x82, 0xa1, 0x62)
err = en.Append(0x83, 0xa1, 0x62)
if err != nil {
return
}
@@ -603,19 +634,32 @@ func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Object")
return
}
// write "rc"
err = en.Append(0xa2, 0x72, 0x63)
if err != nil {
return
}
err = en.WriteInt(z.RetryCount)
if err != nil {
err = msgp.WrapError(err, "RetryCount")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z MRFReplicateEntry) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
// map header, size 3
// string "b"
o = append(o, 0x82, 0xa1, 0x62)
o = append(o, 0x83, 0xa1, 0x62)
o = msgp.AppendString(o, z.Bucket)
// string "o"
o = append(o, 0xa1, 0x6f)
o = msgp.AppendString(o, z.Object)
// string "rc"
o = append(o, 0xa2, 0x72, 0x63)
o = msgp.AppendInt(o, z.RetryCount)
return
}
@@ -649,6 +693,12 @@ func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "Object")
return
}
case "rc":
z.RetryCount, bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "RetryCount")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -663,7 +713,7 @@ func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z MRFReplicateEntry) Msgsize() (s int) {
s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object)
s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.IntSize
return
}