replication: improve MRF healing. (#15556)

This PR improves the replication failure healing by persisting
most recent failures to disk and re-queuing them until the replication
is successful.

While this does not eliminate the need for healing during a full scan, 
queuing MRF vastly improves the ETA to keeping replicated buckets 
in sync as it does not wait for the scanner visit to detect unreplicated 
object versions.
This commit is contained in:
Poorna
2022-08-22 16:53:06 -07:00
committed by GitHub
parent 471467d310
commit 4155c5b695
4 changed files with 886 additions and 28 deletions

View File

@@ -737,3 +737,32 @@ func extractReplicateDiffOpts(q url.Values) (opts madmin.ReplDiffOpts) {
opts.Prefix = q.Get("prefix")
return
}
const (
replicationMRFDir = bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "mrf"
mrfMetaFormat = 1
mrfMetaVersionV1 = 1
mrfMetaVersion = mrfMetaVersionV1
)
// MRFReplicateEntry mrf entry to save to disk
type MRFReplicateEntry struct {
Bucket string `json:"bucket" msg:"b"`
Object string `json:"object" msg:"o"`
versionID string `json:"-"`
}
// MRFReplicateEntries has the map of MRF entries to save to disk
type MRFReplicateEntries struct {
Entries map[string]MRFReplicateEntry `json:"entries" msg:"e"`
Version int `json:"version" msg:"v"`
}
// ToMRFEntry returns the relevant info needed by MRF
func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
return MRFReplicateEntry{
Bucket: ri.Bucket,
Object: ri.Name,
versionID: ri.VersionID,
}
}