From 78f1f69d57fc4e347887ebee45d7f11e6916585a Mon Sep 17 00:00:00 2001 From: Poorna Date: Fri, 13 Oct 2023 22:17:22 -0700 Subject: [PATCH] fix site replication resync status (#18245) To persist status changes on disk upon completion. Adds new tests to handle this functionality. --- cmd/site-replication-utils.go | 7 +++++-- cmd/site-replication.go | 6 ++++-- docs/site-replication/run-multi-site-ldap.sh | 19 ++++++++++++++++++- .../run-multi-site-minio-idp.sh | 19 ++++++++++++++++++- docs/site-replication/run-multi-site-oidc.sh | 17 +++++++++++++++++ 5 files changed, 62 insertions(+), 6 deletions(-) diff --git a/cmd/site-replication-utils.go b/cmd/site-replication-utils.go index ac85806c7..d81a8aee5 100644 --- a/cmd/site-replication-utils.go +++ b/cmd/site-replication-utils.go @@ -198,9 +198,9 @@ func (sm *siteResyncMetrics) save(ctx context.Context) { } // update overall site resync state -func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) { +func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) error { if !globalSiteReplicationSys.isEnabled() { - return + return nil } sm.Lock() defer sm.Unlock() @@ -213,9 +213,12 @@ func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) { if ok { st.LastUpdate = s.LastUpdate st.Status = s.Status + return nil } sm.resyncStatus[s.ResyncID] = st + return saveSiteResyncMetadata(GlobalContext, st, newObjectLayerFn()) } + return nil } // increment SyncedBuckets count diff --git a/cmd/site-replication.go b/cmd/site-replication.go index 9d7cc7436..0d883674e 100644 --- a/cmd/site-replication.go +++ b/cmd/site-replication.go @@ -5157,8 +5157,7 @@ func (c *SiteReplicationSys) startResync(ctx context.Context, objAPI ObjectLayer } }() - globalSiteResyncMetrics.updateState(rs) - if err := saveSiteResyncMetadata(ctx, rs, objAPI); err != nil { + if err := globalSiteResyncMetrics.updateState(rs); err != nil { return res, err } @@ -5408,6 +5407,9 @@ func loadSiteResyncMetadata(ctx context.Context, objAPI ObjectLayer, dID string) // save resync status of peer to resync/depl-id.meta func saveSiteResyncMetadata(ctx context.Context, ss SiteResyncStatus, objectAPI ObjectLayer) error { + if objectAPI == nil { + return errSRObjectLayerNotReady + } data := make([]byte, 4, ss.Msgsize()+4) // Initialize the resync meta header. diff --git a/docs/site-replication/run-multi-site-ldap.sh b/docs/site-replication/run-multi-site-ldap.sh index 102ec8f0a..e42e809b2 100755 --- a/docs/site-replication/run-multi-site-ldap.sh +++ b/docs/site-replication/run-multi-site-ldap.sh @@ -247,7 +247,7 @@ if [ $? -ne 0 ]; then echo "expecting object to be present. exiting.." exit_1 fi -./mc tag set --version-id "${vID}" minio2/newbucket/README.md "k=v" +./mc tag set --version-id "${vID}" minio2/newbucket/README.md "key=val" if [ $? -ne 0 ]; then echo "expecting tag set to be successful. exiting.." exit_1 @@ -353,4 +353,21 @@ if [ $? -ne 0 ]; then exit_1 fi +# force a resync after removing all site replication +./mc admin replicate rm --all --force minio1 +./mc rb minio2 --force --dangerous +./mc admin replicate add minio1 minio2 +./mc admin replicate resync start minio1 minio2 +sleep 30 + +./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt +./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt + +out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after replication resync: $out" + exit 1 +fi + cleanup diff --git a/docs/site-replication/run-multi-site-minio-idp.sh b/docs/site-replication/run-multi-site-minio-idp.sh index 02d3000f4..d75d450ed 100755 --- a/docs/site-replication/run-multi-site-minio-idp.sh +++ b/docs/site-replication/run-multi-site-minio-idp.sh @@ -234,7 +234,7 @@ if [ $? -ne 0 ]; then echo "expecting object to be present. exiting.." exit_1 fi -./mc tag set --version-id "${vID}" minio2/newbucket/README.md "k=v" +./mc tag set --version-id "${vID}" minio2/newbucket/README.md "key=val" if [ $? -ne 0 ]; then echo "expecting tag set to be successful. exiting.." exit_1 @@ -370,3 +370,20 @@ if [ $? -ne 0 ]; then echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." exit_1 fi + +# force a resync after removing all site replication +./mc admin replicate rm --all --force minio1 +./mc rb minio2 --force --dangerous +./mc admin replicate add minio1 minio2 +./mc admin replicate resync start minio1 minio2 +sleep 30 + +./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt +./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt + +out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after replication resync: $out" + exit 1 +fi diff --git a/docs/site-replication/run-multi-site-oidc.sh b/docs/site-replication/run-multi-site-oidc.sh index 27d6025b5..2e537a603 100755 --- a/docs/site-replication/run-multi-site-oidc.sh +++ b/docs/site-replication/run-multi-site-oidc.sh @@ -293,3 +293,20 @@ if [ $? -ne 0 ]; then echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." exit_1 fi + +# force a resync after removing all site replication +./mc admin replicate rm --all --force minio1 +./mc rb minio2 --force --dangerous +./mc admin replicate add minio1 minio2 +./mc admin replicate resync start minio1 minio2 +sleep 30 + +./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt +./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt + +out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after replication resync: $out" + exit 1 +fi