fix site replication resync status (#18245)

To persist status changes on disk upon completion.

Adds new tests to handle this functionality.

cmd/site-replication-utils.go

@@ -198,9 +198,9 @@ func (sm *siteResyncMetrics) save(ctx context.Context) {
 }
 
 // update overall site resync state
-func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) {
+func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) error {
 	if !globalSiteReplicationSys.isEnabled() {
-		return
+		return nil
 	}
 	sm.Lock()
 	defer sm.Unlock()
@@ -213,9 +213,12 @@ func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) {
 		if ok {
 			st.LastUpdate = s.LastUpdate
 			st.Status = s.Status
+			return nil
 		}
 		sm.resyncStatus[s.ResyncID] = st
+		return saveSiteResyncMetadata(GlobalContext, st, newObjectLayerFn())
 	}
+	return nil
 }
 
 // increment SyncedBuckets count