fix site replication resync status (#18245)

To persist status changes on disk upon completion.

Adds new tests to handle this functionality.

cmd/site-replication-utils.go

@@ -198,9 +198,9 @@ func (sm *siteResyncMetrics) save(ctx context.Context) {
 }
 
 // update overall site resync state
-func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) {
+func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) error {
 	if !globalSiteReplicationSys.isEnabled() {
-		return
+		return nil
 	}
 	sm.Lock()
 	defer sm.Unlock()
@@ -213,9 +213,12 @@ func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) {
 		if ok {
 			st.LastUpdate = s.LastUpdate
 			st.Status = s.Status
+			return nil
 		}
 		sm.resyncStatus[s.ResyncID] = st
+		return saveSiteResyncMetadata(GlobalContext, st, newObjectLayerFn())
 	}
+	return nil
 }
 
 // increment SyncedBuckets count

cmd/site-replication.go

@@ -5157,8 +5157,7 @@ func (c *SiteReplicationSys) startResync(ctx context.Context, objAPI ObjectLayer
 		}
 	}()
 
-	globalSiteResyncMetrics.updateState(rs)
-	if err := saveSiteResyncMetadata(ctx, rs, objAPI); err != nil {
+	if err := globalSiteResyncMetrics.updateState(rs); err != nil {
 		return res, err
 	}
 
@@ -5408,6 +5407,9 @@ func loadSiteResyncMetadata(ctx context.Context, objAPI ObjectLayer, dID string)
 
 // save resync status of peer to resync/depl-id.meta
 func saveSiteResyncMetadata(ctx context.Context, ss SiteResyncStatus, objectAPI ObjectLayer) error {
+	if objectAPI == nil {
+		return errSRObjectLayerNotReady
+	}
 	data := make([]byte, 4, ss.Msgsize()+4)
 
 	// Initialize the resync meta header.