Fix site replication meta info call to be non-blocking (#16526)

Co-authored-by: Harshavardhana <harsha@minio.io>

cmd/iam-store.go

@@ -874,6 +874,25 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 	return
 }
 
+// listGroups - lists groups - fetch groups from cache
+func (store *IAMStoreSys) listGroups(ctx context.Context) (res []string, err error) {
+	cache := store.rlock()
+	defer store.runlock()
+
+	if store.getUsersSysType() == MinIOUsersSysType {
+		for k := range cache.iamGroupsMap {
+			res = append(res, k)
+		}
+	}
+
+	if store.getUsersSysType() == LDAPUsersSysType {
+		for k := range cache.iamGroupPolicyMap {
+			res = append(res, k)
+		}
+	}
+	return
+}
+
 // PolicyDBUpdate - adds or removes given policies to/from the user or group's
 // policy associations.
 func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGroup bool,
@@ -1231,6 +1250,20 @@ func (store *IAMStoreSys) ListPolicyDocs(ctx context.Context, bucketName string)
 	return ret, nil
 }
 
+// fetches all policy docs from cache.
+// If bucketName is non-empty, returns policy docs matching the bucket.
+func (store *IAMStoreSys) listPolicyDocs(ctx context.Context, bucketName string) (map[string]PolicyDoc, error) {
+	cache := store.rlock()
+	defer store.runlock()
+	ret := map[string]PolicyDoc{}
+	for k, v := range cache.iamPolicyDocsMap {
+		if bucketName == "" || v.Policy.MatchResource(bucketName) {
+			ret[k] = v
+		}
+	}
+	return ret, nil
+}
+
 // helper function - does not take locks.
 func filterPolicies(cache *iamCache, policyName string, bucketName string) (string, iampolicy.Policy) {
 	var policies []string

cmd/site-replication.go

@@ -191,6 +191,9 @@ type SiteReplicationSys struct {
 
 	// In-memory and persisted multi-site replication state.
 	state srState
+
+	iamMetaCache srIAMCache
+	iamInfoLk    sync.RWMutex
 }
 
 type srState srStateV1
@@ -3124,6 +3127,32 @@ func isBktReplCfgReplicated(total int, cfgs []*sreplication.Config) bool {
 	return true
 }
 
+// cache of IAM info fetched in last SiteReplicationMetaInfo call
+type srIAMCache struct {
+	sync.RWMutex
+	lastUpdate time.Time
+	srIAMInfo  madmin.SRInfo // caches IAM info
+}
+
+func (c *SiteReplicationSys) getSRCachedIAMInfo() (info madmin.SRInfo, ok bool) {
+	c.iamMetaCache.RLock()
+	defer c.iamMetaCache.RUnlock()
+	if c.iamMetaCache.lastUpdate.IsZero() {
+		return info, false
+	}
+	if time.Since(c.iamMetaCache.lastUpdate) < siteHealTimeInterval {
+		return c.iamMetaCache.srIAMInfo, true
+	}
+	return info, false
+}
+
+func (c *SiteReplicationSys) srCacheIAMInfo(info madmin.SRInfo) {
+	c.iamMetaCache.Lock()
+	defer c.iamMetaCache.Unlock()
+	c.iamMetaCache.srIAMInfo = info
+	c.iamMetaCache.lastUpdate = time.Now()
+}
+
 // SiteReplicationMetaInfo returns the metadata info on buckets, policies etc for the replicated site
 func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI ObjectLayer, opts madmin.SRStatusOptions) (info madmin.SRInfo, err error) {
 	if objAPI == nil {
@@ -3217,6 +3246,15 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 		}
 	}
 
+	if opts.Users && opts.Groups && opts.Policies && !opts.Buckets {
+		// serialize SiteReplicationMetaInfo calls - if data in cache is within
+		// healing interval, avoid fetching IAM data again from disk.
+		c.iamInfoLk.Lock()
+		defer c.iamInfoLk.Unlock()
+		if metaInfo, ok := c.getSRCachedIAMInfo(); ok {
+			return metaInfo, nil
+		}
+	}
 	if opts.Policies || opts.Entity == madmin.SRPolicyEntity {
 		var allPolicies map[string]PolicyDoc
 		if opts.Entity == madmin.SRPolicyEntity {
@@ -3225,7 +3263,7 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 			}
 		} else {
 			// Replicate IAM policies on local to all peers.
-			allPolicies, err = globalIAMSys.ListPolicyDocs(ctx, "")
+			allPolicies, err = globalIAMSys.store.listPolicyDocs(ctx, "")
 			if err != nil {
 				return info, errSRBackendIssue(err)
 			}
@@ -3361,7 +3399,7 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 			}
 		} else {
 			//  get users/group info on local.
-			groups, errG := globalIAMSys.ListGroups(ctx)
+			groups, errG := globalIAMSys.store.listGroups(ctx)
 			if errG != nil {
 				return info, errSRBackendIssue(errG)
 			}
@@ -3377,6 +3415,11 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 			}
 		}
 	}
+	// cache SR metadata info for IAM
+	if opts.Users && opts.Groups && opts.Policies && !opts.Buckets {
+		c.srCacheIAMInfo(info)
+	}
+
 	return info, nil
 }