Try reconnect IAM systems if failed initially (#20333)

Fixes: https://github.com/minio/minio/issues/20118

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>

cmd/iam.go

@@ -248,62 +248,135 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	s := globalServerConfig
 	globalServerConfigMu.RUnlock()
 
-	openidConfig, err := openid.LookupConfig(s,
+	sys.Lock()
-		NewHTTPTransport(), xhttp.DrainBody, globalSite.Region())
+	sys.iamRefreshInterval = iamRefreshInterval
-	if err != nil {
+	sys.Unlock()
-		iamLogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err), logger.WarningKind)
-	}
 
-	// Initialize if LDAP is enabled
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	ldapConfig, err := xldap.Lookup(s, globalRootCAs)
+
-	if err != nil {
+	var (
-		iamLogIf(ctx, fmt.Errorf("Unable to load LDAP configuration (LDAP configuration will be disabled!): %w", err), logger.WarningKind)
+		openidInit bool
-	}
+		ldapInit   bool
+		authNInit  bool
+		authZInit  bool
+	)
 
 	stsTLSConfig, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
 	if err != nil {
 		iamLogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err), logger.WarningKind)
-	}
+	} else {
-
+		if stsTLSConfig.InsecureSkipVerify {
-	if stsTLSConfig.InsecureSkipVerify {
+			iamLogIf(ctx, fmt.Errorf("Enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify), logger.WarningKind)
-		iamLogIf(ctx, fmt.Errorf("Enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify), logger.WarningKind)
-	}
-
-	authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
-		NewHTTPTransport(), xhttp.DrainBody, globalSite.Region())
-	if err != nil {
-		iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err), logger.WarningKind)
-	}
-
-	setGlobalAuthNPlugin(idplugin.New(GlobalContext, authNPluginCfg))
-
-	authZPluginCfg, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody)
-	if err != nil {
-		iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err), logger.WarningKind)
-	}
-
-	if authZPluginCfg.URL == nil {
-		opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
-			NewHTTPTransport(), xhttp.DrainBody)
-		if err != nil {
-			iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
-		} else {
-			authZPluginCfg.URL = opaCfg.URL
-			authZPluginCfg.AuthToken = opaCfg.AuthToken
-			authZPluginCfg.Transport = opaCfg.Transport
-			authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn
 		}
+		sys.Lock()
+		sys.STSTLSConfig = stsTLSConfig
+		sys.Unlock()
 	}
 
-	setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
+	for {
+		if !openidInit {
+			openidConfig, err := openid.LookupConfig(s,
+				NewHTTPTransport(), xhttp.DrainBody, globalSite.Region())
+			if err != nil {
+				iamLogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err), logger.WarningKind)
+			} else {
+				openidInit = true
+				sys.Lock()
+				sys.OpenIDConfig = openidConfig
+				sys.Unlock()
+			}
+		}
+
+		if !ldapInit {
+			// Initialize if LDAP is enabled
+			ldapConfig, err := xldap.Lookup(s, globalRootCAs)
+			if err != nil {
+				iamLogIf(ctx, fmt.Errorf("Unable to load LDAP configuration (LDAP configuration will be disabled!): %w", err), logger.WarningKind)
+			} else {
+				ldapInit = true
+				sys.Lock()
+				sys.LDAPConfig = ldapConfig
+				sys.Unlock()
+			}
+		}
+
+		if !authNInit {
+			authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
+				NewHTTPTransport(), xhttp.DrainBody, globalSite.Region())
+			if err != nil {
+				iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err), logger.WarningKind)
+			} else {
+				authNInit = true
+				setGlobalAuthNPlugin(idplugin.New(GlobalContext, authNPluginCfg))
+			}
+		}
+
+		if !authZInit {
+			authZPluginCfg, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody)
+			if err != nil {
+				iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err), logger.WarningKind)
+			} else {
+				authZInit = true
+			}
+			if authZPluginCfg.URL == nil {
+				opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
+					NewHTTPTransport(), xhttp.DrainBody)
+				if err != nil {
+					iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
+				} else {
+					authZPluginCfg.URL = opaCfg.URL
+					authZPluginCfg.AuthToken = opaCfg.AuthToken
+					authZPluginCfg.Transport = opaCfg.Transport
+					authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn
+					authZInit = true
+				}
+			}
+			if authZInit {
+				setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
+			}
+		}
+
+		if !openidInit || !ldapInit || !authNInit || !authZInit {
+			retryInterval := time.Duration(r.Float64() * float64(3*time.Second))
+			if !openidInit {
+				logger.Info("Waiting for OpenID to be initialized.. (retrying in %s)", retryInterval)
+			}
+			if !ldapInit {
+				logger.Info("Waiting for LDAP to be initialized.. (retrying in %s)", retryInterval)
+			}
+			if !authNInit {
+				logger.Info("Waiting for AuthN to be initialized.. (retrying in %s)", retryInterval)
+			}
+			if !authZInit {
+				logger.Info("Waiting for AuthZ to be initialized.. (retrying in %s)", retryInterval)
+			}
+			time.Sleep(retryInterval)
+			continue
+		}
+
+		break
+	}
 
-	sys.Lock()
-	sys.LDAPConfig = ldapConfig
-	sys.OpenIDConfig = openidConfig
-	sys.STSTLSConfig = stsTLSConfig
-	sys.iamRefreshInterval = iamRefreshInterval
 	// Initialize IAM store
+	sys.Lock()
+
 	sys.initStore(objAPI, etcdClient)
+
+	// Initialize RoleARNs
+	sys.rolesMap = make(map[arn.ARN]string)
+
+	// From OpenID
+	if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
+	}
+
+	// From AuthN plugin if enabled.
+	if authn := newGlobalAuthNPluginFn(); authn != nil {
+		riMap := authn.GetRoleInfo()
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
+	}
+
+	sys.printIAMRoles()
 	sys.Unlock()
 
 	retryCtx, cancel := context.WithCancel(ctx)
@@ -311,8 +384,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	// Indicate to our routine to exit cleanly upon return.
 	defer cancel()
 
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
 	// Migrate storage format if needed.
 	for {
 		// Migrate IAM configuration, if necessary.
@@ -334,20 +405,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
 	sys.store.unlock()
 
-	// Load RoleARNs
-	sys.rolesMap = make(map[arn.ARN]string)
-
-	// From OpenID
-	if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
-		sys.validateAndAddRolePolicyMappings(ctx, riMap)
-	}
-
-	// From AuthN plugin if enabled.
-	if authn := newGlobalAuthNPluginFn(); authn != nil {
-		riMap := authn.GetRoleInfo()
-		sys.validateAndAddRolePolicyMappings(ctx, riMap)
-	}
-
 	// Load IAM data from storage.
 	for {
 		if err := sys.Load(retryCtx, true); err != nil {
@@ -365,11 +422,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	}
 
 	refreshInterval := sys.iamRefreshInterval
-
 	go sys.periodicRoutines(ctx, refreshInterval)
 
-	sys.printIAMRoles()
-
 	bootstrapTraceMsg("finishing IAM loading")
 }