fix: optimize IAM users load, add fallback (#9809)

Bonus fix, load service accounts properly
when service accounts were generated with
LDAP

cmd/iam.go

@@ -216,7 +216,8 @@ type IAMSys struct {
 	iamGroupPolicyMap map[string]MappedPolicy
 
 	// Persistence layer for IAM subsystem
-	store IAMStorageAPI
+	store         IAMStorageAPI
+	storeFallback bool
 }
 
 // IAMUserType represents a user type inside MinIO server
@@ -413,7 +414,7 @@ func startBackgroundIAMLoad(ctx context.Context) {
 	go globalIAMSys.Init(ctx, newObjectLayerWithoutSafeModeFn())
 }
 
-// Init - initializes config system from iam.json
+// Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	if objAPI == nil {
 		logger.LogIf(ctx, errServerNotInitialized)
@@ -461,7 +462,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 			// IAM sub-system, make sure that we do not move the above codeblock elsewhere.
 			if err := migrateIAMConfigsEtcdToEncrypted(ctx, globalEtcdClient); err != nil {
 				txnLk.Unlock()
-				logger.LogIf(ctx, fmt.Errorf("Unable to handle encrypted backend for iam and policies: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to decrypt an encrypted ETCD backend for IAM users and policies: %w", err))
+				logger.LogIf(ctx, errors.New("IAM sub-system is partially initialized, some users may not be available"))
 				return
 			}
 		}
@@ -471,7 +473,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 			logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. lock acquired")
 		}
 
-		// Migrate IAM configuration
+		// Migrate IAM configuration, if necessary.
 		if err := sys.doIAMConfigMigration(ctx); err != nil {
 			txnLk.Unlock()
 			if errors.Is(err, errDiskNotFound) ||
@@ -484,20 +486,25 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
 				continue
 			}
-			logger.LogIf(ctx, fmt.Errorf("Unable to migration IAM users and policies: %w", err))
+			logger.LogIf(ctx, fmt.Errorf("Unable to migrate IAM users and policies to new format: %w", err))
+			logger.LogIf(ctx, errors.New("IAM sub-system is partially initialized, some users may not be available"))
 			return
 		}
 
-		// Successfully migrated
+		// Successfully migrated, proceed to load the users.
 		txnLk.Unlock()
 		break
 	}
 
-	logger.LogIf(ctx, sys.store.loadAll(ctx, sys))
+	err := sys.store.loadAll(ctx, sys)
 
-	// Invalidate the old cred after finishing IAM initialization
+	// Invalidate the old cred always, even upon error to avoid any leakage.
 	globalOldCred = auth.Credentials{}
 
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize IAM sub-system, some users may not be available %w", err))
+	}
+
 	go sys.store.watch(ctx, sys)
 }
 
@@ -582,6 +589,10 @@ func (sys *IAMSys) ListPolicies() (map[string]iampolicy.Policy, error) {
 	sys.store.rlock()
 	defer sys.store.runlock()
 
+	if sys.storeFallback {
+		return nil, errIAMNotInitialized
+	}
+
 	policyDocsMap := make(map[string]iampolicy.Policy, len(sys.iamPolicyDocsMap))
 	for k, v := range sys.iamPolicyDocsMap {
 		policyDocsMap[k] = v
@@ -731,6 +742,10 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 		return nil, errIAMActionNotAllowed
 	}
 
+	if sys.storeFallback {
+		return nil, errIAMNotInitialized
+	}
+
 	for k, v := range sys.iamUsersMap {
 		if !v.IsTemp() && !v.IsServiceAccount() {
 			users[k] = madmin.UserInfo{
@@ -965,6 +980,10 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 	sys.store.rlock()
 	defer sys.store.runlock()
 
+	if sys.storeFallback {
+		return nil, errIAMNotInitialized
+	}
+
 	var serviceAccounts []string
 
 	for k, v := range sys.iamUsersMap {
@@ -1096,6 +1115,44 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 		return cred, false
 	}
 
+	sys.store.rlock()
+	fallback := sys.storeFallback
+	sys.store.runlock()
+	if fallback {
+		sys.store.lock()
+		// If user is already found proceed.
+		if _, found := sys.iamUsersMap[accessKey]; !found {
+			sys.store.loadUser(accessKey, regularUser, sys.iamUsersMap)
+			if _, found = sys.iamUsersMap[accessKey]; found {
+				// found user, load its mapped policies
+				sys.store.loadMappedPolicy(accessKey, regularUser, false, sys.iamUserPolicyMap)
+			} else {
+				sys.store.loadUser(accessKey, srvAccUser, sys.iamUsersMap)
+				if svc, found := sys.iamUsersMap[accessKey]; found {
+					// Found service account, load its parent user and its mapped policies.
+					if sys.usersSysType == MinIOUsersSysType {
+						sys.store.loadUser(svc.ParentUser, regularUser, sys.iamUsersMap)
+					}
+					sys.store.loadMappedPolicy(svc.ParentUser, regularUser, false, sys.iamUserPolicyMap)
+				} else {
+					// None found fall back to STS users.
+					sys.store.loadUser(accessKey, stsUser, sys.iamUsersMap)
+					if _, found = sys.iamUsersMap[accessKey]; found {
+						// STS user found, load its mapped policy.
+						sys.store.loadMappedPolicy(accessKey, stsUser, false, sys.iamUserPolicyMap)
+					}
+				}
+			}
+		}
+		// Load associated policies if any.
+		for _, policy := range sys.iamUserPolicyMap[accessKey].toSlice() {
+			if _, found := sys.iamPolicyDocsMap[policy]; !found {
+				sys.store.loadPolicyDoc(policy, sys.iamPolicyDocsMap)
+			}
+		}
+		sys.store.unlock()
+	}
+
 	sys.store.rlock()
 	defer sys.store.runlock()
 
@@ -1344,6 +1401,10 @@ func (sys *IAMSys) ListGroups() (r []string, err error) {
 	sys.store.rlock()
 	defer sys.store.runlock()
 
+	if sys.storeFallback {
+		return nil, errIAMNotInitialized
+	}
+
 	if sys.usersSysType != MinIOUsersSysType {
 		return nil, errIAMActionNotAllowed
 	}
@@ -1869,5 +1930,6 @@ func NewIAMSys() *IAMSys {
 		iamGroupPolicyMap:       make(map[string]MappedPolicy),
 		iamGroupsMap:            make(map[string]GroupInfo),
 		iamUserGroupMemberships: make(map[string]set.StringSet),
+		storeFallback:           true,
 	}
 }