fix: load credentials from etcd directly when possible (#11339)

under large deployments loading credentials might be
time consuming, while this is okay and we will not
respond quickly for `mc admin user list` like queries
but it is possible to support `mc admin user info`

just like how we handle authentication by fetching
the user directly from persistent store.

additionally support service accounts properly,
reloaded from etcd during watch() - this was missing

This PR is also half way remedy for #11305

cmd/admin-handlers-users.go

@@ -426,7 +426,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
+	if err = globalIAMSys.CreateUser(accessKey, uinfo); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/config/etcd/etcd.go

@@ -27,6 +27,7 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 	"go.etcd.io/etcd/clientv3"
 	"go.etcd.io/etcd/clientv3/namespace"
+	"go.uber.org/zap"
 )
 
 const (
@@ -144,6 +145,13 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
 	cfg.Enabled = true
 	cfg.DialTimeout = defaultDialTimeout
 	cfg.DialKeepAliveTime = defaultDialKeepAlive
+	// Disable etcd client SDK logging, etcd client
+	// incorrectly starts logging in unexpected data
+	// format.
+	cfg.LogConfig = &zap.Config{
+		Level:    zap.NewAtomicLevelAt(zap.FatalLevel),
+		Encoding: "console",
+	}
 	cfg.Endpoints = etcdEndpoints
 	cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath))
 	// Default path prefix for all keys on etcd, other than CoreDNSPath.

cmd/iam-etcd-store.go

@@ -264,6 +264,9 @@ func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map
 	var p iampolicy.Policy
 	err := ies.loadIAMConfig(ctx, &p, getPolicyDocPath(policy))
 	if err != nil {
+		if err == errConfigNotFound {
+			return errNoSuchPolicy
+		}
 		return err
 	}
 	m[policy] = p
@@ -282,8 +285,7 @@ func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]iampol
 
 	// Reload config and policies for all policys.
 	for _, policyName := range policies.ToSlice() {
-		err = ies.loadPolicyDoc(ctx, policyName, m)
+		if err = ies.loadPolicyDoc(ctx, policyName, m); err != nil && err != errNoSuchPolicy {
-		if err != nil {
 			return err
 		}
 	}
@@ -358,7 +360,7 @@ func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m
 
 	// Reload config for all users.
 	for _, user := range users.ToSlice() {
-		if err = ies.loadUser(ctx, user, userType, m); err != nil {
+		if err = ies.loadUser(ctx, user, userType, m); err != nil && err != errNoSuchUser {
 			return err
 		}
 	}
@@ -392,7 +394,7 @@ func (ies *IAMEtcdStore) loadGroups(ctx context.Context, m map[string]GroupInfo)
 
 	// Reload config for all groups.
 	for _, group := range groups.ToSlice() {
-		if err = ies.loadGroup(ctx, group, m); err != nil {
+		if err = ies.loadGroup(ctx, group, m); err != nil && err != errNoSuchGroup {
 			return err
 		}
 	}
@@ -440,7 +442,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 
 	// Reload config and policies for all users.
 	for _, user := range users.ToSlice() {
-		if err = ies.loadMappedPolicy(ctx, user, userType, isGroup, m); err != nil {
+		if err = ies.loadMappedPolicy(ctx, user, userType, isGroup, m); err != nil && err != errNoSuchPolicy {
 			return err
 		}
 	}
@@ -449,117 +451,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 }
 
 func (ies *IAMEtcdStore) loadAll(ctx context.Context, sys *IAMSys) error {
-	iamUsersMap := make(map[string]auth.Credentials)
+	return sys.Load(ctx, ies)
-	iamGroupsMap := make(map[string]GroupInfo)
-	iamUserPolicyMap := make(map[string]MappedPolicy)
-	iamGroupPolicyMap := make(map[string]MappedPolicy)
-
-	ies.rlock()
-	isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
-	ies.runlock()
-
-	ies.lock()
-	if err := ies.loadPolicyDocs(ctx, sys.iamPolicyDocsMap); err != nil {
-		ies.unlock()
-		return err
-	}
-	// Sets default canned policies, if none are set.
-	setDefaultCannedPolicies(sys.iamPolicyDocsMap)
-
-	ies.unlock()
-
-	if isMinIOUsersSys {
-		if err := ies.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
-			return err
-		}
-		if err := ies.loadGroups(ctx, iamGroupsMap); err != nil {
-			return err
-		}
-	}
-
-	// load polices mapped to users
-	if err := ies.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
-		return err
-	}
-
-	// load policies mapped to groups
-	if err := ies.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
-		return err
-	}
-
-	if err := ies.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
-		return err
-	}
-
-	// load STS temp users
-	if err := ies.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
-		return err
-	}
-
-	// load STS policy mappings
-	if err := ies.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
-		return err
-	}
-
-	ies.lock()
-	defer ies.Unlock()
-
-	// Merge the new reloaded entries into global map.
-	// See issue https://github.com/minio/minio/issues/9651
-	// where the present list of entries on disk are not yet
-	// latest, there is a small window where this can make
-	// valid users invalid.
-	for k, v := range iamUsersMap {
-		sys.iamUsersMap[k] = v
-	}
-
-	for k, v := range iamUserPolicyMap {
-		sys.iamUserPolicyMap[k] = v
-	}
-
-	// purge any expired entries which became expired now.
-	var expiredEntries []string
-	for k, v := range sys.iamUsersMap {
-		if v.IsExpired() {
-			delete(sys.iamUsersMap, k)
-			delete(sys.iamUserPolicyMap, k)
-			expiredEntries = append(expiredEntries, k)
-			// Deleting on the disk is taken care of in the next cycle
-		}
-	}
-
-	for _, v := range sys.iamUsersMap {
-		if v.IsServiceAccount() {
-			for _, accessKey := range expiredEntries {
-				if v.ParentUser == accessKey {
-					_ = ies.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
-					delete(sys.iamUsersMap, v.AccessKey)
-				}
-			}
-		}
-	}
-
-	// purge any expired entries which became expired now.
-	for k, v := range sys.iamUsersMap {
-		if v.IsExpired() {
-			delete(sys.iamUsersMap, k)
-			delete(sys.iamUserPolicyMap, k)
-			// Deleting on the etcd is taken care of in the next cycle
-		}
-	}
-
-	for k, v := range iamGroupPolicyMap {
-		sys.iamGroupPolicyMap[k] = v
-	}
-
-	for k, v := range iamGroupsMap {
-		sys.iamGroupsMap[k] = v
-	}
-
-	sys.buildUserGroupMemberships()
-	sys.storeFallback = false
-
-	return nil
 }
 
 func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
@@ -653,6 +545,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 	usersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigUsersPrefix)
 	groupsPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigGroupsPrefix)
 	stsPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigSTSPrefix)
+	svcPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigServiceAccountsPrefix)
 	policyPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPoliciesPrefix)
 	policyDBUsersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPolicyDBUsersPrefix)
 	policyDBSTSUsersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPolicyDBSTSUsersPrefix)
@@ -672,6 +565,10 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigSTSPrefix))
 			ies.loadUser(ctx, accessKey, stsUser, sys.iamUsersMap)
+		case svcPrefix:
+			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
+				iamConfigServiceAccountsPrefix))
+			ies.loadUser(ctx, accessKey, srvAccUser, sys.iamUsersMap)
 		case groupsPrefix:
 			group := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigGroupsPrefix))

cmd/iam-object-store.go

@@ -402,110 +402,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 	return nil
 }
 
-// Refresh IAMSys. If an object layer is passed in use that, otherwise
+// Refresh IAMSys. If an object layer is passed in use that, otherwise load from global.
-// load from global.
 func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
-	iamUsersMap := make(map[string]auth.Credentials)
+	return sys.Load(ctx, iamOS)
-	iamGroupsMap := make(map[string]GroupInfo)
-	iamUserPolicyMap := make(map[string]MappedPolicy)
-	iamGroupPolicyMap := make(map[string]MappedPolicy)
-
-	iamOS.rlock()
-	isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
-	iamOS.runlock()
-
-	iamOS.lock()
-	if err := iamOS.loadPolicyDocs(ctx, sys.iamPolicyDocsMap); err != nil {
-		iamOS.unlock()
-		return err
-	}
-	// Sets default canned policies, if none are set.
-	setDefaultCannedPolicies(sys.iamPolicyDocsMap)
-	iamOS.unlock()
-
-	if isMinIOUsersSys {
-		if err := iamOS.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
-			return err
-		}
-		if err := iamOS.loadGroups(ctx, iamGroupsMap); err != nil {
-			return err
-		}
-	}
-
-	// load polices mapped to users
-	if err := iamOS.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
-		return err
-	}
-
-	// load policies mapped to groups
-	if err := iamOS.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
-		return err
-	}
-
-	if err := iamOS.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
-		return err
-	}
-
-	// load STS temp users
-	if err := iamOS.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
-		return err
-	}
-
-	// load STS policy mappings
-	if err := iamOS.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
-		return err
-	}
-
-	iamOS.lock()
-	defer iamOS.unlock()
-
-	// Merge the new reloaded entries into global map.
-	// See issue https://github.com/minio/minio/issues/9651
-	// where the present list of entries on disk are not yet
-	// latest, there is a small window where this can make
-	// valid users invalid.
-	for k, v := range iamUsersMap {
-		sys.iamUsersMap[k] = v
-	}
-
-	for k, v := range iamUserPolicyMap {
-		sys.iamUserPolicyMap[k] = v
-	}
-
-	// purge any expired entries which became expired now.
-	var expiredEntries []string
-	for k, v := range sys.iamUsersMap {
-		if v.IsExpired() {
-			delete(sys.iamUsersMap, k)
-			delete(sys.iamUserPolicyMap, k)
-			expiredEntries = append(expiredEntries, k)
-			// Deleting on the disk is taken care of in the next cycle
-		}
-	}
-
-	for _, v := range sys.iamUsersMap {
-		if v.IsServiceAccount() {
-			for _, accessKey := range expiredEntries {
-				if v.ParentUser == accessKey {
-					_ = iamOS.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
-					delete(sys.iamUsersMap, v.AccessKey)
-				}
-			}
-		}
-	}
-
-	for k, v := range iamGroupPolicyMap {
-		sys.iamGroupPolicyMap[k] = v
-	}
-
-	for k, v := range iamGroupsMap {
-		sys.iamGroupsMap[k] = v
-	}
-
-	sys.buildUserGroupMemberships()
-	sys.storeFallback = false
-
-	return nil
 }
 
 func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {

cmd/iam.go

@@ -442,6 +442,122 @@ func (sys *IAMSys) Initialized() bool {
 	return sys.store != nil
 }
 
+// Load - loads all credentials
+func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
+	iamUsersMap := make(map[string]auth.Credentials)
+	iamGroupsMap := make(map[string]GroupInfo)
+	iamUserPolicyMap := make(map[string]MappedPolicy)
+	iamGroupPolicyMap := make(map[string]MappedPolicy)
+	iamPolicyDocsMap := make(map[string]iampolicy.Policy)
+
+	store.rlock()
+	isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
+	store.runlock()
+
+	if err := store.loadPolicyDocs(ctx, iamPolicyDocsMap); err != nil {
+		return err
+	}
+
+	// Sets default canned policies, if none are set.
+	setDefaultCannedPolicies(iamPolicyDocsMap)
+
+	if isMinIOUsersSys {
+		if err := store.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
+			return err
+		}
+		if err := store.loadGroups(ctx, iamGroupsMap); err != nil {
+			return err
+		}
+	}
+
+	// load polices mapped to users
+	if err := store.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
+		return err
+	}
+
+	// load policies mapped to groups
+	if err := store.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
+		return err
+	}
+
+	if err := store.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
+		return err
+	}
+
+	// load STS temp users
+	if err := store.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
+		return err
+	}
+
+	// load STS policy mappings
+	if err := store.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
+		return err
+	}
+
+	store.lock()
+	defer store.unlock()
+
+	for k, v := range iamPolicyDocsMap {
+		sys.iamPolicyDocsMap[k] = v
+	}
+
+	// Merge the new reloaded entries into global map.
+	// See issue https://github.com/minio/minio/issues/9651
+	// where the present list of entries on disk are not yet
+	// latest, there is a small window where this can make
+	// valid users invalid.
+	for k, v := range iamUsersMap {
+		sys.iamUsersMap[k] = v
+	}
+
+	for k, v := range iamUserPolicyMap {
+		sys.iamUserPolicyMap[k] = v
+	}
+
+	// purge any expired entries which became expired now.
+	var expiredEntries []string
+	for k, v := range sys.iamUsersMap {
+		if v.IsExpired() {
+			delete(sys.iamUsersMap, k)
+			delete(sys.iamUserPolicyMap, k)
+			expiredEntries = append(expiredEntries, k)
+			// Deleting on the disk is taken care of in the next cycle
+		}
+	}
+
+	for _, v := range sys.iamUsersMap {
+		if v.IsServiceAccount() {
+			for _, accessKey := range expiredEntries {
+				if v.ParentUser == accessKey {
+					_ = store.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
+					delete(sys.iamUsersMap, v.AccessKey)
+				}
+			}
+		}
+	}
+
+	// purge any expired entries which became expired now.
+	for k, v := range sys.iamUsersMap {
+		if v.IsExpired() {
+			delete(sys.iamUsersMap, k)
+			delete(sys.iamUserPolicyMap, k)
+			// Deleting on the etcd is taken care of in the next cycle
+		}
+	}
+
+	for k, v := range iamGroupPolicyMap {
+		sys.iamGroupPolicyMap[k] = v
+	}
+
+	for k, v := range iamGroupsMap {
+		sys.iamGroupsMap[k] = v
+	}
+
+	sys.buildUserGroupMemberships()
+	sys.storeFallback = false
+	return nil
+}
+
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	// Initialize IAM store
@@ -664,6 +780,10 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 		return errServerNotInitialized
 	}
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	// First we remove the user from their groups.
 	userInfo, getErr := sys.GetUserInfo(accessKey)
 	if getErr != nil {
@@ -681,18 +801,21 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	sys.store.lock()
 	defer sys.store.unlock()
 
-	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
-	}
-
-	// Delete any service accounts if any first.
 	for _, u := range sys.iamUsersMap {
+		// Delete any service accounts if any first.
 		if u.IsServiceAccount() {
 			if u.ParentUser == accessKey {
 				_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, srvAccUser)
 				delete(sys.iamUsersMap, u.AccessKey)
 			}
 		}
+		// Delete any associated STS users.
+		if u.IsTemp() {
+			if u.ParentUser == accessKey {
+				_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, stsUser)
+				delete(sys.iamUsersMap, u.AccessKey)
+			}
+		}
 	}
 
 	// It is ok to ignore deletion error on the mapped policy
@@ -869,13 +992,19 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 	}
 
 	sys.store.rlock()
-	defer sys.store.runlock()
+	fallback := sys.storeFallback
+	sys.store.runlock()
+	if fallback {
+		sys.loadUserFromStore(name)
+	}
 
 	if sys.usersSysType != MinIOUsersSysType {
+		sys.store.rlock()
 		// If the user has a mapped policy or is a member of a group, we
 		// return that info. Otherwise we return error.
 		mappedPolicy, ok1 := sys.iamUserPolicyMap[name]
 		memberships, ok2 := sys.iamUserGroupMemberships[name]
+		sys.store.runlock()
 		if !ok1 && !ok2 {
 			return u, errNoSuchUser
 		}
@@ -885,6 +1014,9 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 		}, nil
 	}
 
+	sys.store.rlock()
+	defer sys.store.runlock()
+
 	cred, found := sys.iamUsersMap[name]
 	if !found {
 		return u, errNoSuchUser
@@ -894,7 +1026,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 		return u, errIAMActionNotAllowed
 	}
 
-	u = madmin.UserInfo{
+	return madmin.UserInfo{
 		PolicyName: sys.iamUserPolicyMap[name].Policies,
 		Status: func() madmin.AccountStatus {
 			if cred.IsValid() {
@@ -903,8 +1035,8 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 			return madmin.AccountDisabled
 		}(),
 		MemberOf: sys.iamUserGroupMemberships[name].ToSlice(),
-	}
+	}, nil
-	return u, nil
+
 }
 
 // SetUserStatus - sets current user status, supports disabled or enabled.
@@ -913,6 +1045,10 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 		return errServerNotInitialized
 	}
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	if status != madmin.AccountEnabled && status != madmin.AccountDisabled {
 		return errInvalidArgument
 	}
@@ -920,10 +1056,6 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 	sys.store.lock()
 	defer sys.store.unlock()
 
-	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
-	}
-
 	cred, ok := sys.iamUsersMap[accessKey]
 	if !ok {
 		return errNoSuchUser
@@ -1091,30 +1223,31 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
 	return nil
 }
 
-// SetUser - set user credentials and policy.
+// CreateUser - create new user credentials and policy, if user already exists
-func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
+// they shall be rewritten with new inputs.
+func (sys *IAMSys) CreateUser(accessKey string, uinfo madmin.UserInfo) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
+	sys.store.lock()
+	defer sys.store.unlock()
+
+	cr, ok := sys.iamUsersMap[accessKey]
+	if cr.IsTemp() && ok {
+		return errIAMActionNotAllowed
+	}
+
 	u := newUserIdentity(auth.Credentials{
 		AccessKey: accessKey,
 		SecretKey: uinfo.SecretKey,
 		Status:    string(uinfo.Status),
 	})
 
-	sys.store.lock()
-	defer sys.store.unlock()
-
-	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
-	}
-
-	cr, ok := sys.iamUsersMap[accessKey]
-	if cr.IsTemp() && ok {
-		return errIAMActionNotAllowed
-	}
-
 	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regularUser, u); err != nil {
 		return err
 	}
@@ -1156,55 +1289,55 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 	return nil
 }
 
+func (sys *IAMSys) loadUserFromStore(accessKey string) {
+	sys.store.lock()
+	// If user is already found proceed.
+	if _, found := sys.iamUsersMap[accessKey]; !found {
+		sys.store.loadUser(context.Background(), accessKey, regularUser, sys.iamUsersMap)
+		if _, found = sys.iamUsersMap[accessKey]; found {
+			// found user, load its mapped policies
+			sys.store.loadMappedPolicy(context.Background(), accessKey, regularUser, false, sys.iamUserPolicyMap)
+		} else {
+			sys.store.loadUser(context.Background(), accessKey, srvAccUser, sys.iamUsersMap)
+			if svc, found := sys.iamUsersMap[accessKey]; found {
+				// Found service account, load its parent user and its mapped policies.
+				if sys.usersSysType == MinIOUsersSysType {
+					sys.store.loadUser(context.Background(), svc.ParentUser, regularUser, sys.iamUsersMap)
+				}
+				sys.store.loadMappedPolicy(context.Background(), svc.ParentUser, regularUser, false, sys.iamUserPolicyMap)
+			} else {
+				// None found fall back to STS users.
+				sys.store.loadUser(context.Background(), accessKey, stsUser, sys.iamUsersMap)
+				if _, found = sys.iamUsersMap[accessKey]; found {
+					// STS user found, load its mapped policy.
+					sys.store.loadMappedPolicy(context.Background(), accessKey, stsUser, false, sys.iamUserPolicyMap)
+				}
+			}
+		}
+	}
+
+	// Load associated policies if any.
+	for _, policy := range sys.iamUserPolicyMap[accessKey].toSlice() {
+		if _, found := sys.iamPolicyDocsMap[policy]; !found {
+			sys.store.loadPolicyDoc(context.Background(), policy, sys.iamPolicyDocsMap)
+		}
+	}
+
+	sys.buildUserGroupMemberships()
+	sys.store.unlock()
+}
+
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 	if !sys.Initialized() {
 		return cred, false
 	}
 
-	reloadUser := func() {
-		sys.store.lock()
-		// If user is already found proceed.
-		if _, found := sys.iamUsersMap[accessKey]; !found {
-			sys.store.loadUser(context.Background(), accessKey, regularUser, sys.iamUsersMap)
-			if _, found = sys.iamUsersMap[accessKey]; found {
-				// found user, load its mapped policies
-				sys.store.loadMappedPolicy(context.Background(), accessKey, regularUser, false, sys.iamUserPolicyMap)
-			} else {
-				sys.store.loadUser(context.Background(), accessKey, srvAccUser, sys.iamUsersMap)
-				if svc, found := sys.iamUsersMap[accessKey]; found {
-					// Found service account, load its parent user and its mapped policies.
-					if sys.usersSysType == MinIOUsersSysType {
-						sys.store.loadUser(context.Background(), svc.ParentUser, regularUser, sys.iamUsersMap)
-					}
-					sys.store.loadMappedPolicy(context.Background(), svc.ParentUser, regularUser, false, sys.iamUserPolicyMap)
-				} else {
-					// None found fall back to STS users.
-					sys.store.loadUser(context.Background(), accessKey, stsUser, sys.iamUsersMap)
-					if _, found = sys.iamUsersMap[accessKey]; found {
-						// STS user found, load its mapped policy.
-						sys.store.loadMappedPolicy(context.Background(), accessKey, stsUser, false, sys.iamUserPolicyMap)
-					}
-				}
-			}
-		}
-
-		// Load associated policies if any.
-		for _, policy := range sys.iamUserPolicyMap[accessKey].toSlice() {
-			if _, found := sys.iamPolicyDocsMap[policy]; !found {
-				sys.store.loadPolicyDoc(context.Background(), policy, sys.iamPolicyDocsMap)
-			}
-		}
-
-		sys.buildUserGroupMemberships()
-		sys.store.unlock()
-	}
-
 	sys.store.rlock()
 	fallback := sys.storeFallback
 	sys.store.runlock()
 	if fallback {
-		reloadUser()
+		sys.loadUserFromStore(accessKey)
 	}
 
 	sys.store.rlock()
@@ -1217,7 +1350,7 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 		// the IAM store and see if credential
 		// exists now. If it doesn't proceed to
 		// fail.
-		reloadUser()
+		sys.loadUserFromStore(accessKey)
 
 		sys.store.rlock()
 		cred, ok = sys.iamUsersMap[accessKey]
@@ -1305,14 +1438,14 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 		return errServerNotInitialized
 	}
 
-	if group == "" {
-		return errInvalidArgument
-	}
-
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
 	}
 
+	if group == "" {
+		return errInvalidArgument
+	}
+
 	sys.store.lock()
 	defer sys.store.unlock()
 

go.mod

@@ -81,6 +81,7 @@ require (
 	github.com/willf/bloom v2.0.3+incompatible
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b
+	go.uber.org/zap v1.13.0
 	golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392
 	golang.org/x/net v0.0.0-20201216054612-986b41b23924
 	golang.org/x/sys v0.0.0-20201119102817-f84b799fce68