diff --git a/cmd/globals.go b/cmd/globals.go index 49cad382e..55d0b0617 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -102,7 +102,7 @@ const ( GlobalStaleUploadsCleanupInterval = time.Hour * 6 // 6 hrs. // Refresh interval to update in-memory iam config cache. - globalRefreshIAMInterval = 5 * time.Minute + globalRefreshIAMInterval = 30 * time.Minute // Limit of location constraint XML for unauthenticated PUT bucket operations. maxLocationConstraintSize = 3 * humanize.MiByte diff --git a/cmd/iam-object-store.go b/cmd/iam-object-store.go index 736c523e2..d0c52ecba 100644 --- a/cmd/iam-object-store.go +++ b/cmd/iam-object-store.go @@ -19,6 +19,7 @@ package cmd import ( "context" + "fmt" "path" "strings" "sync" @@ -405,6 +406,136 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA return nil } +var ( + usersListKey = "/users/" + svcAccListKey = "/service-accounts/" + groupsListKey = "/groups/" + policiesListKey = "/policies/" + stsListKey = "/sts/" + policyDBUsersListKey = "/policydb/users/" + policyDBSTSUsersListKey = "/policydb/sts-users/" + policyDBServiceAccountsListKey = "/policydb/service-accounts/" + policyDBGroupsListKey = "/policydb/groups/" + + allListKeys = []string{ + usersListKey, + svcAccListKey, + groupsListKey, + policiesListKey, + stsListKey, + policyDBUsersListKey, + policyDBSTSUsersListKey, + policyDBServiceAccountsListKey, + policyDBGroupsListKey, + } +) + +func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[string][]string, error) { + res := make(map[string][]string) + + for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix) { + if item.Err != nil { + return nil, item.Err + } + + found := false + for _, listKey := range allListKeys { + if strings.HasPrefix(item.Item, listKey) { + found = true + name := strings.TrimPrefix(item.Item, listKey) + res[listKey] = append(res[listKey], name) + break + } + } + + if !found && !(item.Item == "config/config.json" || item.Item == "/format.json") { + logger.LogIf(ctx, fmt.Errorf("unknown type of IAM file listed: %v", item.Item)) + } + } + return res, nil +} + +// Assumes cache is locked by caller. +func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error { + listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx) + if err != nil { + return err + } + + // Loads things in the same order as `LoadIAMCache()` + + policiesList := listedConfigItems[policiesListKey] + for _, item := range policiesList { + policyName := path.Dir(item) + if err := iamOS.loadPolicyDoc(ctx, policyName, cache.iamPolicyDocsMap); err != nil && err != errNoSuchPolicy { + return err + } + } + setDefaultCannedPolicies(cache.iamPolicyDocsMap) + + if iamOS.usersSysType == MinIOUsersSysType { + + regUsersList := listedConfigItems[usersListKey] + for _, item := range regUsersList { + userName := path.Dir(item) + if err := iamOS.loadUser(ctx, userName, regUser, cache.iamUsersMap); err != nil && err != errNoSuchUser { + return err + } + } + + groupsList := listedConfigItems[groupsListKey] + for _, item := range groupsList { + group := path.Dir(item) + if err := iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && err != errNoSuchGroup { + return err + } + } + } + + userPolicyMappingsList := listedConfigItems[policyDBUsersListKey] + for _, item := range userPolicyMappingsList { + userName := strings.TrimSuffix(item, ".json") + if err := iamOS.loadMappedPolicy(ctx, userName, regUser, false, cache.iamUserPolicyMap); err != nil && err != errNoSuchPolicy { + return err + } + } + + groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey] + for _, item := range groupPolicyMappingsList { + groupName := strings.TrimSuffix(item, ".json") + if err := iamOS.loadMappedPolicy(ctx, groupName, regUser, true, cache.iamGroupPolicyMap); err != nil && err != errNoSuchPolicy { + return err + } + } + + svcAccList := listedConfigItems[svcAccListKey] + for _, item := range svcAccList { + userName := path.Dir(item) + if err := iamOS.loadUser(ctx, userName, svcUser, cache.iamUsersMap); err != nil && err != errNoSuchUser { + return err + } + } + + stsUsersList := listedConfigItems[stsListKey] + for _, item := range stsUsersList { + userName := path.Dir(item) + if err := iamOS.loadUser(ctx, userName, stsUser, cache.iamUsersMap); err != nil && err != errNoSuchUser { + return err + } + } + + stsPolicyMappingsList := listedConfigItems[policyDBSTSUsersListKey] + for _, item := range stsPolicyMappingsList { + stsName := strings.TrimSuffix(item, ".json") + if err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, cache.iamUserPolicyMap); err != nil && err != errNoSuchPolicy { + return err + } + } + + cache.buildUserGroupMemberships() + return nil +} + func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error { return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName)) } diff --git a/cmd/iam-store.go b/cmd/iam-store.go index 5c126ff32..2bf0c6fc6 100644 --- a/cmd/iam-store.go +++ b/cmd/iam-store.go @@ -432,48 +432,56 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error { cache := store.lock() defer store.unlock() - if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil { - return err - } - - // Sets default canned policies, if none are set. - setDefaultCannedPolicies(newCache.iamPolicyDocsMap) - - if store.getUsersSysType() == MinIOUsersSysType { - if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil { + if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok { + err := iamOS.loadAllFromObjStore(ctx, newCache) + if err != nil { return err } - if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil { + } else { + + if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil { return err } - } - // load polices mapped to users - if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil { - return err - } + // Sets default canned policies, if none are set. + setDefaultCannedPolicies(newCache.iamPolicyDocsMap) - // load policies mapped to groups - if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil { - return err - } + if store.getUsersSysType() == MinIOUsersSysType { + if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil { + return err + } + if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil { + return err + } + } - // load service accounts - if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil { - return err - } + // load polices mapped to users + if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil { + return err + } - // load STS temp users - if err := store.loadUsers(ctx, stsUser, newCache.iamUsersMap); err != nil { - return err - } + // load policies mapped to groups + if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil { + return err + } - // load STS policy mappings - if err := store.loadMappedPolicies(ctx, stsUser, false, newCache.iamUserPolicyMap); err != nil { - return err - } + // load service accounts + if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil { + return err + } - newCache.buildUserGroupMemberships() + // load STS temp users + if err := store.loadUsers(ctx, stsUser, newCache.iamUsersMap); err != nil { + return err + } + + // load STS policy mappings + if err := store.loadMappedPolicies(ctx, stsUser, false, newCache.iamUserPolicyMap); err != nil { + return err + } + + newCache.buildUserGroupMemberships() + } cache.iamGroupPolicyMap = newCache.iamGroupPolicyMap cache.iamGroupsMap = newCache.iamGroupsMap diff --git a/cmd/iam.go b/cmd/iam.go index 0a14c2059..caefb9fb1 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -333,6 +333,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc } sys.printIAMRoles() + + logger.Info("Finished loading IAM sub-system.") } // Prints IAM role ARNs. @@ -366,7 +368,7 @@ func (sys *IAMSys) watch(ctx context.Context) { for event := range ch { // we simply log errors err := sys.loadWatchedEvent(ctx, event) - logger.LogIf(ctx, err) + logger.LogIf(ctx, fmt.Errorf("Failure in loading watch event: %v", err)) } return } @@ -378,7 +380,7 @@ func (sys *IAMSys) watch(ctx context.Context) { select { case <-ticker.C: if err := sys.Load(ctx); err != nil { - logger.LogIf(ctx, err) + logger.LogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM: %v", err)) } case <-ctx.Done(): return