MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Load STS accounts into IAM cache lazily (#17994)

Browse Source 
In situations with large number of STS credentials on disk, IAM load
time is high. To mitigate this, STS accounts will now be loaded into
memory only on demand - i.e. when the credential is used.

In each IAM cache (re)load we skip loading STS credentials and STS
policy mappings into memory. Since STS accounts only expire and cannot
be deleted, there is no risk of invalid credentials being reused,
because credential validity is checked when it is used.
This commit is contained in:
Aditya Manthramurthy
2023-09-13 12:43:46 -07:00
committed by GitHub
parent 18e23bafd9
commit ed2c2a285f
3 changed files with 184 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
cmd/iam-object-store.go
View File

@@ -29,9 +29,9 @@ import (
jsoniter "github.com/json-iterator/go"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/kms"
"github.com/minio/minio/internal/logger"
)
// IAMObjectStore implements IAMStorageAPI
@@ -343,6 +343,7 @@ var (
policyDBServiceAccountsListKey = "policydb/service-accounts/"
policyDBGroupsListKey = "policydb/groups/"
// List of directories from which to read iam data into memory.
allListKeys = []string{
usersListKey,
svcAccListKey,
@@ -354,29 +355,29 @@ var (
policyDBServiceAccountsListKey,
policyDBGroupsListKey,
}
// List of directories to skip: we do not read STS directories for better
// performance. STS credentials would be stored in memory when they are
// first used.
iamLoadSkipListKeySet = set.CreateStringSet(
stsListKey,
policyDBSTSUsersListKey,
)
)
func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[string][]string, error) {
res := make(map[string][]string)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator) {
if item.Err != nil {
return nil, item.Err
for _, listKey := range allListKeys {
if iamLoadSkipListKeySet.Contains(listKey) {
continue
}
found := false
for _, listKey := range allListKeys {
if strings.HasPrefix(item.Item, listKey) {
found = true
name := strings.TrimPrefix(item.Item, listKey)
res[listKey] = append(res[listKey], name)
break
for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator+listKey) {
if item.Err != nil {
return nil, item.Err
}
}
if !found && (item.Item != "format.json") {
logger.LogIf(ctx, fmt.Errorf("unknown type of IAM file listed: %v", item.Item))
res[listKey] = append(res[listKey], item.Item)
}
}
return res, nil
@@ -455,24 +456,6 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
}
}
bootstrapTraceMsg("loading STS users")
stsUsersList := listedConfigItems[stsListKey]
for _, item := range stsUsersList {
userName := path.Dir(item)
if err := iamOS.loadUser(ctx, userName, stsUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
return fmt.Errorf("unable to load the STS user `%s`: %w", userName, err)
}
}
bootstrapTraceMsg("loading STS policy mapping")
stsPolicyMappingsList := listedConfigItems[policyDBSTSUsersListKey]
for _, item := range stsPolicyMappingsList {
stsName := strings.TrimSuffix(item, ".json")
if err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, cache.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
return fmt.Errorf("unable to load the policy mapping for the STS user `%s`: %w", stsName, err)
}
}
cache.buildUserGroupMemberships()
return nil
}