From 0b7aa6af879e088030d63e91a29dabf22fdd3a18 Mon Sep 17 00:00:00 2001 From: Shubhendu Date: Mon, 9 Sep 2024 22:29:28 +0530 Subject: [PATCH] Skip non existent ldap entities while import (#20352) Dont hard error for nonexisting LDAP entries instead of logging them report them via `mc` Signed-off-by: Shubhendu Ram Tripathi --- cmd/admin-handlers-users.go | 102 ++++++++++++++++++++++++++++++------ cmd/admin-router.go | 1 + cmd/iam.go | 32 ++++++----- go.mod | 2 +- go.sum | 4 +- 5 files changed, 105 insertions(+), 36 deletions(-) diff --git a/cmd/admin-handlers-users.go b/cmd/admin-handlers-users.go index 53b5c173d..188acc6c7 100644 --- a/cmd/admin-handlers-users.go +++ b/cmd/admin-handlers-users.go @@ -26,8 +26,10 @@ import ( "io" "net/http" "os" + "slices" "sort" "strconv" + "strings" "time" "unicode/utf8" @@ -2046,6 +2048,16 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) { // ImportIAM - imports all IAM info into MinIO func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { + a.importIAM(w, r, "") +} + +// ImportIAMV2 - imports all IAM info into MinIO +func (a adminAPIHandlers) ImportIAMV2(w http.ResponseWriter, r *http.Request) { + a.importIAM(w, r, "v2") +} + +// ImportIAM - imports all IAM info into MinIO +func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiVer string) { ctx := r.Context() // Get current object layer instance. @@ -2070,6 +2082,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) return } + + var skipped, removed, added madmin.IAMEntities + var failed madmin.IAMErrEntities + // import policies first { @@ -2095,8 +2111,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { for policyName, policy := range allPolicies { if policy.IsEmpty() { err = globalIAMSys.DeletePolicy(ctx, policyName, true) + removed.Policies = append(removed.Policies, policyName) } else { _, err = globalIAMSys.SetPolicy(ctx, policyName, policy) + added.Policies = append(added.Policies, policyName) } if err != nil { writeErrorResponseJSON(ctx, w, importError(ctx, err, allPoliciesFile, policyName), r.URL) @@ -2175,8 +2193,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { return } if _, err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil { - writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, toAdminAPIErrCode(ctx, err), err, allUsersFile, accessKey), r.URL) - return + failed.Users = append(failed.Users, madmin.IAMErrEntity{Name: accessKey, Error: err}) + } else { + added.Users = append(added.Users, accessKey) } } @@ -2214,8 +2233,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { } } if _, gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members); gerr != nil { - writeErrorResponseJSON(ctx, w, importError(ctx, gerr, allGroupsFile, group), r.URL) - return + failed.Groups = append(failed.Groups, madmin.IAMErrEntity{Name: group, Error: err}) + } else { + added.Groups = append(added.Groups, group) } } } @@ -2244,7 +2264,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { // Validations for LDAP enabled deployments. if globalIAMSys.LDAPConfig.Enabled() { - err := globalIAMSys.NormalizeLDAPAccessKeypairs(ctx, serviceAcctReqs) + skippedAccessKeys, err := globalIAMSys.NormalizeLDAPAccessKeypairs(ctx, serviceAcctReqs) + skipped.ServiceAccounts = append(skipped.ServiceAccounts, skippedAccessKeys...) if err != nil { writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, ""), r.URL) return @@ -2252,6 +2273,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { } for user, svcAcctReq := range serviceAcctReqs { + if slices.Contains(skipped.ServiceAccounts, user) { + continue + } var sp *policy.Policy var err error if len(svcAcctReq.SessionPolicy) > 0 { @@ -2309,10 +2333,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { } if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil { - writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL) - return + failed.ServiceAccounts = append(failed.ServiceAccounts, madmin.IAMErrEntity{Name: user, Error: err}) + } else { + added.ServiceAccounts = append(added.ServiceAccounts, user) } - } } } @@ -2349,8 +2373,15 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { return } if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, regUser, false); err != nil { - writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL) - return + failed.UserPolicies = append( + failed.UserPolicies, + madmin.IAMErrPolicyEntity{ + Name: u, + Policies: strings.Split(pm.Policies, ","), + Error: err, + }) + } else { + added.UserPolicies = append(added.UserPolicies, map[string][]string{u: strings.Split(pm.Policies, ",")}) } } } @@ -2380,7 +2411,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { // Validations for LDAP enabled deployments. if globalIAMSys.LDAPConfig.Enabled() { isGroup := true - err := globalIAMSys.NormalizeLDAPMappingImport(ctx, isGroup, grpPolicyMap) + skippedDN, err := globalIAMSys.NormalizeLDAPMappingImport(ctx, isGroup, grpPolicyMap) + skipped.Groups = append(skipped.Groups, skippedDN...) if err != nil { writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, ""), r.URL) return @@ -2388,9 +2420,19 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { } for g, pm := range grpPolicyMap { + if slices.Contains(skipped.Groups, g) { + continue + } if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil { - writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL) - return + failed.GroupPolicies = append( + failed.GroupPolicies, + madmin.IAMErrPolicyEntity{ + Name: g, + Policies: strings.Split(pm.Policies, ","), + Error: err, + }) + } else { + added.GroupPolicies = append(added.GroupPolicies, map[string][]string{g: strings.Split(pm.Policies, ",")}) } } } @@ -2420,13 +2462,17 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { // Validations for LDAP enabled deployments. if globalIAMSys.LDAPConfig.Enabled() { isGroup := true - err := globalIAMSys.NormalizeLDAPMappingImport(ctx, !isGroup, userPolicyMap) + skippedDN, err := globalIAMSys.NormalizeLDAPMappingImport(ctx, !isGroup, userPolicyMap) + skipped.Users = append(skipped.Users, skippedDN...) if err != nil { writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, ""), r.URL) return } } for u, pm := range userPolicyMap { + if slices.Contains(skipped.Users, u) { + continue + } // disallow setting policy mapping if user is a temporary user ok, _, err := globalIAMSys.IsTempUser(u) if err != nil && err != errNoSuchUser { @@ -2439,12 +2485,36 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { } if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, stsUser, false); err != nil { - writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL) - return + failed.STSPolicies = append( + failed.STSPolicies, + madmin.IAMErrPolicyEntity{ + Name: u, + Policies: strings.Split(pm.Policies, ","), + Error: err, + }) + } else { + added.STSPolicies = append(added.STSPolicies, map[string][]string{u: strings.Split(pm.Policies, ",")}) } } } } + + if apiVer == "v2" { + iamr := madmin.ImportIAMResult{ + Skipped: skipped, + Removed: removed, + Added: added, + Failed: failed, + } + + b, err := json.Marshal(iamr) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, b) + } } func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) error { diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 6bb138574..c2754b143 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -290,6 +290,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) { // Import IAM info adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(adminMiddleware(adminAPI.ImportIAM, noGZFlag)) + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam-v2").HandlerFunc(adminMiddleware(adminAPI.ImportIAMV2, noGZFlag)) // IDentity Provider configuration APIs adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.AddIdentityProviderCfg)) diff --git a/cmd/iam.go b/cmd/iam.go index bb4b9f561..9a6167ecd 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -1568,16 +1568,16 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) { // accounts) for LDAP users. This normalizes the parent user and the group names // whenever the parent user parses validly as a DN. func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap map[string]madmin.SRSvcAccCreate, -) (err error) { +) (skippedAccessKeys []string, err error) { conn, err := sys.LDAPConfig.LDAP.Connect() if err != nil { - return err + return skippedAccessKeys, err } defer conn.Close() // Bind to the lookup user account if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil { - return err + return skippedAccessKeys, err } var collectedErrors []error @@ -1602,8 +1602,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap continue } if validatedParent == nil || !isUnderBaseDN { - err := fmt.Errorf("DN parent was not found in the LDAP directory") - collectedErrors = append(collectedErrors, err) + skippedAccessKeys = append(skippedAccessKeys, ak) continue } @@ -1621,8 +1620,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap continue } if validatedGroup == nil { - err := fmt.Errorf("DN group was not found in the LDAP directory") - collectedErrors = append(collectedErrors, err) + // DN group was not found in the LDAP directory for access-key continue } @@ -1643,7 +1641,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap // if there are any errors, return a collected error. if len(collectedErrors) > 0 { - return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...)) + return skippedAccessKeys, fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...)) } for k, v := range updatedKeysMap { @@ -1651,7 +1649,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap accessKeyMap[k] = v } - return nil + return skippedAccessKeys, nil } func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup bool) set.StringSet { @@ -1677,16 +1675,16 @@ func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup b // normalized form. func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, policyMap map[string]MappedPolicy, -) error { +) ([]string, error) { conn, err := sys.LDAPConfig.LDAP.Connect() if err != nil { - return err + return []string{}, err } defer conn.Close() // Bind to the lookup user account if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil { - return err + return []string{}, err } // We map keys that correspond to LDAP DNs and validate that they exist in @@ -1699,6 +1697,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, // map of normalized DN keys to original keys. normalizedDNKeysMap := make(map[string][]string) var collectedErrors []error + var skipped []string for k := range policyMap { _, err := ldap.NormalizeDN(k) if err != nil { @@ -1711,8 +1710,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, continue } if validatedDN == nil || !underBaseDN { - err := fmt.Errorf("DN was not found in the LDAP directory") - collectedErrors = append(collectedErrors, err) + skipped = append(skipped, k) continue } @@ -1723,7 +1721,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, // if there are any errors, return a collected error. if len(collectedErrors) > 0 { - return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...)) + return []string{}, fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...)) } entityKeysInStorage := sys.getStoredLDAPPolicyMappingKeys(ctx, isGroup) @@ -1744,7 +1742,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, } if policiesDiffer { - return fmt.Errorf("multiple DNs map to the same LDAP DN[%s]: %v; please remove DNs that are not needed", + return []string{}, fmt.Errorf("multiple DNs map to the same LDAP DN[%s]: %v; please remove DNs that are not needed", normKey, origKeys) } @@ -1788,7 +1786,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, } } } - return nil + return skipped, nil } // CheckKey validates the incoming accessKey diff --git a/go.mod b/go.mod index 6d395704c..48403d871 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/minio/highwayhash v1.0.3 github.com/minio/kms-go/kes v0.3.0 github.com/minio/kms-go/kms v0.4.0 - github.com/minio/madmin-go/v3 v3.0.64 + github.com/minio/madmin-go/v3 v3.0.66 github.com/minio/minio-go/v7 v7.0.76 github.com/minio/mux v1.9.0 github.com/minio/pkg/v3 v3.0.13 diff --git a/go.sum b/go.sum index 9ac7cb6d0..f773bc6e8 100644 --- a/go.sum +++ b/go.sum @@ -426,8 +426,8 @@ github.com/minio/kms-go/kes v0.3.0 h1:SU8VGVM/Hk9w1OiSby3OatkcojooUqIdDHl6dtM6Nk github.com/minio/kms-go/kes v0.3.0/go.mod h1:w6DeVT878qEOU3nUrYVy1WOT5H1Ig9hbDIh698NYJKY= github.com/minio/kms-go/kms v0.4.0 h1:cLPZceEp+05xHotVBaeFJrgL7JcXM4lBy6PU0idkE7I= github.com/minio/kms-go/kms v0.4.0/go.mod h1:q12CehiIy2qgBnDKq6Q7wmPi2PHSyRVug5DKp0HAVeE= -github.com/minio/madmin-go/v3 v3.0.64 h1:Btwgs3CrgSciVaCWv/3clOxuDdUzylo/oTQp0M8GkwE= -github.com/minio/madmin-go/v3 v3.0.64/go.mod h1:IFAwr0XMrdsLovxAdCcuq/eoL4nRuMVQQv0iubJANQw= +github.com/minio/madmin-go/v3 v3.0.66 h1:O4w7L3vTxhORqTeyegFdbuO4kKVbAUarJfcmsDXQMTs= +github.com/minio/madmin-go/v3 v3.0.66/go.mod h1:IFAwr0XMrdsLovxAdCcuq/eoL4nRuMVQQv0iubJANQw= github.com/minio/mc v0.0.0-20240826104958-a55d9a8d17da h1:/JNoAwFPhCG0hUkc9k/wUlzMUO7tA9WLIoWgqQjYph4= github.com/minio/mc v0.0.0-20240826104958-a55d9a8d17da/go.mod h1:g/LyYpzVUVY+ZxfIDtzigg+L3NjAn88EBsLAkFvo+M0= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=