Skip non existent ldap entities while import (#20352)

Dont hard error for nonexisting LDAP entries instead of logging them
report them via `mc`

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>

cmd/admin-handlers-users.go

@@ -26,8 +26,10 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"slices"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 	"unicode/utf8"
 
@@ -2046,6 +2048,16 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 
 // ImportIAM - imports all IAM info into MinIO
 func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
+	a.importIAM(w, r, "")
+}
+
+// ImportIAMV2 - imports all IAM info into MinIO
+func (a adminAPIHandlers) ImportIAMV2(w http.ResponseWriter, r *http.Request) {
+	a.importIAM(w, r, "v2")
+}
+
+// ImportIAM - imports all IAM info into MinIO
+func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiVer string) {
 	ctx := r.Context()
 
 	// Get current object layer instance.
@@ -2070,6 +2082,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
+
+	var skipped, removed, added madmin.IAMEntities
+	var failed madmin.IAMErrEntities
+
 	// import policies first
 	{
 
@@ -2095,8 +2111,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			for policyName, policy := range allPolicies {
 				if policy.IsEmpty() {
 					err = globalIAMSys.DeletePolicy(ctx, policyName, true)
+					removed.Policies = append(removed.Policies, policyName)
 				} else {
 					_, err = globalIAMSys.SetPolicy(ctx, policyName, policy)
+					added.Policies = append(added.Policies, policyName)
 				}
 				if err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, allPoliciesFile, policyName), r.URL)
@@ -2175,8 +2193,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					return
 				}
 				if _, err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
-					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, toAdminAPIErrCode(ctx, err), err, allUsersFile, accessKey), r.URL)
-					return
+					failed.Users = append(failed.Users, madmin.IAMErrEntity{Name: accessKey, Error: err})
+				} else {
+					added.Users = append(added.Users, accessKey)
 				}
 
 			}
@@ -2214,8 +2233,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					}
 				}
 				if _, gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members); gerr != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, gerr, allGroupsFile, group), r.URL)
-					return
+					failed.Groups = append(failed.Groups, madmin.IAMErrEntity{Name: group, Error: err})
+				} else {
+					added.Groups = append(added.Groups, group)
 				}
 			}
 		}
@@ -2244,7 +2264,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 
 			// Validations for LDAP enabled deployments.
 			if globalIAMSys.LDAPConfig.Enabled() {
-				err := globalIAMSys.NormalizeLDAPAccessKeypairs(ctx, serviceAcctReqs)
+				skippedAccessKeys, err := globalIAMSys.NormalizeLDAPAccessKeypairs(ctx, serviceAcctReqs)
+				skipped.ServiceAccounts = append(skipped.ServiceAccounts, skippedAccessKeys...)
 				if err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, ""), r.URL)
 					return
@@ -2252,6 +2273,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 
 			for user, svcAcctReq := range serviceAcctReqs {
+				if slices.Contains(skipped.ServiceAccounts, user) {
+					continue
+				}
 				var sp *policy.Policy
 				var err error
 				if len(svcAcctReq.SessionPolicy) > 0 {
@@ -2309,10 +2333,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				}
 
 				if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
-					return
+					failed.ServiceAccounts = append(failed.ServiceAccounts, madmin.IAMErrEntity{Name: user, Error: err})
+				} else {
+					added.ServiceAccounts = append(added.ServiceAccounts, user)
 				}
-
 			}
 		}
 	}
@@ -2349,8 +2373,15 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					return
 				}
 				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, regUser, false); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL)
-					return
+					failed.UserPolicies = append(
+						failed.UserPolicies,
+						madmin.IAMErrPolicyEntity{
+							Name:     u,
+							Policies: strings.Split(pm.Policies, ","),
+							Error:    err,
+						})
+				} else {
+					added.UserPolicies = append(added.UserPolicies, map[string][]string{u: strings.Split(pm.Policies, ",")})
 				}
 			}
 		}
@@ -2380,7 +2411,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			// Validations for LDAP enabled deployments.
 			if globalIAMSys.LDAPConfig.Enabled() {
 				isGroup := true
-				err := globalIAMSys.NormalizeLDAPMappingImport(ctx, isGroup, grpPolicyMap)
+				skippedDN, err := globalIAMSys.NormalizeLDAPMappingImport(ctx, isGroup, grpPolicyMap)
+				skipped.Groups = append(skipped.Groups, skippedDN...)
 				if err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, ""), r.URL)
 					return
@@ -2388,9 +2420,19 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 
 			for g, pm := range grpPolicyMap {
+				if slices.Contains(skipped.Groups, g) {
+					continue
+				}
 				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL)
-					return
+					failed.GroupPolicies = append(
+						failed.GroupPolicies,
+						madmin.IAMErrPolicyEntity{
+							Name:     g,
+							Policies: strings.Split(pm.Policies, ","),
+							Error:    err,
+						})
+				} else {
+					added.GroupPolicies = append(added.GroupPolicies, map[string][]string{g: strings.Split(pm.Policies, ",")})
 				}
 			}
 		}
@@ -2420,13 +2462,17 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			// Validations for LDAP enabled deployments.
 			if globalIAMSys.LDAPConfig.Enabled() {
 				isGroup := true
-				err := globalIAMSys.NormalizeLDAPMappingImport(ctx, !isGroup, userPolicyMap)
+				skippedDN, err := globalIAMSys.NormalizeLDAPMappingImport(ctx, !isGroup, userPolicyMap)
+				skipped.Users = append(skipped.Users, skippedDN...)
 				if err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, ""), r.URL)
 					return
 				}
 			}
 			for u, pm := range userPolicyMap {
+				if slices.Contains(skipped.Users, u) {
+					continue
+				}
 				// disallow setting policy mapping if user is a temporary user
 				ok, _, err := globalIAMSys.IsTempUser(u)
 				if err != nil && err != errNoSuchUser {
@@ -2439,12 +2485,36 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				}
 
 				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, stsUser, false); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
-					return
+					failed.STSPolicies = append(
+						failed.STSPolicies,
+						madmin.IAMErrPolicyEntity{
+							Name:     u,
+							Policies: strings.Split(pm.Policies, ","),
+							Error:    err,
+						})
+				} else {
+					added.STSPolicies = append(added.STSPolicies, map[string][]string{u: strings.Split(pm.Policies, ",")})
 				}
 			}
 		}
 	}
+
+	if apiVer == "v2" {
+		iamr := madmin.ImportIAMResult{
+			Skipped: skipped,
+			Removed: removed,
+			Added:   added,
+			Failed:  failed,
+		}
+
+		b, err := json.Marshal(iamr)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		writeSuccessResponseJSON(w, b)
+	}
 }
 
 func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) error {

cmd/admin-router.go

@@ -290,6 +290,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		// Import IAM info
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(adminMiddleware(adminAPI.ImportIAM, noGZFlag))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam-v2").HandlerFunc(adminMiddleware(adminAPI.ImportIAMV2, noGZFlag))
 
 		// IDentity Provider configuration APIs
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.AddIdentityProviderCfg))

cmd/iam.go

@@ -1568,16 +1568,16 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 // accounts) for LDAP users. This normalizes the parent user and the group names
 // whenever the parent user parses validly as a DN.
 func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap map[string]madmin.SRSvcAccCreate,
-) (err error) {
+) (skippedAccessKeys []string, err error) {
 	conn, err := sys.LDAPConfig.LDAP.Connect()
 	if err != nil {
-		return err
+		return skippedAccessKeys, err
 	}
 	defer conn.Close()
 
 	// Bind to the lookup user account
 	if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil {
-		return err
+		return skippedAccessKeys, err
 	}
 
 	var collectedErrors []error
@@ -1602,8 +1602,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 			continue
 		}
 		if validatedParent == nil || !isUnderBaseDN {
-			err := fmt.Errorf("DN parent was not found in the LDAP directory")
-			collectedErrors = append(collectedErrors, err)
+			skippedAccessKeys = append(skippedAccessKeys, ak)
 			continue
 		}
 
@@ -1621,8 +1620,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 				continue
 			}
 			if validatedGroup == nil {
-				err := fmt.Errorf("DN group was not found in the LDAP directory")
-				collectedErrors = append(collectedErrors, err)
+				// DN group was not found in the LDAP directory for access-key
 				continue
 			}
 
@@ -1643,7 +1641,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 
 	// if there are any errors, return a collected error.
 	if len(collectedErrors) > 0 {
-		return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
+		return skippedAccessKeys, fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
 	}
 
 	for k, v := range updatedKeysMap {
@@ -1651,7 +1649,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 		accessKeyMap[k] = v
 	}
 
-	return nil
+	return skippedAccessKeys, nil
 }
 
 func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup bool) set.StringSet {
@@ -1677,16 +1675,16 @@ func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup b
 // normalized form.
 func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 	policyMap map[string]MappedPolicy,
-) error {
+) ([]string, error) {
 	conn, err := sys.LDAPConfig.LDAP.Connect()
 	if err != nil {
-		return err
+		return []string{}, err
 	}
 	defer conn.Close()
 
 	// Bind to the lookup user account
 	if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil {
-		return err
+		return []string{}, err
 	}
 
 	// We map keys that correspond to LDAP DNs and validate that they exist in
@@ -1699,6 +1697,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 	// map of normalized DN keys to original keys.
 	normalizedDNKeysMap := make(map[string][]string)
 	var collectedErrors []error
+	var skipped []string
 	for k := range policyMap {
 		_, err := ldap.NormalizeDN(k)
 		if err != nil {
@@ -1711,8 +1710,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 			continue
 		}
 		if validatedDN == nil || !underBaseDN {
-			err := fmt.Errorf("DN was not found in the LDAP directory")
-			collectedErrors = append(collectedErrors, err)
+			skipped = append(skipped, k)
 			continue
 		}
 
@@ -1723,7 +1721,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 
 	// if there are any errors, return a collected error.
 	if len(collectedErrors) > 0 {
-		return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
+		return []string{}, fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
 	}
 
 	entityKeysInStorage := sys.getStoredLDAPPolicyMappingKeys(ctx, isGroup)
@@ -1744,7 +1742,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 			}
 
 			if policiesDiffer {
-				return fmt.Errorf("multiple DNs map to the same LDAP DN[%s]: %v; please remove DNs that are not needed",
+				return []string{}, fmt.Errorf("multiple DNs map to the same LDAP DN[%s]: %v; please remove DNs that are not needed",
 					normKey, origKeys)
 			}
 
@@ -1788,7 +1786,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 			}
 		}
 	}
-	return nil
+	return skipped, nil
 }
 
 // CheckKey validates the incoming accessKey

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/minio/highwayhash v1.0.3
 	github.com/minio/kms-go/kes v0.3.0
 	github.com/minio/kms-go/kms v0.4.0
-	github.com/minio/madmin-go/v3 v3.0.64
+	github.com/minio/madmin-go/v3 v3.0.66
 	github.com/minio/minio-go/v7 v7.0.76
 	github.com/minio/mux v1.9.0
 	github.com/minio/pkg/v3 v3.0.13

go.sum

@@ -426,8 +426,8 @@ github.com/minio/kms-go/kes v0.3.0 h1:SU8VGVM/Hk9w1OiSby3OatkcojooUqIdDHl6dtM6Nk
 github.com/minio/kms-go/kes v0.3.0/go.mod h1:w6DeVT878qEOU3nUrYVy1WOT5H1Ig9hbDIh698NYJKY=
 github.com/minio/kms-go/kms v0.4.0 h1:cLPZceEp+05xHotVBaeFJrgL7JcXM4lBy6PU0idkE7I=
 github.com/minio/kms-go/kms v0.4.0/go.mod h1:q12CehiIy2qgBnDKq6Q7wmPi2PHSyRVug5DKp0HAVeE=
-github.com/minio/madmin-go/v3 v3.0.64 h1:Btwgs3CrgSciVaCWv/3clOxuDdUzylo/oTQp0M8GkwE=
-github.com/minio/madmin-go/v3 v3.0.64/go.mod h1:IFAwr0XMrdsLovxAdCcuq/eoL4nRuMVQQv0iubJANQw=
+github.com/minio/madmin-go/v3 v3.0.66 h1:O4w7L3vTxhORqTeyegFdbuO4kKVbAUarJfcmsDXQMTs=
+github.com/minio/madmin-go/v3 v3.0.66/go.mod h1:IFAwr0XMrdsLovxAdCcuq/eoL4nRuMVQQv0iubJANQw=
 github.com/minio/mc v0.0.0-20240826104958-a55d9a8d17da h1:/JNoAwFPhCG0hUkc9k/wUlzMUO7tA9WLIoWgqQjYph4=
 github.com/minio/mc v0.0.0-20240826104958-a55d9a8d17da/go.mod h1:g/LyYpzVUVY+ZxfIDtzigg+L3NjAn88EBsLAkFvo+M0=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=