fix: IAM import for LDAP should replace mappings (#19607)

Existing IAM import logic for LDAP creates new mappings when the
normalized form of the mapping key differs from the existing mapping key
in storage. This change effectively replaces the existing mapping key by
first deleting it and then recreating with the normalized form of the
mapping key.

For e.g. if an older deployment had a policy mapped to a user DN -

`UID=alice1,OU=people,OU=hwengg,DC=min,DC=io`

instead of adding a mapping for the normalized form -

`uid=alice1,ou=people,ou=hwengg,dc=min,dc=io`

we should replace the existing mapping.

This ensures that duplicates mappings won't remain after the import.

Some additional cleanup cases are also covered. If there are multiple
mappings for the name normalized key such as:

`UID=alice1,OU=people,OU=hwengg,DC=min,DC=io`
`uid=alice1,ou=people,ou=hwengg,DC=min,DC=io`
`uid=alice1,ou=people,ou=hwengg,dc=min,dc=io`

we check if the list of policies mapped to all these keys are exactly
the same, and if so remove all of them and create a single mapping with
the normalized key. However, if the policies mapped to such keys differ,
the import operation returns an error as the server cannot automatically
pick the "right" list of policies to map.

.github/workflows/iam-integrations.yaml

@@ -3,8 +3,8 @@ name: IAM integration
 on:
   pull_request:
     branches:
-    - master
-    - next
+      - master
+      - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -112,6 +112,12 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           go run docs/iam/access-manager-plugin.go &
           make test-iam
+      - name: Test MinIO Old Version data to IAM import current version
+        if: matrix.ldap == 'ldaphost:389'
+        env:
+          _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
+        run: |
+          make test-iam-ldap-upgrade-import
       - name: Test LDAP for automatic site replication
         if: matrix.ldap == 'localhost:389'
         run: |

Makefile

@@ -85,6 +85,10 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
 	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
+test-iam-ldap-upgrade-import: build ## verify IAM (external LDAP IDP)
+	@echo "Running upgrade tests for IAM (LDAP backend)"
+	@env bash $(PWD)/buildscripts/minio-iam-ldap-upgrade-import-test.sh
+
 test-sio-error:
 	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
 

buildscripts/minio-iam-ldap-upgrade-import-test.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+
+# This script is used to test the migration of IAM content from old minio
+# instance to new minio instance.
+#
+# To run it locally, start the LDAP server in github.com/minio/minio-iam-testing
+# repo (e.g. make podman-run), and then run this script.
+#
+# This script assumes that LDAP server is at:
+#
+#   `localhost:1389`
+#
+# if this is not the case, set the environment variable
+# `_MINIO_LDAP_TEST_SERVER`.
+
+OLD_VERSION=RELEASE.2024-03-26T22-10-45Z
+OLD_BINARY_LINK=https://dl.min.io/server/minio/release/linux-amd64/archive/minio.${OLD_VERSION}
+
+__init__() {
+	if which curl &>/dev/null; then
+		echo "curl is already installed"
+	else
+		echo "Installing curl:"
+		sudo apt install curl -y
+	fi
+
+	export GOPATH=/tmp/gopath
+	export PATH="${PATH}":"${GOPATH}"/bin
+
+	if which mc &>/dev/null; then
+		echo "mc is already installed"
+	else
+		echo "Installing mc:"
+		go install github.com/minio/mc@latest
+	fi
+
+	if [ ! -x ./minio.${OLD_VERSION} ]; then
+		echo "Downloading minio.${OLD_VERSION} binary"
+		curl -o minio.${OLD_VERSION} ${OLD_BINARY_LINK}
+		chmod +x minio.${OLD_VERSION}
+	fi
+
+	if [ -z "$_MINIO_LDAP_TEST_SERVER" ]; then
+		export _MINIO_LDAP_TEST_SERVER=localhost:1389
+		echo "Using default LDAP endpoint: $_MINIO_LDAP_TEST_SERVER"
+	fi
+
+	rm -rf /tmp/data
+}
+
+create_iam_content_in_old_minio() {
+	echo "Creating IAM content in old minio instance."
+
+	MINIO_CI_CD=1 ./minio.${OLD_VERSION} server /tmp/data/{1...4} &
+	sleep 5
+
+	set -x
+	mc alias set old-minio http://localhost:9000 minioadmin minioadmin
+	mc idp ldap add old-minio \
+		server_addr=localhost:1389 \
+		server_insecure=on \
+		lookup_bind_dn=cn=admin,dc=min,dc=io \
+		lookup_bind_password=admin \
+		user_dn_search_base_dn=dc=min,dc=io \
+		user_dn_search_filter="(uid=%s)" \
+		group_search_base_dn=ou=swengg,dc=min,dc=io \
+		group_search_filter="(&(objectclass=groupOfNames)(member=%d))"
+	mc admin service restart old-minio
+
+	mc idp ldap policy attach old-minio readwrite --user=UID=dillon,ou=people,ou=swengg,dc=min,dc=io
+	mc idp ldap policy attach old-minio readwrite --group=CN=project.c,ou=groups,ou=swengg,dc=min,dc=io
+
+	mc idp ldap policy entities old-minio
+
+	mc admin cluster iam export old-minio
+	set +x
+
+	mc admin service stop old-minio
+}
+
+import_iam_content_in_new_minio() {
+	echo "Importing IAM content in new minio instance."
+	# Assume current minio binary exists.
+	MINIO_CI_CD=1 ./minio server /tmp/data/{1...4} &
+	sleep 5
+
+	set -x
+	mc alias set new-minio http://localhost:9000 minioadmin minioadmin
+	echo "BEFORE IMPORT mappings:"
+	mc idp ldap policy entities new-minio
+	mc admin cluster iam import new-minio ./old-minio-iam-info.zip
+	echo "AFTER IMPORT mappings:"
+	mc idp ldap policy entities new-minio
+	set +x
+
+	# mc admin service stop new-minio
+}
+
+verify_iam_content_in_new_minio() {
+	output=$(mc idp ldap policy entities new-minio --json)
+
+	groups=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .groups[]')
+	if [ "$groups" != "cn=project.c,ou=groups,ou=swengg,dc=min,dc=io" ]; then
+		echo "Failed to verify groups: $groups"
+		exit 1
+	fi
+
+	users=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .users[]')
+	if [ "$users" != "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" ]; then
+		echo "Failed to verify users: $users"
+		exit 1
+	fi
+
+	mc admin service stop new-minio
+}
+
+main() {
+	create_iam_content_in_old_minio
+
+	import_iam_content_in_new_minio
+
+	verify_iam_content_in_new_minio
+}
+
+(__init__ "$@" && main "$@")

cmd/iam.go

@@ -1559,6 +1559,37 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 	return nil
 }
 
+func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup bool) set.StringSet {
+	entityKeysInStorage := set.NewStringSet()
+	if iamOS, ok := sys.store.IAMStorageAPI.(*IAMObjectStore); ok {
+		// Load existing mapping keys from the cached listing for
+		// `IAMObjectStore`.
+		iamFilesListing := iamOS.cachedIAMListing.Load().(map[string][]string)
+		listKey := policyDBSTSUsersListKey
+		if isGroup {
+			listKey = policyDBGroupsListKey
+		}
+		for _, item := range iamFilesListing[listKey] {
+			stsUserName := strings.TrimSuffix(item, ".json")
+			entityKeysInStorage.Add(stsUserName)
+		}
+	} else {
+		// For non-iam object store, we copy the mapping keys from the cache.
+		cache := sys.store.rlock()
+		defer sys.store.runlock()
+		cachedPolicyMap := cache.iamSTSPolicyMap
+		if isGroup {
+			cachedPolicyMap = cache.iamGroupPolicyMap
+		}
+		cachedPolicyMap.Range(func(k string, v MappedPolicy) bool {
+			entityKeysInStorage.Add(k)
+			return true
+		})
+	}
+
+	return entityKeysInStorage
+}
+
 // NormalizeLDAPMappingImport - validates the LDAP policy mappings. Keys in the
 // given map may not correspond to LDAP DNs - these keys are ignored.
 //
@@ -1615,6 +1646,8 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 		return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
 	}
 
+	entityKeysInStorage := sys.getStoredLDAPPolicyMappingKeys(ctx, isGroup)
+
 	for normKey, origKeys := range normalizedDNKeysMap {
 		if len(origKeys) > 1 {
 			// If there are multiple DN keys that normalize to the same value,
@@ -1639,6 +1672,12 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 			// ones from the map.
 			for i := 1; i < len(origKeys); i++ {
 				delete(policyMap, origKeys[i])
+
+				// Remove the mapping from storage by setting the policy to "".
+				if entityKeysInStorage.Contains(origKeys[i]) {
+					// Ignore any deletion error.
+					_, _ = sys.PolicyDBSet(ctx, origKeys[i], "", stsUser, isGroup)
+				}
 			}
 		}
 
@@ -1648,6 +1687,11 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 		mappingValue := policyMap[origKeys[0]]
 		delete(policyMap, origKeys[0])
 		policyMap[normKey] = mappingValue
+		// Remove the mapping from storage by setting the policy to "".
+		if entityKeysInStorage.Contains(origKeys[0]) {
+			// Ignore any deletion error.
+			_, _ = sys.PolicyDBSet(ctx, origKeys[0], "", stsUser, isGroup)
+		}
 	}
 	return nil
 }