diff --git a/.github/workflows/iam-integrations.yaml b/.github/workflows/iam-integrations.yaml index 284ceec65..47a0c3a29 100644 --- a/.github/workflows/iam-integrations.yaml +++ b/.github/workflows/iam-integrations.yaml @@ -3,8 +3,8 @@ name: IAM integration on: pull_request: branches: - - master - - next + - master + - next # This ensures that previous jobs for the PR are canceled when the PR is # updated. @@ -112,6 +112,12 @@ jobs: sudo sysctl net.ipv6.conf.default.disable_ipv6=0 go run docs/iam/access-manager-plugin.go & make test-iam + - name: Test MinIO Old Version data to IAM import current version + if: matrix.ldap == 'ldaphost:389' + env: + _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }} + run: | + make test-iam-ldap-upgrade-import - name: Test LDAP for automatic site replication if: matrix.ldap == 'localhost:389' run: | diff --git a/Makefile b/Makefile index 1109e73f4..51d9dd710 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,10 @@ test-iam: build ## verify IAM (external IDP, etcd backends) @echo "Running tests for IAM (external IDP, etcd backends) with -race" @MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd +test-iam-ldap-upgrade-import: build ## verify IAM (external LDAP IDP) + @echo "Running upgrade tests for IAM (LDAP backend)" + @env bash $(PWD)/buildscripts/minio-iam-ldap-upgrade-import-test.sh + test-sio-error: @(env bash $(PWD)/docs/bucket/replication/sio-error.sh) diff --git a/buildscripts/minio-iam-ldap-upgrade-import-test.sh b/buildscripts/minio-iam-ldap-upgrade-import-test.sh new file mode 100755 index 000000000..a8f3a6cf3 --- /dev/null +++ b/buildscripts/minio-iam-ldap-upgrade-import-test.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +# This script is used to test the migration of IAM content from old minio +# instance to new minio instance. +# +# To run it locally, start the LDAP server in github.com/minio/minio-iam-testing +# repo (e.g. make podman-run), and then run this script. +# +# This script assumes that LDAP server is at: +# +# `localhost:1389` +# +# if this is not the case, set the environment variable +# `_MINIO_LDAP_TEST_SERVER`. + +OLD_VERSION=RELEASE.2024-03-26T22-10-45Z +OLD_BINARY_LINK=https://dl.min.io/server/minio/release/linux-amd64/archive/minio.${OLD_VERSION} + +__init__() { + if which curl &>/dev/null; then + echo "curl is already installed" + else + echo "Installing curl:" + sudo apt install curl -y + fi + + export GOPATH=/tmp/gopath + export PATH="${PATH}":"${GOPATH}"/bin + + if which mc &>/dev/null; then + echo "mc is already installed" + else + echo "Installing mc:" + go install github.com/minio/mc@latest + fi + + if [ ! -x ./minio.${OLD_VERSION} ]; then + echo "Downloading minio.${OLD_VERSION} binary" + curl -o minio.${OLD_VERSION} ${OLD_BINARY_LINK} + chmod +x minio.${OLD_VERSION} + fi + + if [ -z "$_MINIO_LDAP_TEST_SERVER" ]; then + export _MINIO_LDAP_TEST_SERVER=localhost:1389 + echo "Using default LDAP endpoint: $_MINIO_LDAP_TEST_SERVER" + fi + + rm -rf /tmp/data +} + +create_iam_content_in_old_minio() { + echo "Creating IAM content in old minio instance." + + MINIO_CI_CD=1 ./minio.${OLD_VERSION} server /tmp/data/{1...4} & + sleep 5 + + set -x + mc alias set old-minio http://localhost:9000 minioadmin minioadmin + mc idp ldap add old-minio \ + server_addr=localhost:1389 \ + server_insecure=on \ + lookup_bind_dn=cn=admin,dc=min,dc=io \ + lookup_bind_password=admin \ + user_dn_search_base_dn=dc=min,dc=io \ + user_dn_search_filter="(uid=%s)" \ + group_search_base_dn=ou=swengg,dc=min,dc=io \ + group_search_filter="(&(objectclass=groupOfNames)(member=%d))" + mc admin service restart old-minio + + mc idp ldap policy attach old-minio readwrite --user=UID=dillon,ou=people,ou=swengg,dc=min,dc=io + mc idp ldap policy attach old-minio readwrite --group=CN=project.c,ou=groups,ou=swengg,dc=min,dc=io + + mc idp ldap policy entities old-minio + + mc admin cluster iam export old-minio + set +x + + mc admin service stop old-minio +} + +import_iam_content_in_new_minio() { + echo "Importing IAM content in new minio instance." + # Assume current minio binary exists. + MINIO_CI_CD=1 ./minio server /tmp/data/{1...4} & + sleep 5 + + set -x + mc alias set new-minio http://localhost:9000 minioadmin minioadmin + echo "BEFORE IMPORT mappings:" + mc idp ldap policy entities new-minio + mc admin cluster iam import new-minio ./old-minio-iam-info.zip + echo "AFTER IMPORT mappings:" + mc idp ldap policy entities new-minio + set +x + + # mc admin service stop new-minio +} + +verify_iam_content_in_new_minio() { + output=$(mc idp ldap policy entities new-minio --json) + + groups=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .groups[]') + if [ "$groups" != "cn=project.c,ou=groups,ou=swengg,dc=min,dc=io" ]; then + echo "Failed to verify groups: $groups" + exit 1 + fi + + users=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .users[]') + if [ "$users" != "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" ]; then + echo "Failed to verify users: $users" + exit 1 + fi + + mc admin service stop new-minio +} + +main() { + create_iam_content_in_old_minio + + import_iam_content_in_new_minio + + verify_iam_content_in_new_minio +} + +(__init__ "$@" && main "$@") diff --git a/cmd/iam.go b/cmd/iam.go index 223595394..84419314c 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -1559,6 +1559,37 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap return nil } +func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup bool) set.StringSet { + entityKeysInStorage := set.NewStringSet() + if iamOS, ok := sys.store.IAMStorageAPI.(*IAMObjectStore); ok { + // Load existing mapping keys from the cached listing for + // `IAMObjectStore`. + iamFilesListing := iamOS.cachedIAMListing.Load().(map[string][]string) + listKey := policyDBSTSUsersListKey + if isGroup { + listKey = policyDBGroupsListKey + } + for _, item := range iamFilesListing[listKey] { + stsUserName := strings.TrimSuffix(item, ".json") + entityKeysInStorage.Add(stsUserName) + } + } else { + // For non-iam object store, we copy the mapping keys from the cache. + cache := sys.store.rlock() + defer sys.store.runlock() + cachedPolicyMap := cache.iamSTSPolicyMap + if isGroup { + cachedPolicyMap = cache.iamGroupPolicyMap + } + cachedPolicyMap.Range(func(k string, v MappedPolicy) bool { + entityKeysInStorage.Add(k) + return true + }) + } + + return entityKeysInStorage +} + // NormalizeLDAPMappingImport - validates the LDAP policy mappings. Keys in the // given map may not correspond to LDAP DNs - these keys are ignored. // @@ -1615,6 +1646,8 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...)) } + entityKeysInStorage := sys.getStoredLDAPPolicyMappingKeys(ctx, isGroup) + for normKey, origKeys := range normalizedDNKeysMap { if len(origKeys) > 1 { // If there are multiple DN keys that normalize to the same value, @@ -1639,6 +1672,12 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, // ones from the map. for i := 1; i < len(origKeys); i++ { delete(policyMap, origKeys[i]) + + // Remove the mapping from storage by setting the policy to "". + if entityKeysInStorage.Contains(origKeys[i]) { + // Ignore any deletion error. + _, _ = sys.PolicyDBSet(ctx, origKeys[i], "", stsUser, isGroup) + } } } @@ -1648,6 +1687,11 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, mappingValue := policyMap[origKeys[0]] delete(policyMap, origKeys[0]) policyMap[normKey] = mappingValue + // Remove the mapping from storage by setting the policy to "". + if entityKeysInStorage.Contains(origKeys[0]) { + // Ignore any deletion error. + _, _ = sys.PolicyDBSet(ctx, origKeys[0], "", stsUser, isGroup) + } } return nil }