IAM import test with missing entities (#20368)

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>

.github/workflows/iam-integrations.yaml

@@ -125,3 +125,20 @@ jobs:
         if: matrix.openid == 'http://127.0.0.1:5556/dex'
         run: |
           make test-site-replication-oidc
+  iam-import-with-missing-entities:
+    name: Test IAM import in new cluster with missing entities
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+          check-latest: true
+      - name: Checkout minio-iam-testing
+        uses: actions/checkout@v4
+        with:
+          repository: minio/minio-iam-testing
+          path: minio-iam-testing
+      - name: Test import of IAM artifacts when in fresh cluster there are missing groups etc
+        run: |
+          make test-iam-import-with-missing-entities

Makefile

@@ -97,6 +97,10 @@ test-iam-ldap-upgrade-import: install-race ## verify IAM (external LDAP IDP)
 	@echo "Running upgrade tests for IAM (LDAP backend)"
 	@env bash $(PWD)/buildscripts/minio-iam-ldap-upgrade-import-test.sh
 
+test-iam-import-with-missing-entities: install-race ## test import of external iam config withg missing entities
+	@echo "Test IAM import configurations with missing entities"
+	@env bash $(PWD)/docs/distributed/iam-import-with-missing-entities.sh
+
 test-sio-error:
 	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
 

docs/distributed/iam-import-with-missing-entities.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+if [ -n "$TEST_DEBUG" ]; then
+	set -x
+fi
+
+pkill minio
+docker rm -f $(docker ps -aq)
+rm -rf /tmp/ldap{1..4}
+rm -rf /tmp/ldap1{1..4}
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+fi
+
+mc -v
+
+# Start LDAP server
+echo "Copying docs/distributed/samples/bootstrap-complete.ldif => minio-iam-testing/ldap/50-bootstrap.ldif"
+cp docs/distributed/samples/bootstrap-complete.ldif minio-iam-testing/ldap/50-bootstrap.ldif || exit 1
+cd ./minio-iam-testing
+make docker-images
+make docker-run
+cd -
+
+export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:22000"
+export MC_HOST_myminio1="http://minioadmin:minioadmin@localhost:24000"
+
+# Start MinIO instance
+export CI=true
+(minio server --address :22000 --console-address :10000 http://localhost:22000/tmp/ldap{1...4} 2>&1 >/dev/null) &
+sleep 30
+./mc ready myminio
+
+./mc idp ldap add myminio server_addr=localhost:1389 server_insecure=on lookup_bind_dn=cn=admin,dc=min,dc=io lookup_bind_password=admin user_dn_search_base_dn=dc=min,dc=io user_dn_search_filter="(uid=%s)" group_search_base_dn=ou=swengg,dc=min,dc=io group_search_filter="(&(objectclass=groupOfNames)(member=%d))"
+./mc admin service restart myminio --json
+./mc ready myminio
+./mc admin cluster iam import myminio docs/distributed/samples/myminio-iam-info.zip
+sleep 10
+
+# Verify the list of users and service accounts from the import
+./mc admin user list myminio
+USER_COUNT=$(./mc admin user list myminio | wc -l)
+if [ "${USER_COUNT}" -ne 2 ]; then
+	echo "BUG: Expected no of users: 2 Found: ${USER_COUNT}"
+	exit 1
+fi
+./mc admin user svcacct list myminio "uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io" --json
+SVCACCT_COUNT_1=$(./mc admin user svcacct list myminio "uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io" --json | jq '.accessKey' | wc -l)
+if [ "${SVCACCT_COUNT_1}" -ne 2 ]; then
+	echo "BUG: Expected svcacct count for 'uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io': 2. Found: ${SVCACCT_COUNT_1}"
+	exit 1
+fi
+./mc admin user svcacct list myminio "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" --json
+SVCACCT_COUNT_2=$(./mc admin user svcacct list myminio "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" --json | jq '.accessKey' | wc -l)
+if [ "${SVCACCT_COUNT_2}" -ne 2 ]; then
+	echo "BUG: Expected svcacct count for 'uid=dillon,ou=people,ou=swengg,dc=min,dc=io': 2. Found: ${SVCACCT_COUNT_2}"
+	exit 1
+fi
+
+# Kill MinIO and LDAP to start afresh with missing groups/DN
+pkill minio
+docker rm -f $(docker ps -aq)
+rm -rf /tmp/ldap{1..4}
+
+# Deploy the LDAP config witg missing groups/DN
+echo "Copying docs/distributed/samples/bootstrap-partial.ldif => minio-iam-testing/ldap/50-bootstrap.ldif"
+cp docs/distributed/samples/bootstrap-partial.ldif minio-iam-testing/ldap/50-bootstrap.ldif || exit 1
+cd ./minio-iam-testing
+make docker-images
+make docker-run
+cd -
+
+(minio server --address ":24000" --console-address :10000 http://localhost:24000/tmp/ldap1{1...4} 2>&1 >/dev/null) &
+sleep 30
+./mc ready myminio1
+
+./mc idp ldap add myminio1 server_addr=localhost:1389 server_insecure=on lookup_bind_dn=cn=admin,dc=min,dc=io lookup_bind_password=admin user_dn_search_base_dn=dc=min,dc=io user_dn_search_filter="(uid=%s)" group_search_base_dn=ou=hwengg,dc=min,dc=io group_search_filter="(&(objectclass=groupOfNames)(member=%d))"
+./mc admin service restart myminio1 --json
+./mc ready myminio1
+./mc admin cluster iam import myminio1 docs/distributed/samples/myminio-iam-info.zip
+sleep 10
+
+# Verify the list of users and service accounts from the import
+./mc admin user list myminio1
+USER_COUNT=$(./mc admin user list myminio1 | wc -l)
+if [ "${USER_COUNT}" -ne 1 ]; then
+	echo "BUG: Expected no of users: 1 Found: ${USER_COUNT}"
+	exit 1
+fi
+./mc admin user svcacct list myminio1 "uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io" --json
+SVCACCT_COUNT_1=$(./mc admin user svcacct list myminio1 "uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io" --json | jq '.accessKey' | wc -l)
+if [ "${SVCACCT_COUNT_1}" -ne 2 ]; then
+	echo "BUG: Expected svcacct count for 'uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io': 2. Found: ${SVCACCT_COUNT_1}"
+	exit 1
+fi
+./mc admin user svcacct list myminio1 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" --json
+SVCACCT_COUNT_2=$(./mc admin user svcacct list myminio1 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" --json | jq '.accessKey' | wc -l)
+if [ "${SVCACCT_COUNT_2}" -ne 0 ]; then
+	echo "BUG: Expected svcacct count for 'uid=dillon,ou=people,ou=swengg,dc=min,dc=io': 0. Found: ${SVCACCT_COUNT_2}"
+	exit 1
+fi
+
+# Finally kill running processes
+pkill minio
+docker rm -f $(docker ps -aq)

docs/distributed/samples/bootstrap-complete.ldif

@@ -0,0 +1,123 @@
+# Create hardware engg org unit
+dn: ou=hwengg,dc=min,dc=io
+objectClass: organizationalUnit
+ou: hwengg
+
+# Create people sub-org
+dn: ou=people,ou=hwengg,dc=min,dc=io
+objectClass: organizationalUnit
+ou: people
+
+# Create Alice, Bob and Cody in hwengg
+dn: uid=alice1,ou=people,ou=hwengg,dc=min,dc=io
+objectClass: inetOrgPerson
+cn: Alice Smith
+sn: Smith
+uid: alice1
+mail: alice@example.io
+userPassword: {SSHA}Yeh2/IV/q/HjG2yzN3YdE9CAF3EJFCLu
+
+dn: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io
+objectClass: inetOrgPerson
+cn: Robert Fisher
+sn: Fisher
+uid: bobfisher
+mail: bob@example.io
+userPassword: {SSHA}LktfbhK5oXSdDWCNzauJ9JA+Poxinl3y
+
+dn: uid=cody3,ou=people,ou=hwengg,dc=min,dc=io
+objectClass: inetOrgPerson
+cn: Cody Thomas
+sn: Thomas
+uid: cody3
+mail: cody@example.io
+userPassword: {SSHA}H8B0gaOd4bRklK3fXj9ltHvJXWQFXW5Q
+
+# Create groups ou for hwengg
+dn: ou=groups,ou=hwengg,dc=min,dc=io
+objectclass: organizationalUnit
+ou: groups
+description: groups branch
+
+# Create project groups
+
+dn: cn=projectx,ou=groups,ou=hwengg,dc=min,dc=io
+objectclass: groupofnames
+cn: projectx
+description: Project X group members
+member: uid=alice1,ou=people,ou=hwengg,dc=min,dc=io
+member: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io
+
+dn: cn=projecty,ou=groups,ou=hwengg,dc=min,dc=io
+objectclass: groupofnames
+cn: projecty
+description: Project Y group members
+member: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io
+member: uid=cody3,ou=people,ou=hwengg,dc=min,dc=io
+
+# Create software engg org unit
+dn: ou=swengg,dc=min,dc=io
+objectClass: organizationalUnit
+ou: swengg
+
+# Create people sub-org
+dn: ou=people,ou=swengg,dc=min,dc=io
+objectClass: organizationalUnit
+ou: people
+
+# Create Dillon, Elizabeth and Fahim in swengg
+dn: uid=dillon,ou=people,ou=swengg,dc=min,dc=io
+objectClass: inetOrgPerson
+cn: Dillon Harper
+sn: Harper
+uid: dillon
+mail: dillon@example.io
+userPassword: {SSHA}UH+LmoEhWWW6s9rjgdpqHPI0qCMouY8+
+
+dn: uid=liza,ou=people,ou=swengg,dc=min,dc=io
+objectClass: inetOrgPerson
+cn: Elizabeth Jones
+sn: Jones
+uid: liza
+mail: ejones@example.io
+userPassword: {SSHA}feVkKkafHtsu2Io7n0tQP4Cnh8/Oy1PK
+
+dn: uid=fahim,ou=people,ou=swengg,dc=min,dc=io
+objectClass: inetOrgPerson
+cn: Fahim Ahmed
+sn: Ahmed
+uid: fahim
+mail: fahmed@example.io
+userPassword: {SSHA}lRNH+PHooRaruiEb+CBEA21EZLMkAmcc
+
+# Add a user with special chars. The password = example here.
+dn: uid=Пользователь,OU=people,OU=swengg,DC=min,DC=io
+objectClass: inetOrgPerson
+cn: Special Charsman
+sn: Charsman
+uid: Пользователь
+mail: scharsman@example.io
+userPassword: {SSHA}XQSZqLPvYgm30wR7pk67a1GW+q+DDvSj
+
+# Creates groups ou for swengg
+dn: ou=groups,ou=swengg,dc=min,dc=io
+objectclass: organizationalUnit
+ou: groups
+description: groups branch
+
+# Create project groups
+
+dn: cn=projecta,ou=groups,ou=swengg,dc=min,dc=io
+objectclass: groupofnames
+cn: projecta
+description: Project A group members
+member: uid=dillon,ou=people,ou=swengg,dc=min,dc=io
+
+dn: cn=projectb,ou=groups,ou=swengg,dc=min,dc=io
+objectclass: groupofnames
+cn: projectb
+description: Project B group members
+member: uid=dillon,ou=people,ou=swengg,dc=min,dc=io
+member: uid=liza,ou=people,ou=swengg,dc=min,dc=io
+member: uid=fahim,ou=people,ou=swengg,dc=min,dc=io
+member: uid=Пользователь,OU=people,OU=swengg,DC=min,DC=io

docs/distributed/samples/bootstrap-partial.ldif

@@ -0,0 +1,56 @@
+# Create hardware engg org unit
+dn: ou=hwengg,dc=min,dc=io
+objectClass: organizationalUnit
+ou: hwengg
+
+# Create people sub-org
+dn: ou=people,ou=hwengg,dc=min,dc=io
+objectClass: organizationalUnit
+ou: people
+
+# Create Alice, Bob and Cody in hwengg
+dn: uid=alice1,ou=people,ou=hwengg,dc=min,dc=io
+objectClass: inetOrgPerson
+cn: Alice Smith
+sn: Smith
+uid: alice1
+mail: alice@example.io
+userPassword: {SSHA}Yeh2/IV/q/HjG2yzN3YdE9CAF3EJFCLu
+
+dn: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io
+objectClass: inetOrgPerson
+cn: Robert Fisher
+sn: Fisher
+uid: bobfisher
+mail: bob@example.io
+userPassword: {SSHA}LktfbhK5oXSdDWCNzauJ9JA+Poxinl3y
+
+dn: uid=cody3,ou=people,ou=hwengg,dc=min,dc=io
+objectClass: inetOrgPerson
+cn: Cody Thomas
+sn: Thomas
+uid: cody3
+mail: cody@example.io
+userPassword: {SSHA}H8B0gaOd4bRklK3fXj9ltHvJXWQFXW5Q
+
+# Create groups ou for hwengg
+dn: ou=groups,ou=hwengg,dc=min,dc=io
+objectclass: organizationalUnit
+ou: groups
+description: groups branch
+
+# Create project groups
+
+dn: cn=projectx,ou=groups,ou=hwengg,dc=min,dc=io
+objectclass: groupofnames
+cn: projectx
+description: Project X group members
+member: uid=alice1,ou=people,ou=hwengg,dc=min,dc=io
+member: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io
+
+dn: cn=projecty,ou=groups,ou=hwengg,dc=min,dc=io
+objectclass: groupofnames
+cn: projecty
+description: Project Y group members
+member: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io
+member: uid=cody3,ou=people,ou=hwengg,dc=min,dc=io