Add CI for etcd IAM backend (#13614)

Runs when ETCD_SERVER env var is set

.github/workflows/iam-integrations.yaml

@@ -0,0 +1,125 @@
+name: IAM integration with external systems
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  ldap-test:
+    name: LDAP Tests with Go ${{ matrix.go-version }}
+    runs-on: ubuntu-latest
+
+    services:
+      openldap:
+        image: quay.io/minio/openldap
+        ports:
+          - "389:389"
+          - "636:636"
+        env:
+          LDAP_ORGANIZATION: "MinIO Inc"
+          LDAP_DOMAIN: "min.io"
+          LDAP_ADMIN_PASSWORD: "admin"
+
+    strategy:
+      matrix:
+        go-version: [1.16.x, 1.17.x]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Test LDAP
+        env:
+          LDAP_TEST_SERVER: "localhost:389"
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-iam
+
+  etcd-test:
+    name: Etcd Backend Tests with Go ${{ matrix.go-version }}
+    runs-on: ubuntu-latest
+
+    services:
+      etcd:
+        image: "quay.io/coreos/etcd:v3.5.1"
+        env:
+          ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
+          ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379"
+        ports:
+          - "2379:2379"
+        options: >-
+          --health-cmd "etcdctl endpoint health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    strategy:
+      matrix:
+        go-version: [1.16.x, 1.17.x]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Test Etcd IAM backend
+        env:
+          ETCD_SERVER: "http://localhost:2379"
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-iam
+
+  iam-etcd-test:
+    name: Etcd Backend + LDAP Tests with Go ${{ matrix.go-version }}
+    runs-on: ubuntu-latest
+
+    services:
+      openldap:
+        image: quay.io/minio/openldap
+        ports:
+          - "389:389"
+          - "636:636"
+        env:
+          LDAP_ORGANIZATION: "MinIO Inc"
+          LDAP_DOMAIN: "min.io"
+          LDAP_ADMIN_PASSWORD: "admin"
+      etcd:
+        image: "quay.io/coreos/etcd:v3.5.1"
+        env:
+          ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
+          ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379"
+        ports:
+          - "2379:2379"
+        options: >-
+          --health-cmd "etcdctl endpoint health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    strategy:
+      matrix:
+        go-version: [1.16.x, 1.17.x]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Test Etcd IAM backend with LDAP IDP
+        env:
+          ETCD_SERVER: "http://localhost:2379"
+          LDAP_TEST_SERVER: "localhost:389"
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-iam

.github/workflows/ldap-integration.yaml

@@ -1,45 +0,0 @@
-name: External IDP Integration Tests
-
-on:
-  pull_request:
-    branches:
-    - master
-
-# This ensures that previous jobs for the PR are canceled when the PR is
-# updated.
-concurrency: 
-  group: ${{ github.workflow }}-${{ github.head_ref }}
-  cancel-in-progress: true
-
-jobs:
-  ldap-test:
-    name: LDAP Tests with Go ${{ matrix.go-version }}
-    runs-on: ubuntu-latest
-
-    services:
-      openldap:
-        image: quay.io/minio/openldap
-        ports:
-          - 389:389
-          - 636:636
-        env:
-          LDAP_ORGANIZATION: "MinIO Inc"
-          LDAP_DOMAIN: "min.io"
-          LDAP_ADMIN_PASSWORD: "admin"
-
-
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
-      - name: Test LDAP
-        env:
-          LDAP_TEST_SERVER: "localhost:389"
-        run: |
-          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
-          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
-          make test-ldap

Makefile

@@ -46,11 +46,11 @@ test-race: verifiers build
 	@echo "Running unit tests under -race"
 	@(env bash $(PWD)/buildscripts/race.sh)
 
-test-ldap: build
+test-iam: build
-	@echo "Running tests for LDAP integration"
+	@echo "Running tests for IAM (external IDP, etcd backends)"
-	@CGO_ENABLED=0 go test -tags kqueue -v -run TestIAMWithLDAPServerSuite ./cmd
+	@CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
-	@echo "Running tests for LDAP integration with -race"
+	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
-	@CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAMWithLDAPServerSuite ./cmd
+	@CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
 verify: ## verify minio various setups
 	@echo "Verifying build with race"

cmd/admin-handlers-users-race_test.go

@@ -15,6 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+//go:build !race
 // +build !race
 
 // Tests in this file are not run under the `-race` flag as they are too slow
@@ -40,20 +41,34 @@ func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
 }
 
 func TestIAMInternalIDPConcurrencyServerSuite(t *testing.T) {
-	testCases := []*TestSuiteIAM{
+	baseTestCases := []TestSuiteCommon{
 		// Init and run test on FS backend with signature v4.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4}),
+		{serverType: "FS", signer: signerV4},
 		// Init and run test on FS backend, with tls enabled.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4, secure: true}),
+		{serverType: "FS", signer: signerV4, secure: true},
 		// Init and run test on Erasure backend.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "Erasure", signer: signerV4}),
+		{serverType: "Erasure", signer: signerV4},
 		// Init and run test on ErasureSet backend.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "ErasureSet", signer: signerV4}),
+		{serverType: "ErasureSet", signer: signerV4},
+	}
+	testCases := []*TestSuiteIAM{}
+	for _, bt := range baseTestCases {
+		testCases = append(testCases,
+			newTestSuiteIAM(bt, false),
+			newTestSuiteIAM(bt, true),
+		)
 	}
 	for i, testCase := range testCases {
-		t.Run(fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.serverType), func(t *testing.T) {
+		etcdStr := ""
+		if testCase.withEtcdBackend {
+			etcdStr = " (with etcd backend)"
+		}
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			func(t *testing.T) {
 				runAllIAMConcurrencyTests(testCase, &check{t, testCase.serverType})
-		})
+			},
+		)
 	}
 }
 

cmd/admin-handlers-users_test.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -39,13 +40,16 @@ const (
 type TestSuiteIAM struct {
 	TestSuiteCommon
 
+	// Flag to turn on tests for etcd backend IAM
+	withEtcdBackend bool
+
 	endpoint string
 	adm      *madmin.AdminClient
 	client   *minio.Client
 }
 
-func newTestSuiteIAM(c TestSuiteCommon) *TestSuiteIAM {
+func newTestSuiteIAM(c TestSuiteCommon, withEtcdBackend bool) *TestSuiteIAM {
-	return &TestSuiteIAM{TestSuiteCommon: c}
+	return &TestSuiteIAM{TestSuiteCommon: c, withEtcdBackend: withEtcdBackend}
 }
 
 func (s *TestSuiteIAM) iamSetup(c *check) {
@@ -73,10 +77,42 @@ func (s *TestSuiteIAM) iamSetup(c *check) {
 	}
 }
 
+const (
+	EnvTestEtcdBackend = "ETCD_SERVER"
+)
+
+func (s *TestSuiteIAM) setUpEtcd(c *check, etcdServer string) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	configCmds := []string{
+		"etcd",
+		"endpoints=" + etcdServer,
+		"path_prefix=" + mustGetUUID(),
+	}
+	_, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " "))
+	if err != nil {
+		c.Fatalf("unable to setup Etcd for tests: %v", err)
+	}
+
+	s.RestartIAMSuite(c)
+}
+
 func (s *TestSuiteIAM) SetUpSuite(c *check) {
+	// If etcd backend is specified and etcd server is not present, the test
+	// is skipped.
+	etcdServer := os.Getenv(EnvTestEtcdBackend)
+	if s.withEtcdBackend && etcdServer == "" {
+		c.Skip("Skipping etcd backend IAM test as no etcd server is configured.")
+	}
+
 	s.TestSuiteCommon.SetUpSuite(c)
 
 	s.iamSetup(c)
+
+	if s.withEtcdBackend {
+		s.setUpEtcd(c, etcdServer)
+	}
 }
 
 func (s *TestSuiteIAM) RestartIAMSuite(c *check) {
@@ -108,20 +144,34 @@ func runAllIAMTests(suite *TestSuiteIAM, c *check) {
 }
 
 func TestIAMInternalIDPServerSuite(t *testing.T) {
-	testCases := []*TestSuiteIAM{
+	baseTestCases := []TestSuiteCommon{
 		// Init and run test on FS backend with signature v4.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4}),
+		{serverType: "FS", signer: signerV4},
 		// Init and run test on FS backend, with tls enabled.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4, secure: true}),
+		{serverType: "FS", signer: signerV4, secure: true},
 		// Init and run test on Erasure backend.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "Erasure", signer: signerV4}),
+		{serverType: "Erasure", signer: signerV4},
 		// Init and run test on ErasureSet backend.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "ErasureSet", signer: signerV4}),
+		{serverType: "ErasureSet", signer: signerV4},
+	}
+	testCases := []*TestSuiteIAM{}
+	for _, bt := range baseTestCases {
+		testCases = append(testCases,
+			newTestSuiteIAM(bt, false),
+			newTestSuiteIAM(bt, true),
+		)
 	}
 	for i, testCase := range testCases {
-		t.Run(fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.serverType), func(t *testing.T) {
+		etcdStr := ""
+		if testCase.withEtcdBackend {
+			etcdStr = " (with etcd backend)"
+		}
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			func(t *testing.T) {
 				runAllIAMTests(testCase, &check{t, testCase.serverType})
-		})
+			},
+		)
 	}
 }
 

cmd/sts-handlers_test.go

@@ -36,20 +36,34 @@ func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) {
 }
 
 func TestIAMInternalIDPSTSServerSuite(t *testing.T) {
-	testCases := []*TestSuiteIAM{
+	baseTestCases := []TestSuiteCommon{
 		// Init and run test on FS backend with signature v4.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4}),
+		{serverType: "FS", signer: signerV4},
 		// Init and run test on FS backend, with tls enabled.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4, secure: true}),
+		{serverType: "FS", signer: signerV4, secure: true},
 		// Init and run test on Erasure backend.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "Erasure", signer: signerV4}),
+		{serverType: "Erasure", signer: signerV4},
 		// Init and run test on ErasureSet backend.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "ErasureSet", signer: signerV4}),
+		{serverType: "ErasureSet", signer: signerV4},
+	}
+	testCases := []*TestSuiteIAM{}
+	for _, bt := range baseTestCases {
+		testCases = append(testCases,
+			newTestSuiteIAM(bt, false),
+			newTestSuiteIAM(bt, true),
+		)
 	}
 	for i, testCase := range testCases {
-		t.Run(fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.serverType), func(t *testing.T) {
+		etcdStr := ""
+		if testCase.withEtcdBackend {
+			etcdStr = " (with etcd backend)"
+		}
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			func(t *testing.T) {
 				runAllIAMSTSTests(testCase, &check{t, testCase.serverType})
-		})
+			},
+		)
 	}
 }
 
@@ -135,21 +149,16 @@ func (s *TestSuiteIAM) TestSTS(c *check) {
 	}
 }
 
-const (
-	EnvTestLDAPServer = "LDAP_TEST_SERVER"
-)
-
 func (s *TestSuiteIAM) GetLDAPServer(c *check) string {
 	return os.Getenv(EnvTestLDAPServer)
 }
 
 // SetUpLDAP - expects to setup an LDAP test server using the test LDAP
 // container and canned data from https://github.com/minio/minio-ldap-testing
-func (s *TestSuiteIAM) SetUpLDAP(c *check) {
+func (s *TestSuiteIAM) SetUpLDAP(c *check, serverAddr string) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
 
-	serverAddr := s.GetLDAPServer(c)
 	configCmds := []string{
 		"identity_ldap",
 		fmt.Sprintf("server_addr=%s", serverAddr),
@@ -169,31 +178,50 @@ func (s *TestSuiteIAM) SetUpLDAP(c *check) {
 	s.RestartIAMSuite(c)
 }
 
+const (
+	EnvTestLDAPServer = "LDAP_TEST_SERVER"
+)
+
 func TestIAMWithLDAPServerSuite(t *testing.T) {
-	testCases := []*TestSuiteIAM{
+	baseTestCases := []TestSuiteCommon{
 		// Init and run test on FS backend with signature v4.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4}),
+		{serverType: "FS", signer: signerV4},
 		// Init and run test on FS backend, with tls enabled.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4, secure: true}),
+		{serverType: "FS", signer: signerV4, secure: true},
 		// Init and run test on Erasure backend.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "Erasure", signer: signerV4}),
+		{serverType: "Erasure", signer: signerV4},
 		// Init and run test on ErasureSet backend.
-		newTestSuiteIAM(TestSuiteCommon{serverType: "ErasureSet", signer: signerV4}),
+		{serverType: "ErasureSet", signer: signerV4},
+	}
+	testCases := []*TestSuiteIAM{}
+	for _, bt := range baseTestCases {
+		testCases = append(testCases,
+			newTestSuiteIAM(bt, false),
+			newTestSuiteIAM(bt, true),
+		)
 	}
 	for i, testCase := range testCases {
-		t.Run(fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.serverType), func(t *testing.T) {
+		etcdStr := ""
+		if testCase.withEtcdBackend {
+			etcdStr = " (with etcd backend)"
+		}
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			func(t *testing.T) {
 				c := &check{t, testCase.serverType}
 				suite := testCase
 
-			if suite.GetLDAPServer(c) == "" {
+				ldapServer := os.Getenv(EnvTestLDAPServer)
-				return
+				if ldapServer == "" {
+					c.Skip("Skipping LDAP test as no LDAP server is provided.")
 				}
 
 				suite.SetUpSuite(c)
-			suite.SetUpLDAP(c)
+				suite.SetUpLDAP(c, ldapServer)
 				suite.TestLDAPSTS(c)
 				suite.TearDownSuite(c)
-		})
+			},
+		)
 	}
 }
 

cmd/test-utils_test.go

@@ -355,6 +355,8 @@ func initTestServerWithBackend(ctx context.Context, t TestErrHandler, testServer
 
 	newAllSubsystems()
 
+	globalEtcdClient = nil
+
 	initAllSubsystems(ctx, objLayer)
 
 	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)