fix: progagation of service accounts for site replication (#14054)

- Only non-root-owned service accounts are replicated for now.
- Add integration tests for OIDC with site replication

.github/workflows/iam-integrations.yaml

@@ -88,4 +88,8 @@ jobs:
       - name: Test LDAP for automatic site replication
         if: matrix.ldap == 'localhost:389'
         run: |
-          make test-site-replication
+          make test-site-replication-ldap
+      - name: Test OIDC for automatic site replication
+        if: matrix.openid == 'http://127.0.0.1:5556/dex'
+        run: |
+          make test-site-replication-oidc

Makefile

@@ -59,9 +59,13 @@ test-replication: install ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
 
-test-site-replication: install ## verify automatic site replication
+test-site-replication-ldap: install ## verify automatic site replication
-	@echo "Running tests for automatic site replication of IAM"
+	@echo "Running tests for automatic site replication of IAM (with LDAP)"
-	@(env bash $(PWD)/docs/site-replication/run-multi-site.sh)
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
+
+test-site-replication-oidc: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with OIDC)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)
 
 verify: ## verify minio various setups
 	@echo "Verifying build with race"

cmd/admin-handlers-users.go

@@ -743,14 +743,8 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Call site replication hook. Only LDAP accounts are supported for
+	// Call site replication hook - non-root user accounts are replicated.
-	// replication operations.
+	if svcAccount.ParentUser != globalActiveCred.AccessKey {
-	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, accessKey)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
@@ -767,6 +761,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 			return
 		}
 	}
+
 	writeSuccessNoContent(w)
 }
 
@@ -990,22 +985,14 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	// Save svc acc claims before deletion (for site replication hook).
-	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, serviceAccount)
-	if err != nil && err != errNoSuchServiceAccount {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
 	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Call site replication hook. Only LDAP accounts are supported for
+	// Call site replication hook - non-root user accounts are replicated.
-	// replication operations.
+	if svcAccount.ParentUser != globalActiveCred.AccessKey {
-	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{

cmd/sts-handlers_test.go

@@ -20,19 +20,15 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"net/http"
-	"net/url"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/coreos/go-oidc"
 	"github.com/minio/madmin-go"
 	minio "github.com/minio/minio-go/v7"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"golang.org/x/oauth2"
 )
 
 func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) {
@@ -742,7 +738,7 @@ func (s *TestSuiteIAM) TestOpenIDSTS(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -817,7 +813,7 @@ func (s *TestSuiteIAM) TestOpenIDSTSAddUser(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -907,7 +903,7 @@ func (s *TestSuiteIAM) TestOpenIDServiceAcc(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -985,132 +981,11 @@ func (s *TestSuiteIAM) TestOpenIDServiceAcc(c *check) {
 	c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket)
 }
 
-type providerParams struct {
+var testAppParams = OpenIDClientAppParams{
-	clientID, clientSecret, providerURL, redirectURL string
+	ClientID:     "minio-client-app",
-}
+	ClientSecret: "minio-client-app-secret",
-
+	ProviderURL:  "http://127.0.0.1:5556/dex",
-var testProvider = providerParams{
+	RedirectURL:  "http://127.0.0.1:10000/oauth_callback",
-	clientID:     "minio-client-app",
-	clientSecret: "minio-client-app-secret",
-	providerURL:  "http://127.0.0.1:5556/dex",
-	redirectURL:  "http://127.0.0.1:10000/oauth_callback",
-}
-
-// mockTestUserInteraction - tries to login to dex using provided credentials.
-// It performs the user's browser interaction to login and retrieves the auth
-// code from dex and exchanges it for a JWT.
-func mockTestUserInteraction(ctx context.Context, pro providerParams, username, password string) (string, error) {
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-
-	provider, err := oidc.NewProvider(ctx, pro.providerURL)
-	if err != nil {
-		return "", fmt.Errorf("unable to create provider: %v", err)
-	}
-
-	// Configure an OpenID Connect aware OAuth2 client.
-	oauth2Config := oauth2.Config{
-		ClientID:     pro.clientID,
-		ClientSecret: pro.clientSecret,
-		RedirectURL:  pro.redirectURL,
-
-		// Discovery returns the OAuth2 endpoints.
-		Endpoint: provider.Endpoint(),
-
-		// "openid" is a required scope for OpenID Connect flows.
-		Scopes: []string{oidc.ScopeOpenID, "groups"},
-	}
-
-	state := "xxx"
-	authCodeURL := oauth2Config.AuthCodeURL(state)
-	// fmt.Printf("authcodeurl: %s\n", authCodeURL)
-
-	var lastReq *http.Request
-	checkRedirect := func(req *http.Request, via []*http.Request) error {
-		// fmt.Printf("CheckRedirect:\n")
-		// fmt.Printf("Upcoming: %s %#v\n", req.URL.String(), req)
-		// for _, c := range via {
-		// 	fmt.Printf("Sofar: %s %#v\n", c.URL.String(), c)
-		// }
-		// Save the last request in a redirect chain.
-		lastReq = req
-		// We do not follow redirect back to client application.
-		if req.URL.Path == "/oauth_callback" {
-			return http.ErrUseLastResponse
-		}
-		return nil
-	}
-
-	dexClient := http.Client{
-		CheckRedirect: checkRedirect,
-	}
-
-	u, err := url.Parse(authCodeURL)
-	if err != nil {
-		return "", fmt.Errorf("url parse err: %v", err)
-	}
-
-	// Start the user auth flow. This page would present the login with
-	// email or LDAP option.
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
-	if err != nil {
-		return "", fmt.Errorf("new request err: %v", err)
-	}
-	_, err = dexClient.Do(req)
-	// fmt.Printf("Do: %#v %#v\n", resp, err)
-	if err != nil {
-		return "", fmt.Errorf("auth url request err: %v", err)
-	}
-
-	// Modify u to choose the ldap option
-	u.Path += "/ldap"
-	// fmt.Println(u)
-
-	// Pick the LDAP login option. This would return a form page after
-	// following some redirects. `lastReq` would be the URL of the form
-	// page, where we need to POST (submit) the form.
-	req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
-	if err != nil {
-		return "", fmt.Errorf("new request err (/ldap): %v", err)
-	}
-	_, err = dexClient.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("request err: %v", err)
-	}
-
-	// Fill the login form with our test creds:
-	// fmt.Printf("login form url: %s\n", lastReq.URL.String())
-	formData := url.Values{}
-	formData.Set("login", username)
-	formData.Set("password", password)
-	req, err = http.NewRequestWithContext(ctx, http.MethodPost, lastReq.URL.String(), strings.NewReader(formData.Encode()))
-	if err != nil {
-		return "", fmt.Errorf("new request err (/login): %v", err)
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	_, err = dexClient.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("post form err: %v", err)
-	}
-	// fmt.Printf("resp: %#v %#v\n", resp.StatusCode, resp.Header)
-	// fmt.Printf("lastReq: %#v\n", lastReq.URL.String())
-
-	// On form submission, the last redirect response contains the auth
-	// code, which we now have in `lastReq`. Exchange it for a JWT id_token.
-	q := lastReq.URL.Query()
-	code := q.Get("code")
-	oauth2Token, err := oauth2Config.Exchange(ctx, code)
-	if err != nil {
-		return "", fmt.Errorf("unable to exchange code for id token: %v", err)
-	}
-
-	rawIDToken, ok := oauth2Token.Extra("id_token").(string)
-	if !ok {
-		return "", fmt.Errorf("id_token not found!")
-	}
-
-	// fmt.Printf("TOKEN: %s\n", rawIDToken)
-	return rawIDToken, nil
 }
 
 const (
@@ -1248,7 +1123,7 @@ func (s *TestSuiteIAM) TestOpenIDSTSWithRolePolicy(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -1294,7 +1169,7 @@ func (s *TestSuiteIAM) TestOpenIDServiceAccWithRolePolicy(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}

cmd/utils.go

@@ -43,6 +43,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/coreos/go-oidc"
 	"github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
@@ -58,6 +59,7 @@ import (
 	"github.com/minio/minio/internal/rest"
 	"github.com/minio/pkg/certs"
 	"github.com/minio/pkg/env"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -1160,3 +1162,142 @@ func newTLSConfig(getCert certs.GetCertificateFunc) *tls.Config {
 	}
 	return tlsConfig
 }
+
+/////////// Types and functions for OpenID IAM testing
+
+// OpenIDClientAppParams - contains openID client application params, used in
+// testing.
+type OpenIDClientAppParams struct {
+	ClientID, ClientSecret, ProviderURL, RedirectURL string
+}
+
+// MockOpenIDTestUserInteraction - tries to login to dex using provided credentials.
+// It performs the user's browser interaction to login and retrieves the auth
+// code from dex and exchanges it for a JWT.
+func MockOpenIDTestUserInteraction(ctx context.Context, pro OpenIDClientAppParams, username, password string) (string, error) {
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	provider, err := oidc.NewProvider(ctx, pro.ProviderURL)
+	if err != nil {
+		return "", fmt.Errorf("unable to create provider: %v", err)
+	}
+
+	// Configure an OpenID Connect aware OAuth2 client.
+	oauth2Config := oauth2.Config{
+		ClientID:     pro.ClientID,
+		ClientSecret: pro.ClientSecret,
+		RedirectURL:  pro.RedirectURL,
+
+		// Discovery returns the OAuth2 endpoints.
+		Endpoint: provider.Endpoint(),
+
+		// "openid" is a required scope for OpenID Connect flows.
+		Scopes: []string{oidc.ScopeOpenID, "groups"},
+	}
+
+	state := fmt.Sprintf("x%dx", time.Now().Unix())
+	authCodeURL := oauth2Config.AuthCodeURL(state)
+	// fmt.Printf("authcodeurl: %s\n", authCodeURL)
+
+	var lastReq *http.Request
+	checkRedirect := func(req *http.Request, via []*http.Request) error {
+		// fmt.Printf("CheckRedirect:\n")
+		// fmt.Printf("Upcoming: %s %s\n", req.Method, req.URL.String())
+		// for i, c := range via {
+		// 	fmt.Printf("Sofar %d: %s %s\n", i, c.Method, c.URL.String())
+		// }
+		// Save the last request in a redirect chain.
+		lastReq = req
+		// We do not follow redirect back to client application.
+		if req.URL.Path == "/oauth_callback" {
+			return http.ErrUseLastResponse
+		}
+		return nil
+	}
+
+	dexClient := http.Client{
+		CheckRedirect: checkRedirect,
+	}
+
+	u, err := url.Parse(authCodeURL)
+	if err != nil {
+		return "", fmt.Errorf("url parse err: %v", err)
+	}
+
+	// Start the user auth flow. This page would present the login with
+	// email or LDAP option.
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return "", fmt.Errorf("new request err: %v", err)
+	}
+	_, err = dexClient.Do(req)
+	// fmt.Printf("Do: %#v %#v\n", resp, err)
+	if err != nil {
+		return "", fmt.Errorf("auth url request err: %v", err)
+	}
+
+	// Modify u to choose the ldap option
+	u.Path += "/ldap"
+	// fmt.Println(u)
+
+	// Pick the LDAP login option. This would return a form page after
+	// following some redirects. `lastReq` would be the URL of the form
+	// page, where we need to POST (submit) the form.
+	req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return "", fmt.Errorf("new request err (/ldap): %v", err)
+	}
+	_, err = dexClient.Do(req)
+	// fmt.Printf("Fetch LDAP login page: %#v %#v\n", resp, err)
+	if err != nil {
+		return "", fmt.Errorf("request err: %v", err)
+	}
+	// {
+	// 	bodyBuf, err := ioutil.ReadAll(resp.Body)
+	// 	if err != nil {
+	// 		return "", fmt.Errorf("Error reading body: %v", err)
+	// 	}
+	// 	fmt.Printf("bodyBuf (for LDAP login page): %s\n", string(bodyBuf))
+	// }
+
+	// Fill the login form with our test creds:
+	// fmt.Printf("login form url: %s\n", lastReq.URL.String())
+	formData := url.Values{}
+	formData.Set("login", username)
+	formData.Set("password", password)
+	req, err = http.NewRequestWithContext(ctx, http.MethodPost, lastReq.URL.String(), strings.NewReader(formData.Encode()))
+	if err != nil {
+		return "", fmt.Errorf("new request err (/login): %v", err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	_, err = dexClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("post form err: %v", err)
+	}
+	// fmt.Printf("resp: %#v %#v\n", resp.StatusCode, resp.Header)
+	// bodyBuf, err := ioutil.ReadAll(resp.Body)
+	// if err != nil {
+	// 	return "", fmt.Errorf("Error reading body: %v", err)
+	// }
+	// fmt.Printf("resp body: %s\n", string(bodyBuf))
+	// fmt.Printf("lastReq: %#v\n", lastReq.URL.String())
+
+	// On form submission, the last redirect response contains the auth
+	// code, which we now have in `lastReq`. Exchange it for a JWT id_token.
+	q := lastReq.URL.Query()
+	// fmt.Printf("lastReq.URL: %#v q: %#v\n", lastReq.URL, q)
+	code := q.Get("code")
+	oauth2Token, err := oauth2Config.Exchange(ctx, code)
+	if err != nil {
+		return "", fmt.Errorf("unable to exchange code for id token: %v", err)
+	}
+
+	rawIDToken, ok := oauth2Token.Extra("id_token").(string)
+	if !ok {
+		return "", fmt.Errorf("id_token not found!")
+	}
+
+	// fmt.Printf("TOKEN: %s\n", rawIDToken)
+	return rawIDToken, nil
+}

docs/site-replication/gen-oidc-sts-cred.go

@@ -0,0 +1,78 @@
+//go:build ignore
+// +build ignore
+
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+// This programs mocks user interaction against Dex IDP and generates STS
+// credentials. It is for MinIO testing purposes only.
+//
+// Run like:
+//
+// $ MINIO_ENDPOINT=http://localhost:9000 go run gen-oidc-sts-cred.go
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+
+	cr "github.com/minio/minio-go/v7/pkg/credentials"
+	cmd "github.com/minio/minio/cmd"
+)
+
+func main() {
+	ctx := context.Background()
+
+	endpoint := os.Getenv("MINIO_ENDPOINT")
+	if endpoint == "" {
+		log.Fatalf("Please specify a MinIO server endpoint environment variable like:\n\n\texport MINIO_ENDPOINT=http://localhost:9000")
+	}
+
+	appParams := cmd.OpenIDClientAppParams{
+		ClientID:     "minio-client-app",
+		ClientSecret: "minio-client-app-secret",
+		ProviderURL:  "http://127.0.0.1:5556/dex",
+		RedirectURL:  "http://127.0.0.1:10000/oauth_callback",
+	}
+
+	oidcToken, err := cmd.MockOpenIDTestUserInteraction(ctx, appParams, "dillon@example.io", "dillon")
+	if err != nil {
+		log.Fatalf("Failed to generate OIDC token: %v", err)
+	}
+
+	webID := cr.STSWebIdentity{
+		Client:      &http.Client{},
+		STSEndpoint: endpoint,
+		GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) {
+			return &cr.WebIdentityToken{
+				Token: oidcToken,
+			}, nil
+		},
+	}
+
+	value, err := webID.Retrieve()
+	if err != nil {
+		log.Fatalf("Expected to generate credentials: %v", err)
+	}
+
+	// Print credentials separated by colons:
+	fmt.Printf("%s:%s:%s\n", value.AccessKeyID, value.SecretAccessKey, value.SessionToken)
+}

docs/site-replication/run-multi-site-oidc.sh

@@ -0,0 +1,223 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2120
+exit_1() {
+    cleanup
+    exit 1
+}
+
+cleanup() {
+    echo "Cleaning up instances of MinIO"
+    pkill minio
+    pkill -9 minio
+    rm -rf /tmp/minio{1,2,3}
+}
+
+cleanup
+
+unset MINIO_KMS_KES_CERT_FILE
+unset MINIO_KMS_KES_KEY_FILE
+unset MINIO_KMS_KES_ENDPOINT
+unset MINIO_KMS_KES_KEY_NAME
+
+export MINIO_BROWSER=off
+export MINIO_ROOT_USER="minio"
+export MINIO_ROOT_PASSWORD="minio123"
+export MINIO_KMS_AUTO_ENCRYPTION=off
+export MINIO_PROMETHEUS_AUTH_TYPE=public
+export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
+export MINIO_IDENTITY_OPENID_CONFIG_URL="http://localhost:5556/dex/.well-known/openid-configuration"
+export MINIO_IDENTITY_OPENID_CLIENT_ID="minio-client-app"
+export MINIO_IDENTITY_OPENID_CLIENT_SECRET="minio-client-app-secret"
+export MINIO_IDENTITY_OPENID_CLAIM_NAME="groups"
+export MINIO_IDENTITY_OPENID_SCOPES="openid,groups"
+
+export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:10000/oauth_callback"
+minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4} >/tmp/minio1_1.log 2>&1 &
+
+export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:11000/oauth_callback"
+minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4} >/tmp/minio2_1.log 2>&1 &
+
+export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:12000/oauth_callback"
+minio server --address ":9003" --console-address ":12000" /tmp/minio3/{1...4} >/tmp/minio3_1.log 2>&1 &
+
+if [ ! -f ./mc ]; then
+    wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \
+        && chmod +x mc
+fi
+
+sleep 10
+
+export MC_HOST_minio1=http://minio:minio123@localhost:9001
+export MC_HOST_minio2=http://minio:minio123@localhost:9002
+export MC_HOST_minio3=http://minio:minio123@localhost:9003
+
+./mc admin replicate add minio1 minio2 minio3
+
+./mc admin policy add minio1 projecta ./docs/site-replication/rw.json
+sleep 5
+
+./mc admin policy info minio2 projecta >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+    echo "expecting the command to succeed, exiting.."
+    exit_1;
+fi
+./mc admin policy info minio3 projecta >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+    echo "expecting the command to succeed, exiting.."
+    exit_1;
+fi
+
+./mc admin policy remove minio3 projecta
+
+sleep 10
+./mc admin policy info minio1 projecta
+if [ $? -eq 0 ]; then
+    echo "expecting the command to fail, exiting.."
+    exit_1;
+fi
+
+./mc admin policy info minio2 projecta
+if [ $? -eq 0 ]; then
+    echo "expecting the command to fail, exiting.."
+    exit_1;
+fi
+
+./mc admin policy add minio1 projecta ./docs/site-replication/rw.json
+sleep 5
+
+# Generate STS credential with STS call to minio1
+STS_CRED=$(MINIO_ENDPOINT=http://localhost:9001 go run ./docs/site-replication/gen-oidc-sts-cred.go)
+
+MC_HOST_foo=http://${STS_CRED}@localhost:9001 ./mc ls foo
+if [ $? -ne 0 ]; then
+    echo "Expected sts credential to work, exiting.."
+    exit_1;
+fi
+
+sleep 2
+
+# Check that the STS credential works on minio2 and minio3.
+MC_HOST_foo=http://${STS_CRED}@localhost:9002 ./mc ls foo
+if [ $? -ne 0 ]; then
+    echo "Expected sts credential to work, exiting.."
+    exit_1;
+fi
+
+MC_HOST_foo=http://${STS_CRED}@localhost:9003 ./mc ls foo
+if [ $? -ne 0 ]; then
+    echo "Expected sts credential to work, exiting.."
+    exit_1;
+fi
+
+STS_ACCESS_KEY=$(echo ${STS_CRED} | cut -d ':' -f 1)
+
+# Create service account for STS user
+./mc admin user svcacct add minio2 $STS_ACCESS_KEY --access-key testsvc --secret-key testsvc123
+if [ $? -ne 0 ]; then
+    echo "adding svc account failed, exiting.."
+    exit_1;
+fi
+
+sleep 10
+
+./mc admin user svcacct info minio1 testsvc
+if [ $? -ne 0 ]; then
+    echo "svc account not mirrored, exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct info minio2 testsvc
+if [ $? -ne 0 ]; then
+    echo "svc account not mirrored, exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct rm minio1 testsvc
+if [ $? -ne 0 ]; then
+    echo "removing svc account failed, exiting.."
+    exit_1;
+fi
+
+sleep 10
+./mc admin user svcacct info minio2 testsvc
+if [ $? -eq 0 ]; then
+    echo "svc account found after delete, exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct info minio3 testsvc
+if [ $? -eq 0 ]; then
+    echo "svc account found after delete, exiting.."
+    exit_1;
+fi
+
+./mc mb minio1/newbucket
+
+sleep 5
+./mc stat minio2/newbucket
+if [ $? -ne 0 ]; then
+    echo "expecting bucket to be present. exiting.."
+    exit_1;
+fi
+
+./mc stat minio3/newbucket
+if [ $? -ne 0 ]; then
+    echo "expecting bucket to be present. exiting.."
+    exit_1;
+fi
+
+./mc cp README.md minio2/newbucket/
+
+sleep 5
+./mc stat minio1/newbucket/README.md
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+
+./mc stat minio3/newbucket/README.md
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+
+./mc rm minio3/newbucket/README.md
+sleep 5
+
+./mc stat minio2/newbucket/README.md
+if [ $? -eq 0 ]; then
+    echo "expected file to be deleted, exiting.."
+    exit_1;
+fi
+
+./mc stat minio1/newbucket/README.md
+if [ $? -eq 0 ]; then
+    echo "expected file to be deleted, exiting.."
+    exit_1;
+fi
+
+./mc mb --with-lock minio3/newbucket-olock
+sleep 5
+
+enabled_minio2=$(./mc stat --json minio2/newbucket-olock| jq -r .metadata.ObjectLock.enabled)
+if [ $? -ne 0 ]; then
+    echo "expected bucket to be mirrored with object-lock but not present, exiting..."
+    exit_1;
+fi
+
+if [ "${enabled_minio2}" != "Enabled" ]; then
+    echo "expected bucket to be mirrored with object-lock enabled, exiting..."
+    exit_1;
+fi
+
+enabled_minio1=$(./mc stat --json minio1/newbucket-olock| jq -r .metadata.ObjectLock.enabled)
+if [ $? -ne 0 ]; then
+    echo "expected bucket to be mirrored with object-lock but not present, exiting..."
+    exit_1;
+fi
+
+if [ "${enabled_minio1}" != "Enabled" ]; then
+    echo "expected bucket to be mirrored with object-lock enabled, exiting..."
+    exit_1;
+fi