diff --git a/.github/workflows/iam-integrations.yaml b/.github/workflows/iam-integrations.yaml index 59dfd24fd..846a2b8f7 100644 --- a/.github/workflows/iam-integrations.yaml +++ b/.github/workflows/iam-integrations.yaml @@ -1,4 +1,4 @@ -name: IAM integration with external systems +name: IAM integration on: pull_request: @@ -12,75 +12,8 @@ concurrency: cancel-in-progress: true jobs: - ldap-test: - name: LDAP Tests with Go ${{ matrix.go-version }} - runs-on: ubuntu-latest - - services: - openldap: - image: quay.io/minio/openldap - ports: - - "389:389" - - "636:636" - env: - LDAP_ORGANIZATION: "MinIO Inc" - LDAP_DOMAIN: "min.io" - LDAP_ADMIN_PASSWORD: "admin" - - strategy: - matrix: - go-version: [1.16.x, 1.17.x] - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - name: Test LDAP - env: - LDAP_TEST_SERVER: "localhost:389" - run: | - sudo sysctl net.ipv6.conf.all.disable_ipv6=0 - sudo sysctl net.ipv6.conf.default.disable_ipv6=0 - make test-iam - - etcd-test: - name: Etcd Backend Tests with Go ${{ matrix.go-version }} - runs-on: ubuntu-latest - - services: - etcd: - image: "quay.io/coreos/etcd:v3.5.1" - env: - ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" - ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379" - ports: - - "2379:2379" - options: >- - --health-cmd "etcdctl endpoint health" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - strategy: - matrix: - go-version: [1.16.x, 1.17.x] - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - name: Test Etcd IAM backend - env: - ETCD_SERVER: "http://localhost:2379" - run: | - sudo sysctl net.ipv6.conf.all.disable_ipv6=0 - sudo sysctl net.ipv6.conf.default.disable_ipv6=0 - make test-iam - - iam-etcd-test: - name: Etcd Backend + LDAP Tests with Go ${{ matrix.go-version }} + iam-matrix-test: + name: "[Go=${{ matrix.go-version }}|ldap=${{ matrix.ldap }}|etcd=${{ matrix.etcd }}|openid=${{ matrix.openid }}]" runs-on: ubuntu-latest services: @@ -105,20 +38,41 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + openid: + image: quay.io/minio/dex + ports: + - "5556:5556" + env: + DEX_LDAP_SERVER: "openldap:389" strategy: + # When ldap, etcd or openid vars are empty below, those external servers + # are turned off - i.e. if ldap="", then ldap server is not enabled for + # the tests. matrix: go-version: [1.16.x, 1.17.x] + ldap: ["", "localhost:389"] + etcd: ["", "http://localhost:2379"] + openid: ["", "http://127.0.0.1:5556/dex"] + exclude: + # exclude combos where all are empty. + - ldap: "" + etcd: "" + openid: "" + # exclude combos where both ldap and openid IDPs are specified. + - ldap: "localhost:389" + openid: "http://127.0.0.1:5556/dex" steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: go-version: ${{ matrix.go-version }} - - name: Test Etcd IAM backend with LDAP IDP + - name: Test LDAP/OpenID with Etcd combo env: - ETCD_SERVER: "http://localhost:2379" - LDAP_TEST_SERVER: "localhost:389" + LDAP_TEST_SERVER: ${{ matrix.ldap }} + ETCD_SERVER: ${{ matrix.etcd }} + OPENID_TEST_SERVER: ${{ matrix.openid }} run: | sudo sysctl net.ipv6.conf.all.disable_ipv6=0 sudo sysctl net.ipv6.conf.default.disable_ipv6=0 diff --git a/cmd/sts-handlers_test.go b/cmd/sts-handlers_test.go index 2b5db81eb..492cf259c 100644 --- a/cmd/sts-handlers_test.go +++ b/cmd/sts-handlers_test.go @@ -20,13 +20,18 @@ package cmd import ( "context" "fmt" + "net/http" + "net/url" "os" "strings" "testing" + "time" + "github.com/coreos/go-oidc" "github.com/minio/madmin-go" minio "github.com/minio/minio-go/v7" cr "github.com/minio/minio-go/v7/pkg/credentials" + "golang.org/x/oauth2" ) func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) { @@ -235,7 +240,7 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) { c.Fatalf("bucket create error: %v", err) } - // Create policy, user and associate policy + // Create policy policy := "mypolicy" policyBytes := []byte(fmt.Sprintf(`{ "Version": "2012-10-17", @@ -267,7 +272,7 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) { _, err = ldapID.Retrieve() if err == nil { - c.Fatalf("Expected to fail to create a user with no associated policy!") + c.Fatalf("Expected to fail to create STS cred with no associated policy!") } userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" @@ -338,3 +343,276 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) { err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) c.Assert(err.Error(), "Access Denied.") } + +func (s *TestSuiteIAM) TestOpenIDSTS(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Generate web identity STS token by interacting with OpenID IDP. + token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon") + if err != nil { + c.Fatalf("mock user err: %v", err) + } + // fmt.Printf("TOKEN: %s\n", token) + + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + }, nil + }, + } + + // Create policy - with name as one of the groups in OpenID the user is + // a member of. + policy := "projecta" + policyBytes := []byte(fmt.Sprintf(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket)) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + value, err := webID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + if err.Error() != "Access Denied." { + c.Fatalf("unexpected non-access-denied err: %v", err) + } +} + +type providerParams struct { + clientID, clientSecret, providerURL, redirectURL string +} + +var testProvider = providerParams{ + clientID: "minio-client-app", + clientSecret: "minio-client-app-secret", + providerURL: "http://127.0.0.1:5556/dex", + redirectURL: "http://127.0.0.1:10000/oauth_callback", +} + +// mockTestUserInteraction - tries to login to dex using provided credentials. +// It performs the user's browser interaction to login and retrieves the auth +// code from dex and exchanges it for a JWT. +func mockTestUserInteraction(ctx context.Context, pro providerParams, username, password string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + provider, err := oidc.NewProvider(ctx, pro.providerURL) + if err != nil { + return "", fmt.Errorf("unable to create provider: %v", err) + } + + // Configure an OpenID Connect aware OAuth2 client. + oauth2Config := oauth2.Config{ + ClientID: pro.clientID, + ClientSecret: pro.clientSecret, + RedirectURL: pro.redirectURL, + + // Discovery returns the OAuth2 endpoints. + Endpoint: provider.Endpoint(), + + // "openid" is a required scope for OpenID Connect flows. + Scopes: []string{oidc.ScopeOpenID, "groups"}, + } + + state := "xxx" + authCodeURL := oauth2Config.AuthCodeURL(state) + // fmt.Printf("authcodeurl: %s\n", authCodeURL) + + var lastReq *http.Request + checkRedirect := func(req *http.Request, via []*http.Request) error { + // fmt.Printf("CheckRedirect:\n") + // fmt.Printf("Upcoming: %s %#v\n", req.URL.String(), req) + // for _, c := range via { + // fmt.Printf("Sofar: %s %#v\n", c.URL.String(), c) + // } + // Save the last request in a redirect chain. + lastReq = req + // We do not follow redirect back to client application. + if req.URL.Path == "/oauth_callback" { + return http.ErrUseLastResponse + } + return nil + } + + dexClient := http.Client{ + CheckRedirect: checkRedirect, + } + + u, err := url.Parse(authCodeURL) + if err != nil { + return "", fmt.Errorf("url parse err: %v", err) + } + + // Start the user auth flow. This page would present the login with + // email or LDAP option. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return "", fmt.Errorf("new request err: %v", err) + } + _, err = dexClient.Do(req) + // fmt.Printf("Do: %#v %#v\n", resp, err) + if err != nil { + return "", fmt.Errorf("auth url request err: %v", err) + } + + // Modify u to choose the ldap option + u.Path += "/ldap" + // fmt.Println(u) + + // Pick the LDAP login option. This would return a form page after + // following some redirects. `lastReq` would be the URL of the form + // page, where we need to POST (submit) the form. + req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return "", fmt.Errorf("new request err (/ldap): %v", err) + } + _, err = dexClient.Do(req) + if err != nil { + return "", fmt.Errorf("request err: %v", err) + } + + // Fill the login form with our test creds: + // fmt.Printf("login form url: %s\n", lastReq.URL.String()) + formData := url.Values{} + formData.Set("login", username) + formData.Set("password", password) + req, err = http.NewRequestWithContext(ctx, http.MethodPost, lastReq.URL.String(), strings.NewReader(formData.Encode())) + if err != nil { + return "", fmt.Errorf("new request err (/login): %v", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + _, err = dexClient.Do(req) + if err != nil { + return "", fmt.Errorf("post form err: %v", err) + } + // fmt.Printf("resp: %#v %#v\n", resp.StatusCode, resp.Header) + // fmt.Printf("lastReq: %#v\n", lastReq.URL.String()) + + // On form submission, the last redirect response contains the auth + // code, which we now have in `lastReq`. Exchange it for a JWT id_token. + q := lastReq.URL.Query() + code := q.Get("code") + oauth2Token, err := oauth2Config.Exchange(ctx, code) + if err != nil { + return "", fmt.Errorf("unable to exchange code for id token: %v", err) + } + + rawIDToken, ok := oauth2Token.Extra("id_token").(string) + if !ok { + return "", fmt.Errorf("id_token not found!") + } + + // fmt.Printf("TOKEN: %s\n", rawIDToken) + return rawIDToken, nil +} + +const ( + EnvTestOpenIDServer = "OPENID_TEST_SERVER" +) + +// SetUpOpenID - expects to setup an OpenID test server using the test OpenID +// container and canned data from https://github.com/minio/minio-ldap-testing +func (s *TestSuiteIAM) SetUpOpenID(c *check, serverAddr string) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + configCmds := []string{ + "identity_openid", + fmt.Sprintf("config_url=%s/.well-known/openid-configuration", serverAddr), + "client_id=minio-client-app", + "client_secret=minio-client-app-secret", + "claim_name=groups", + "scopes=openid,groups", + "redirect_uri=http://127.0.0.1:10000/oauth_callback", + } + _, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " ")) + if err != nil { + c.Fatalf("unable to setup OpenID for tests: %v", err) + } + + s.RestartIAMSuite(c) +} + +func TestIAMWithOpenIDServerSuite(t *testing.T) { + baseTestCases := []TestSuiteCommon{ + // Init and run test on FS backend with signature v4. + {serverType: "FS", signer: signerV4}, + // Init and run test on FS backend, with tls enabled. + {serverType: "FS", signer: signerV4, secure: true}, + // Init and run test on Erasure backend. + {serverType: "Erasure", signer: signerV4}, + // Init and run test on ErasureSet backend. + {serverType: "ErasureSet", signer: signerV4}, + } + testCases := []*TestSuiteIAM{} + for _, bt := range baseTestCases { + testCases = append(testCases, + newTestSuiteIAM(bt, false), + newTestSuiteIAM(bt, true), + ) + } + for i, testCase := range testCases { + etcdStr := "" + if testCase.withEtcdBackend { + etcdStr = " (with etcd backend)" + } + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + openIDServer := os.Getenv(EnvTestOpenIDServer) + if openIDServer == "" { + c.Skip("Skipping OpenID test as no OpenID server is provided.") + } + + suite.SetUpSuite(c) + suite.SetUpOpenID(c, openIDServer) + suite.TestOpenIDSTS(c) + suite.TearDownSuite(c) + }, + ) + } +} diff --git a/go.mod b/go.mod index 3c75c61ad..624af4e5f 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/colinmarc/hdfs/v2 v2.2.0 github.com/coredns/coredns v1.4.0 + github.com/coreos/go-oidc v2.1.0+incompatible github.com/cosnicolaou/pbzip2 v1.0.1 github.com/dchest/siphash v1.2.1 github.com/djherbis/atime v1.0.0 @@ -84,7 +85,8 @@ require ( go.uber.org/atomic v1.9.0 go.uber.org/zap v1.19.1 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - golang.org/x/net v0.0.0-20211020060615-d418f374d309 // indirect + golang.org/x/net v0.0.0-20211020060615-d418f374d309 + golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f golang.org/x/sys v0.0.0-20211020174200-9d6173849985 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac google.golang.org/api v0.58.0 diff --git a/go.sum b/go.sum index 59ae9ccff..5e0c49d67 100644 --- a/go.sum +++ b/go.sum @@ -265,6 +265,7 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -1276,6 +1277,7 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 h1:0XM1XL/OFFJjXsYXlG30spTkV/E9+gmd5GD1w2HE8xM= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.44.1/go.mod h1:3WYi4xqXxGGXWDdQIITnLNmuDzO5n6wYva9spVhR4fg= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.46.0/go.mod h1:3WYi4xqXxGGXWDdQIITnLNmuDzO5n6wYva9spVhR4fg= @@ -2214,6 +2216,7 @@ gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLv gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=