Add etcd path prefix for all IAM assets (#8569)

Currently, we use the top-level prefix "config/"
for all our IAM assets, instead of to provide
tenant-level separation bring 'path_prefix'
to namespace the access properly.

Fixes #8567

cmd/api-errors.go

@@ -28,10 +28,10 @@ import (
 	"google.golang.org/api/googleapi"
 
 	minio "github.com/minio/minio-go/v6"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 )

cmd/bucket-handlers.go

@@ -32,10 +32,10 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"

cmd/config-current.go

@@ -25,6 +25,8 @@ import (
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
 	"github.com/minio/minio/cmd/config/etcd"
+	xetcd "github.com/minio/minio/cmd/config/etcd"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/notify"
@@ -34,7 +36,6 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/target/http"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/env"
 )
 
@@ -134,18 +135,18 @@ func lookupConfigs(s config.Config) (err error) {
 		}
 	}
 
-	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
+	etcdCfg, err := xetcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
 		return config.Errorf("Unable to initialize etcd config: %s", err)
 	}
 
-	globalEtcdClient, err = etcd.New(etcdCfg)
+	globalEtcdClient, err = xetcd.New(etcdCfg)
 	if err != nil {
 		return config.Errorf("Unable to initialize etcd config: %s", err)
 	}
 
 	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
-		globalDNSConfig, err = dns.NewCoreDNS(globalEtcdClient,
+		globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
 			dns.DomainNames(globalDomainNames),
 			dns.DomainIPs(globalDomainIPs),
 			dns.DomainPort(globalMinioPort),

cmd/config/etcd/dns/etcd_dns.go

@@ -29,7 +29,7 @@ import (
 	"github.com/minio/minio-go/v6/pkg/set"
 
 	"github.com/coredns/coredns/plugin/etcd/msg"
-	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/clientv3"
 )
 
 // ErrNoEntriesFound - Indicates no entries were found for the given key (directory)
@@ -106,7 +106,7 @@ func msgUnPath(s string) string {
 // Note that this method fetches entries upto only two levels deep.
 func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-	r, err := c.etcdClient.Get(ctx, key, etcd.WithPrefix())
+	r, err := c.etcdClient.Get(ctx, key, clientv3.WithPrefix())
 	defer cancel()
 	if err != nil {
 		return nil, err
@@ -217,7 +217,7 @@ type CoreDNS struct {
 	domainIPs   set.StringSet
 	domainPort  string
 	prefixPath  string
-	etcdClient  *etcd.Client
+	etcdClient  *clientv3.Client
 }
 
 // Option - functional options pattern style
@@ -257,14 +257,14 @@ func CoreDNSPath(prefix string) Option {
 }
 
 // NewCoreDNS - initialize a new coreDNS set/unset values.
-func NewCoreDNS(etcdClient *etcd.Client, setters ...Option) (Config, error) {
+func NewCoreDNS(cfg clientv3.Config, setters ...Option) (*CoreDNS, error) {
-	if etcdClient == nil {
+	etcdClient, err := clientv3.New(cfg)
-		return nil, errors.New("invalid argument")
+	if err != nil {
+		return nil, err
 	}
 
 	args := &CoreDNS{
 		etcdClient: etcdClient,
-		prefixPath: defaultPrefixPath,
 	}
 
 	for _, setter := range setters {

cmd/config/etcd/dns/types.go

@@ -55,13 +55,3 @@ type SrvRecord struct {
 	// Key carries the original key used during Put().
 	Key string `json:"-"`
 }
-
-// Config - represents dns put, get interface. This interface can be
-// used to implement various backends as needed.
-type Config interface {
-	Put(key string) error
-	List() ([]SrvRecord, error)
-	Get(key string) ([]SrvRecord, error)
-	Delete(key string) error
-	DeleteRecord(record SrvRecord) error
-}

cmd/config/etcd/etcd.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/clientv3/namespace"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
@@ -38,12 +39,14 @@ const (
 // etcd environment values
 const (
 	Endpoints     = "endpoints"
+	PathPrefix    = "path_prefix"
 	CoreDNSPath   = "coredns_path"
 	ClientCert    = "client_cert"
 	ClientCertKey = "client_cert_key"
 
 	EnvEtcdState         = "MINIO_ETCD_STATE"
 	EnvEtcdEndpoints     = "MINIO_ETCD_ENDPOINTS"
+	EnvEtcdPathPrefix    = "MINIO_ETCD_PATH_PREFIX"
 	EnvEtcdCoreDNSPath   = "MINIO_ETCD_COREDNS_PATH"
 	EnvEtcdClientCert    = "MINIO_ETCD_CLIENT_CERT"
 	EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
@@ -60,6 +63,10 @@ var (
 			Key:   Endpoints,
 			Value: "",
 		},
+		config.KV{
+			Key:   PathPrefix,
+			Value: "",
+		},
 		config.KV{
 			Key:   CoreDNSPath,
 			Value: "/skydns",
@@ -78,6 +85,7 @@ var (
 // Config - server etcd config.
 type Config struct {
 	Enabled     bool   `json:"enabled"`
+	PathPrefix  string `json:"pathPrefix"`
 	CoreDNSPath string `json:"coreDNSPath"`
 	clientv3.Config
 }
@@ -87,7 +95,14 @@ func New(cfg Config) (*clientv3.Client, error) {
 	if !cfg.Enabled {
 		return nil, nil
 	}
-	return clientv3.New(cfg.Config)
+	cli, err := clientv3.New(cfg.Config)
+	if err != nil {
+		return nil, err
+	}
+	cli.KV = namespace.NewKV(cli.KV, cfg.PathPrefix)
+	cli.Watcher = namespace.NewWatcher(cli.Watcher, cfg.PathPrefix)
+	cli.Lease = namespace.NewLease(cli.Lease, cfg.PathPrefix)
+	return cli, nil
 }
 
 func parseEndpoints(endpoints string) ([]string, bool, error) {
@@ -123,7 +138,9 @@ func lookupLegacyConfig(rootCAs *x509.CertPool) (Config, error) {
 	cfg.DialTimeout = defaultDialTimeout
 	cfg.DialKeepAliveTime = defaultDialKeepAlive
 	cfg.Endpoints = etcdEndpoints
-	cfg.CoreDNSPath = "/skydns"
+	cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, "/skydns")
+	// Default path prefix for all keys on etcd, other than CoreDNSPath.
+	cfg.PathPrefix = env.Get(EnvEtcdPathPrefix, "")
 	if etcdSecure {
 		cfg.TLS = &tls.Config{
 			RootCAs: rootCAs,
@@ -149,13 +166,9 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
 		return cfg, err
 	}
 
-	stateBool, err := config.ParseBool(env.Get(EnvEtcdState, config.StateOn))
+	if env.Get(EnvEtcdEndpoints, "") != "" && env.Get(EnvEtcdState, "") == "" {
-	if err != nil {
-		return cfg, err
-	}
-
-	if stateBool {
 		// By default state is 'on' to honor legacy config.
+		var err error
 		cfg, err = lookupLegacyConfig(rootCAs)
 		if err != nil {
 			return cfg, err
@@ -166,7 +179,7 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
 		}
 	}
 
-	stateBool, err = config.ParseBool(env.Get(EnvEtcdState, kvs.Get(config.State)))
+	stateBool, err := config.ParseBool(env.Get(EnvEtcdState, kvs.Get(config.State)))
 	if err != nil {
 		if kvs.Empty() {
 			return cfg, nil
@@ -193,6 +206,8 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
 	cfg.DialKeepAliveTime = defaultDialKeepAlive
 	cfg.Endpoints = etcdEndpoints
 	cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath))
+	// Default path prefix for all keys on etcd, other than CoreDNSPath.
+	cfg.PathPrefix = env.Get(EnvEtcdPathPrefix, kvs.Get(PathPrefix))
 	if etcdSecure {
 		cfg.TLS = &tls.Config{
 			RootCAs: rootCAs,

cmd/config/etcd/help.go

@@ -26,9 +26,14 @@ var (
 			Description: `Comma separated list of etcd endpoints eg: "http://localhost:2379"`,
 			Type:        "csv",
 		},
+		config.HelpKV{
+			Key:         PathPrefix,
+			Description: `Default etcd path prefix to populate all IAM assets eg: "customer/"`,
+			Type:        "path",
+		},
 		config.HelpKV{
 			Key:         CoreDNSPath,
-			Description: `CoreDNS etcd path location to populate DNS srv records eg: "/skydns"`,
+			Description: `Default etcd path location to populate bucket DNS srv records eg: "/skydns"`,
 			Optional:    true,
 			Type:        "path",
 		},

cmd/generic-handlers.go

@@ -25,10 +25,10 @@ import (
 	"github.com/minio/minio-go/v6/pkg/set"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/rs/cors"
 )

cmd/globals.go

@@ -27,6 +27,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/policy/opa"
@@ -35,7 +36,6 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/pubsub"
 )
 
@@ -208,7 +208,7 @@ var (
 	globalEtcdClient *etcd.Client
 
 	// Allocated DNS config wrapper over etcd client.
-	globalDNSConfig dns.Config
+	globalDNSConfig *dns.CoreDNS
 
 	// Default usage check interval value.
 	globalDefaultUsageCheckInterval = 12 * time.Hour // 12 hours

cmd/iam-etcd-store.go

@@ -549,6 +549,7 @@ func (ies *IAMEtcdStore) watch(sys *IAMSys) {
 			// Refresh IAMSys with etcd watch.
 			watchCh := ies.client.Watch(context.Background(),
 				iamConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+
 			for {
 				select {
 				case <-GlobalServiceDoneCh:

cmd/object-api-utils.go

@@ -37,11 +37,11 @@ import (
 	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/config/compress"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/wildcard"

cmd/object-handlers.go

@@ -35,11 +35,11 @@ import (
 	"github.com/gorilla/mux"
 	miniogo "github.com/minio/minio-go/v6"
 	"github.com/minio/minio-go/v6/pkg/encrypt"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"

cmd/web-handlers.go

@@ -39,12 +39,12 @@ import (
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/browser"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"