rename all remaining packages to internal/ (#12418)

This is to ensure that there are no projects
that try to import `minio/minio/pkg` into
their own repo. Any such common packages should
go to `https://github.com/minio/pkg`

internal/config/etcd/etcd.go

@@ -0,0 +1,176 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/internal/config"
+	xnet "github.com/minio/minio/internal/net"
+	"github.com/minio/pkg/env"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/namespace"
+	"go.uber.org/zap"
+)
+
+const (
+	// Default values used while communicating with etcd.
+	defaultDialTimeout   = 5 * time.Second
+	defaultDialKeepAlive = 30 * time.Second
+)
+
+// etcd environment values
+const (
+	Endpoints     = "endpoints"
+	PathPrefix    = "path_prefix"
+	CoreDNSPath   = "coredns_path"
+	ClientCert    = "client_cert"
+	ClientCertKey = "client_cert_key"
+
+	EnvEtcdEndpoints     = "MINIO_ETCD_ENDPOINTS"
+	EnvEtcdPathPrefix    = "MINIO_ETCD_PATH_PREFIX"
+	EnvEtcdCoreDNSPath   = "MINIO_ETCD_COREDNS_PATH"
+	EnvEtcdClientCert    = "MINIO_ETCD_CLIENT_CERT"
+	EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
+)
+
+// DefaultKVS - default KV settings for etcd.
+var (
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   Endpoints,
+			Value: "",
+		},
+		config.KV{
+			Key:   PathPrefix,
+			Value: "",
+		},
+		config.KV{
+			Key:   CoreDNSPath,
+			Value: "/skydns",
+		},
+		config.KV{
+			Key:   ClientCert,
+			Value: "",
+		},
+		config.KV{
+			Key:   ClientCertKey,
+			Value: "",
+		},
+	}
+)
+
+// Config - server etcd config.
+type Config struct {
+	Enabled     bool   `json:"enabled"`
+	PathPrefix  string `json:"pathPrefix"`
+	CoreDNSPath string `json:"coreDNSPath"`
+	clientv3.Config
+}
+
+// New - initialize new etcd client.
+func New(cfg Config) (*clientv3.Client, error) {
+	if !cfg.Enabled {
+		return nil, nil
+	}
+	cli, err := clientv3.New(cfg.Config)
+	if err != nil {
+		return nil, err
+	}
+	cli.KV = namespace.NewKV(cli.KV, cfg.PathPrefix)
+	cli.Watcher = namespace.NewWatcher(cli.Watcher, cfg.PathPrefix)
+	cli.Lease = namespace.NewLease(cli.Lease, cfg.PathPrefix)
+	return cli, nil
+}
+
+func parseEndpoints(endpoints string) ([]string, bool, error) {
+	etcdEndpoints := strings.Split(endpoints, config.ValueSeparator)
+
+	var etcdSecure bool
+	for _, endpoint := range etcdEndpoints {
+		u, err := xnet.ParseHTTPURL(endpoint)
+		if err != nil {
+			return nil, false, err
+		}
+		if etcdSecure && u.Scheme == "http" {
+			return nil, false, config.Errorf("all endpoints should be https or http: %s", endpoint)
+		}
+		// If one of the endpoint is https, we will use https directly.
+		etcdSecure = etcdSecure || u.Scheme == "https"
+	}
+
+	return etcdEndpoints, etcdSecure, nil
+}
+
+// Enabled returns if etcd is enabled.
+func Enabled(kvs config.KVS) bool {
+	endpoints := kvs.Get(Endpoints)
+	return endpoints != ""
+}
+
+// LookupConfig - Initialize new etcd config.
+func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
+	cfg := Config{}
+	if err := config.CheckValidKeys(config.EtcdSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+
+	endpoints := env.Get(EnvEtcdEndpoints, kvs.Get(Endpoints))
+	if endpoints == "" {
+		return cfg, nil
+	}
+
+	etcdEndpoints, etcdSecure, err := parseEndpoints(endpoints)
+	if err != nil {
+		return cfg, err
+	}
+
+	cfg.Enabled = true
+	cfg.DialTimeout = defaultDialTimeout
+	cfg.DialKeepAliveTime = defaultDialKeepAlive
+	// Disable etcd client SDK logging, etcd client
+	// incorrectly starts logging in unexpected data
+	// format.
+	cfg.LogConfig = &zap.Config{
+		Level:    zap.NewAtomicLevelAt(zap.FatalLevel),
+		Encoding: "console",
+	}
+	cfg.Endpoints = etcdEndpoints
+	cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath))
+	// Default path prefix for all keys on etcd, other than CoreDNSPath.
+	cfg.PathPrefix = env.Get(EnvEtcdPathPrefix, kvs.Get(PathPrefix))
+	if etcdSecure {
+		cfg.TLS = &tls.Config{
+			RootCAs: rootCAs,
+		}
+		// This is only to support client side certificate authentication
+		// https://coreos.com/etcd/docs/latest/op-guide/security.html
+		etcdClientCertFile := env.Get(EnvEtcdClientCert, kvs.Get(ClientCert))
+		etcdClientCertKey := env.Get(EnvEtcdClientCertKey, kvs.Get(ClientCertKey))
+		if etcdClientCertFile != "" && etcdClientCertKey != "" {
+			cfg.TLS.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+				cert, err := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
+				return &cert, err
+			}
+		}
+	}
+	return cfg, nil
+}

internal/config/etcd/etcd_test.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"reflect"
+	"testing"
+)
+
+// TestParseEndpoints - tests parseEndpoints function with valid and invalid inputs.
+func TestParseEndpoints(t *testing.T) {
+	testCases := []struct {
+		s         string
+		endpoints []string
+		secure    bool
+		success   bool
+	}{
+		// Invalid inputs
+		{"https://localhost:2379,http://localhost:2380", nil, false, false},
+		{",,,", nil, false, false},
+		{"", nil, false, false},
+		{"ftp://localhost:2379", nil, false, false},
+		{"http://localhost:2379000", nil, false, false},
+
+		// Valid inputs
+		{"https://localhost:2379,https://localhost:2380", []string{
+			"https://localhost:2379", "https://localhost:2380"},
+			true, true},
+		{"http://localhost:2379", []string{"http://localhost:2379"}, false, true},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.s, func(t *testing.T) {
+			endpoints, secure, err := parseEndpoints(testCase.s)
+			if err != nil && testCase.success {
+				t.Errorf("expected to succeed but failed with %s", err)
+			}
+			if !testCase.success && err == nil {
+				t.Error("expected failure but succeeded instead")
+			}
+			if testCase.success {
+				if !reflect.DeepEqual(endpoints, testCase.endpoints) {
+					t.Errorf("expected %s, got %s", testCase.endpoints, endpoints)
+				}
+				if secure != testCase.secure {
+					t.Errorf("expected %t, got %t", testCase.secure, secure)
+				}
+			}
+		})
+	}
+}

internal/config/etcd/help.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import "github.com/minio/minio/internal/config"
+
+// etcd config documented in default config
+var (
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         Endpoints,
+			Description: `comma separated list of etcd endpoints e.g. "http://localhost:2379"`,
+			Type:        "csv",
+		},
+		config.HelpKV{
+			Key:         PathPrefix,
+			Description: `namespace prefix to isolate tenants e.g. "customer1/"`,
+			Optional:    true,
+			Type:        "path",
+		},
+		config.HelpKV{
+			Key:         CoreDNSPath,
+			Description: `shared bucket DNS records, default is "/skydns"`,
+			Optional:    true,
+			Type:        "path",
+		},
+		config.HelpKV{
+			Key:         ClientCert,
+			Description: `client cert for mTLS authentication`,
+			Optional:    true,
+			Type:        "path",
+		},
+		config.HelpKV{
+			Key:         ClientCertKey,
+			Description: `client cert key for mTLS authentication`,
+			Optional:    true,
+			Type:        "path",
+		},
+		config.HelpKV{
+			Key:         config.Comment,
+			Description: config.DefaultComment,
+			Optional:    true,
+			Type:        "sentence",
+		},
+	}
+)