MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 21:49:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add etcd part of config support, add noColor/json support (#8439)

Browse Source 
- Add color/json mode support for get/help commands
- Support ENV help for all sub-systems
- Add support for etcd as part of config
This commit is contained in:
Harshavardhana
2019-10-30 00:04:39 -07:00
committed by kannappanr
parent 51456e6adc
commit 47b13cdb80
37 changed files with 704 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
cmd/admin-handlers-config-kv.go
View File

@@ -22,12 +22,10 @@ import (
"encoding/json"
"io"
"net/http"
"strings"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/color"
"github.com/minio/minio/pkg/madmin"
)
@@ -68,6 +66,9 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
oldCfg := cfg.Clone()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
for scanner.Scan() {
if scanner.Text() == "" {
continue
}
if err = cfg.DelKVS(scanner.Text()); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -121,20 +122,14 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
defaultKVS := configDefaultKVS()
oldCfg := cfg.Clone()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
var comment string
for scanner.Scan() {
if strings.HasPrefix(scanner.Text(), config.KvComment) {
// Join multiple comments for each newline, separated by ","
comments := []string{comment, strings.TrimPrefix(scanner.Text(), config.KvComment)}
comment = strings.Join(comments, config.KvNewline)
if scanner.Text() == "" {
continue
}
if err = cfg.SetKVS(scanner.Text(), comment, defaultKVS); err != nil {
if err = cfg.SetKVS(scanner.Text(), defaultKVS); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Empty the comment for the next sub-system
comment = ""
}
if err = scanner.Err(); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -169,39 +164,28 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
}
vars := mux.Vars(r)
var body strings.Builder
if vars["key"] != "" {
kvs, err := globalServerConfig.GetKVS(vars["key"])
var buf = &bytes.Buffer{}
key := vars["key"]
if key != "" {
kvs, err := globalServerConfig.GetKVS(key)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
for k, kv := range kvs {
c, ok := kv[config.Comment]
if ok {
// For multiple comments split it correctly.
for _, c1 := range strings.Split(c, config.KvNewline) {
if c1 == "" {
continue
}
body.WriteString(color.YellowBold(config.KvComment))
body.WriteString(config.KvSpaceSeparator)
body.WriteString(color.BlueBold(strings.TrimSpace(c1)))
body.WriteString(config.KvNewline)
}
}
body.WriteString(color.CyanBold(k))
body.WriteString(config.KvSpaceSeparator)
body.WriteString(kv.String())
buf.WriteString(k)
buf.WriteString(config.KvSpaceSeparator)
buf.WriteString(kv.String())
if len(kvs) > 1 {
body.WriteString(config.KvNewline)
buf.WriteString(config.KvNewline)
}
}
} else {
body.WriteString(globalServerConfig.String())
buf.WriteString(globalServerConfig.String())
}
password := globalActiveCred.SecretKey
econfigData, err := madmin.EncryptData(password, []byte(body.String()))
econfigData, err := madmin.EncryptData(password, buf.Bytes())
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -275,18 +259,14 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
defaultKVS := configDefaultKVS()
oldCfg := cfg.Clone()
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
var comment string
for scanner.Scan() {
if strings.HasPrefix(scanner.Text(), config.KvComment) {
// Join multiple comments for each newline, separated by "\n"
comment = strings.Join([]string{comment, scanner.Text()}, config.KvNewline)
if scanner.Text() == "" {
continue
}
if err = cfg.SetKVS(scanner.Text(), comment, defaultKVS); err != nil {
if err = cfg.SetKVS(scanner.Text(), defaultKVS); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
comment = ""
}
if err = scanner.Err(); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -340,20 +320,23 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
}
vars := mux.Vars(r)
subSys := vars["subSys"]
key := vars["key"]
rd, err := GetHelp(subSys, key)
_, envOnly := r.URL.Query()["env"]
rd, err := GetHelp(subSys, key, envOnly)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
io.Copy(w, rd)
json.NewEncoder(w).Encode(rd)
w.(http.Flusher).Flush()
}
// SetConfigHandler - PUT /minio/admin/v1/config
// SetConfigHandler - PUT /minio/admin/v2/config
func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "SetConfigHandler")
@@ -403,7 +386,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
writeSuccessResponseHeadersOnly(w)
}
// GetConfigHandler - GET /minio/admin/v1/config
// GetConfigHandler - GET /minio/admin/v2/config
// Get config.json of this minio setup.
func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "GetConfigHandler")

31
cmd/admin-handlers.go
View File

@@ -982,22 +982,23 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
if err == nil {
return noError
}
apiErr := errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
if apiErr.Code == "InternalError" {
switch e := err.(type) {
case config.Error:
apiErr = APIError{
Code: "XMinioConfigError",
Description: e.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
case AdminError:
apiErr = APIError{
Code: e.Code,
Description: e.Message,
HTTPStatusCode: e.StatusCode,
}
var apiErr APIError
switch e := err.(type) {
case config.Error:
apiErr = APIError{
Code: "XMinioConfigError",
Description: e.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
case AdminError:
apiErr = APIError{
Code: e.Code,
Description: e.Message,
HTTPStatusCode: e.StatusCode,
}
default:
apiErr = errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
}
return apiErr
}

2
cmd/bucket-handlers.go
View File

@@ -106,7 +106,7 @@ func initFederatorBackend(objLayer ObjectLayer) {
}
// This is not for our server, so we can continue
hostPort := net.JoinHostPort(dnsBuckets[index].Host, fmt.Sprintf("%d", dnsBuckets[index].Port))
hostPort := net.JoinHostPort(dnsBuckets[index].Host, dnsBuckets[index].Port)
if globalDomainIPs.Intersection(set.CreateStringSet(hostPort)).IsEmpty() {
return nil
}

19
cmd/common-main.go
View File

@@ -28,10 +28,8 @@ import (
"github.com/minio/cli"
"github.com/minio/minio-go/v6/pkg/set"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/etcd"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/certs"
"github.com/minio/minio/pkg/dns"
"github.com/minio/minio/pkg/env"
)
@@ -159,17 +157,12 @@ func handleCommonCmdArgs(ctx *cli.Context) {
func handleCommonEnvVars() {
var err error
globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, "on"))
globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.StateOn))
if err != nil {
logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
}
globalEtcdClient, err = etcd.New(globalRootCAs)
if err != nil {
logger.FatalIf(err, "Unable to initialize etcd config")
}
for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), ",") {
for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), config.ValueSeparator) {
if domainName != "" {
if _, ok := dns2.IsDomainName(domainName); !ok {
logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
@@ -181,7 +174,7 @@ func handleCommonEnvVars() {
minioEndpointsEnv, ok := env.Lookup(config.EnvPublicIPs)
if ok {
minioEndpoints := strings.Split(minioEndpointsEnv, ",")
minioEndpoints := strings.Split(minioEndpointsEnv, config.ValueSeparator)
var domainIPs = set.NewStringSet()
for _, endpoint := range minioEndpoints {
if net.ParseIP(endpoint) == nil {
@@ -204,12 +197,6 @@ func handleCommonEnvVars() {
updateDomainIPs(localIP4)
}
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
var err error
globalDNSConfig, err = dns.NewCoreDNS(globalDomainNames, globalDomainIPs, globalMinioPort, globalEtcdClient)
logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainNames)
}
// In place update is true by default if the MINIO_UPDATE is not set
// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
// in-place update is off.

85
cmd/config-current.go
View File

@@ -19,14 +19,13 @@ package cmd
import (
"context"
"fmt"
"io"
"strings"
"sync"
"text/tabwriter"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/cache"
"github.com/minio/minio/cmd/config/compress"
"github.com/minio/minio/cmd/config/etcd"
xldap "github.com/minio/minio/cmd/config/identity/ldap"
"github.com/minio/minio/cmd/config/identity/openid"
"github.com/minio/minio/cmd/config/notify"
@@ -36,6 +35,7 @@ import (
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/cmd/logger/target/http"
"github.com/minio/minio/pkg/dns"
"github.com/minio/minio/pkg/env"
)
@@ -61,6 +61,9 @@ func validateConfig(s config.Config) error {
return err
}
}
if _, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs); err != nil {
return err
}
if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
return err
}
@@ -92,14 +95,35 @@ func lookupConfigs(s config.Config) {
var err error
if !globalActiveCred.IsValid() {
// Env doesn't seem to be set, we fallback to lookup
// creds from the config.
// Env doesn't seem to be set, we fallback to lookup creds from the config.
globalActiveCred, err = config.LookupCreds(s[config.CredentialsSubSys][config.Default])
if err != nil {
logger.Fatal(err, "Invalid credentials configuration")
}
}
etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
if err != nil {
logger.Fatal(err, "Unable to initialize etcd config")
}
globalEtcdClient, err = etcd.New(etcdCfg)
if err != nil {
logger.Fatal(err, "Unable to initialize etcd config")
}
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
globalDNSConfig, err = dns.NewCoreDNS(globalEtcdClient,
dns.DomainNames(globalDomainNames),
dns.DomainIPs(globalDomainIPs),
dns.DomainPort(globalMinioPort),
dns.CoreDNSPath(etcdCfg.CoreDNSPath),
)
if err != nil {
logger.Fatal(err, "Unable to initialize DNS config for %s.", globalDomainNames)
}
}
globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
if err != nil {
logger.Fatal(err, "Invalid region configuration")
@@ -202,6 +226,7 @@ func lookupConfigs(s config.Config) {
var helpMap = map[string]config.HelpKV{
config.RegionSubSys: config.RegionHelp,
config.WormSubSys: config.WormHelp,
config.EtcdSubSys: etcd.Help,
config.CacheSubSys: cache.Help,
config.CompressionSubSys: compress.Help,
config.StorageClassSubSys: storageclass.Help,
@@ -224,29 +249,42 @@ var helpMap = map[string]config.HelpKV{
}
// GetHelp - returns help for sub-sys, a key for a sub-system or all the help.
func GetHelp(subSys, key string) (io.Reader, error) {
func GetHelp(subSys, key string, envOnly bool) (config.HelpKV, error) {
if len(subSys) == 0 {
return nil, config.Error("no help available for empty sub-system inputs")
}
help, ok := helpMap[subSys]
if !ok {
subSystemValue := strings.SplitN(subSys, config.SubSystemSeparator, 2)
if len(subSystemValue) == 0 {
return nil, config.Error(fmt.Sprintf("invalid number of arguments %s", subSys))
}
if !config.SubSystems.Contains(subSystemValue[0]) {
return nil, config.Error(fmt.Sprintf("unknown sub-system %s", subSys))
}
help := helpMap[subSystemValue[0]]
if key != "" {
value, ok := help[key]
if !ok {
return nil, config.Error(fmt.Sprintf("unknown key %s for sub-system %s", key, subSys))
}
return strings.NewReader(value), nil
help = config.HelpKV{
key: value,
}
}
var s strings.Builder
w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0)
if err := config.HelpTemplate.Execute(w, help); err != nil {
return nil, config.Error(err.Error())
envHelp := config.HelpKV{}
if envOnly {
for k, v := range help {
envK := config.EnvPrefix + strings.Join([]string{
strings.ToTitle(subSys), strings.ToTitle(k),
}, config.EnvWordDelimiter)
envHelp[envK] = v
}
help = envHelp
}
w.Flush()
return strings.NewReader(s.String()), nil
return help, nil
}
func configDefaultKVS() map[string]config.KVS {
@@ -262,6 +300,8 @@ func newServerConfig() config.Config {
for k := range srvCfg {
// Initialize with default KVS
switch k {
case config.EtcdSubSys:
srvCfg[k][config.Default] = etcd.DefaultKVS
case config.CacheSubSys:
srvCfg[k][config.Default] = cache.DefaultKVS
case config.CompressionSubSys:
@@ -312,22 +352,29 @@ func newSrvConfig(objAPI ObjectLayer) error {
return saveServerConfig(context.Background(), objAPI, globalServerConfig, nil)
}
// getValidConfig - returns valid server configuration
func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
srvCfg, err := readServerConfig(context.Background(), objAPI)
if err != nil {
return nil, err
}
defaultKVS := configDefaultKVS()
for _, k := range config.SubSystems.ToSlice() {
_, ok := srvCfg[k][config.Default]
if !ok {
// Populate default configs for any new
// sub-systems added automatically.
srvCfg[k][config.Default] = defaultKVS[k]
}
}
return srvCfg, nil
}
// loadConfig - loads a new config from disk, overrides params from env
// if found and valid
// loadConfig - loads a new config from disk, overrides params
// from env if found and valid
func loadConfig(objAPI ObjectLayer) error {
srvCfg, err := getValidConfig(objAPI)
if err != nil {
return config.ErrInvalidConfig(err)
return err
}
// Override any values from ENVs.

42
cmd/config-migrate.go
View File

@@ -2563,12 +2563,7 @@ func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
cfg.Version = "28"
cfg.KMS = crypto.KMSConfig{}
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
}
@@ -2595,12 +2590,7 @@ func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
}
cfg.Version = "29"
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
}
@@ -2632,12 +2622,7 @@ func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %v", err)
}
@@ -2672,12 +2657,7 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
AuthToken: "",
}
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %v", err)
}
@@ -2707,12 +2687,7 @@ func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
cfg.Notify.NSQ = make(map[string]target.NSQArgs)
cfg.Notify.NSQ["1"] = target.NSQArgs{}
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %v", err)
}
@@ -2740,12 +2715,7 @@ func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
cfg.Version = "33"
data, err = json.Marshal(cfg)
if err != nil {
return err
}
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
return fmt.Errorf("Failed to migrate config from 32 to 33 . %v", err)
}

20
cmd/config.go
View File

@@ -17,12 +17,10 @@
package cmd
import (
"bytes"
"context"
"encoding/json"
"fmt"
"path"
"runtime"
"sort"
"strings"
"time"
@@ -80,7 +78,11 @@ func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV stri
func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) {
historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
return readConfig(ctx, objAPI, historyFile)
data, err := readConfig(ctx, objAPI, historyFile)
if err != nil {
return nil, err
}
return data, err
}
func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
@@ -108,8 +110,12 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config interface{
if err != nil && err != errConfigNotFound {
return err
}
// Current config not found, so nothing to backup.
freshConfig = true
if err == errConfigNotFound {
// Current config not found, so nothing to backup.
freshConfig = true
}
// Do not need to decrypt oldData since we are going to
// save it anyway if freshConfig is false.
} else {
oldData, err = json.Marshal(oldConfig)
if err != nil {
@@ -135,10 +141,6 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
return nil, err
}
if runtime.GOOS == "windows" {
configData = bytes.Replace(configData, []byte("\r\n"), []byte("\n"), -1)
}
var config = config.New()
if err = json.Unmarshal(configData, &config); err != nil {
return nil, err

8
cmd/config/cache/lookup.go vendored
View File

@@ -75,11 +75,13 @@ func LookupConfig(kvs config.KVS) (Config, error) {
if err != nil {
return cfg, err
}
if !stateBool {
return cfg, nil
}
drives := env.Get(EnvCacheDrives, kvs.Get(Drives))
if stateBool {
if len(drives) == 0 {
return cfg, config.Error("'drives' key cannot be empty if you wish to enable caching")
}
}
if len(drives) == 0 {
return cfg, nil
}

4
cmd/config/compress/legacy.go
View File

@@ -38,7 +38,7 @@ func SetCompressionConfig(s config.Config, cfg Config) {
return config.StateOff
}(),
config.Comment: "Settings for Compression, after migrating config",
Extensions: strings.Join(cfg.Extensions, ","),
MimeTypes: strings.Join(cfg.MimeTypes, ","),
Extensions: strings.Join(cfg.Extensions, config.ValueSeparator),
MimeTypes: strings.Join(cfg.MimeTypes, config.ValueSeparator),
}
}

68
cmd/config/config.go
View File

@@ -23,7 +23,6 @@ import (
"github.com/minio/minio-go/pkg/set"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/color"
"github.com/minio/minio/pkg/env"
)
@@ -36,7 +35,7 @@ func (e Error) Error() string {
// Default keys
const (
Default = "_"
Default = `_`
State = "state"
Comment = "comment"
@@ -58,6 +57,7 @@ const (
WormSubSys = "worm"
CacheSubSys = "cache"
RegionSubSys = "region"
EtcdSubSys = "etcd"
StorageClassSubSys = "storageclass"
CompressionSubSys = "compression"
KmsVaultSubSys = "kms_vault"
@@ -88,6 +88,7 @@ var SubSystems = set.CreateStringSet([]string{
CredentialsSubSys,
WormSubSys,
RegionSubSys,
EtcdSubSys,
CacheSubSys,
StorageClassSubSys,
CompressionSubSys,
@@ -114,6 +115,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
CredentialsSubSys,
WormSubSys,
RegionSubSys,
EtcdSubSys,
CacheSubSys,
StorageClassSubSys,
CompressionSubSys,
@@ -132,6 +134,10 @@ const (
KvNewline = "\n"
KvDoubleQuote = `"`
KvSingleQuote = `'`
// Env prefix used for all envs in MinIO
EnvPrefix = "MINIO_"
EnvWordDelimiter = `_`
)
// KVS - is a shorthand for some wrapper functions
@@ -141,10 +147,6 @@ type KVS map[string]string
func (kvs KVS) String() string {
var s strings.Builder
for k, v := range kvs {
if k == Comment {
// Skip the comment, comment will be printed elsewhere.
continue
}
s.WriteString(k)
s.WriteString(KvSeparator)
s.WriteString(KvDoubleQuote)
@@ -167,20 +169,7 @@ func (c Config) String() string {
var s strings.Builder
for k, v := range c {
for target, kv := range v {
c, ok := kv[Comment]
if ok {
// For multiple comments split it correctly.
for _, c1 := range strings.Split(c, KvNewline) {
if c1 == "" {
continue
}
s.WriteString(color.YellowBold(KvComment))
s.WriteString(KvSpaceSeparator)
s.WriteString(color.BlueBold(strings.TrimSpace(c1)))
s.WriteString(KvNewline)
}
}
s.WriteString(color.CyanBold(k))
s.WriteString(k)
if target != Default {
s.WriteString(SubSystemSeparator)
s.WriteString(target)
@@ -307,7 +296,7 @@ func (c Config) GetKVS(s string) (map[string]KVS, error) {
}
kvs[inputs[0]], ok = c[subSystemValue[0]][subSystemValue[1]]
if !ok {
err := fmt.Sprintf("sub-system target '%s' doesn't exist, proceed to create a new one", s)
err := fmt.Sprintf("sub-system target '%s' doesn't exist", s)
return nil, Error(err)
}
return kvs, nil
@@ -377,7 +366,7 @@ func (c Config) Clone() Config {
}
// SetKVS - set specific key values per sub-system.
func (c Config) SetKVS(s string, comment string, defaultKVS map[string]KVS) error {
func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
if len(s) == 0 {
return Error("input arguments cannot be empty")
}
@@ -422,27 +411,18 @@ func (c Config) SetKVS(s string, comment string, defaultKVS map[string]KVS) erro
kvs[kv[0]] = sanitizeValue(kv[1])
}
tgt := Default
if len(subSystemValue) == 2 {
_, ok := c[subSystemValue[0]][subSystemValue[1]]
if !ok {
c[subSystemValue[0]][subSystemValue[1]] = defaultKVS[subSystemValue[0]]
// Add a comment since its a new target, this comment may be
// overridden if client supplied it.
if comment == "" {
comment = fmt.Sprintf("Settings for sub-system target %s:%s",
subSystemValue[0], subSystemValue[1])
}
c[subSystemValue[0]][subSystemValue[1]][Comment] = comment
}
tgt = subSystemValue[1]
}
_, ok := c[subSystemValue[0]][tgt]
if !ok {
c[subSystemValue[0]][tgt] = defaultKVS[subSystemValue[0]]
comment := fmt.Sprintf("Settings for sub-system target %s:%s", subSystemValue[0], tgt)
c[subSystemValue[0]][tgt][Comment] = comment
}
var commentKv bool
for k, v := range kvs {
if k == Comment {
// Set this to true to indicate comment was
// supplied by client and is going to be preserved.
commentKv = true
}
if len(subSystemValue) == 2 {
c[subSystemValue[0]][subSystemValue[1]][k] = v
} else {
@@ -450,15 +430,5 @@ func (c Config) SetKVS(s string, comment string, defaultKVS map[string]KVS) erro
}
}
// if client didn't supply the comment try to preserve
// the comment if any we found while parsing the incoming
// stream, if not preserve the default.
if !commentKv && comment != "" {
if len(subSystemValue) == 2 {
c[subSystemValue[0]][subSystemValue[1]][Comment] = comment
} else {
c[subSystemValue[0]][Default][Comment] = comment
}
}
return nil
}

6
cmd/config/errors.go
View File

@@ -18,12 +18,6 @@ package config
// UI errors
var (
ErrInvalidConfig = newErrFn(
"Invalid value found in the configuration file",
"Please ensure a valid value in the configuration file",
"For more details, refer to https://docs.min.io/docs/minio-server-configuration-guide",
)
ErrInvalidBrowserValue = newErrFn(
"Invalid browser value",
"Please check the passed value",

105
cmd/config/etcd/etcd.go
View File

@@ -36,61 +36,100 @@ const (
// etcd environment values
const (
Endpoints = "endpoints"
CoreDNSPath = "coredns_path"
ClientCert = "client_cert"
ClientCertKey = "client_cert_key"
EnvEtcdState = "MINIO_ETCD_STATE"
EnvEtcdEndpoints = "MINIO_ETCD_ENDPOINTS"
EnvEtcdCoreDNSPath = "MINIO_ETCD_COREDNS_PATH"
EnvEtcdClientCert = "MINIO_ETCD_CLIENT_CERT"
EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
)
// New - Initialize new etcd client
func New(rootCAs *x509.CertPool) (*clientv3.Client, error) {
envEndpoints := env.Get(EnvEtcdEndpoints, "")
if envEndpoints == "" {
// etcd is not configured, nothing to do.
// DefaultKVS - default KV settings for etcd.
var (
DefaultKVS = config.KVS{
config.State: config.StateOff,
config.Comment: "This is a default etcd configuration",
Endpoints: "",
CoreDNSPath: "/skydns",
ClientCert: "",
ClientCertKey: "",
}
)
// Config - server etcd config.
type Config struct {
Enabled bool `json:"enabled"`
CoreDNSPath string `json:"coreDNSPath"`
clientv3.Config
}
// New - initialize new etcd client.
func New(cfg Config) (*clientv3.Client, error) {
if !cfg.Enabled {
return nil, nil
}
return clientv3.New(cfg.Config)
}
etcdEndpoints := strings.Split(envEndpoints, config.ValueSeparator)
// LookupConfig - Initialize new etcd config.
func LookupConfig(kv config.KVS, rootCAs *x509.CertPool) (Config, error) {
cfg := Config{}
if err := config.CheckValidKeys(config.EtcdSubSys, kv, DefaultKVS); err != nil {
return cfg, err
}
stateBool, err := config.ParseBool(env.Get(EnvEtcdState, kv.Get(config.State)))
if err != nil {
return cfg, err
}
endpoints := env.Get(EnvEtcdEndpoints, kv.Get(Endpoints))
if stateBool && len(endpoints) == 0 {
return cfg, config.Error("'endpoints' key cannot be empty if you wish to enable etcd")
}
if len(endpoints) == 0 {
return cfg, nil
}
cfg.Enabled = true
etcdEndpoints := strings.Split(endpoints, config.ValueSeparator)
var etcdSecure bool
for _, endpoint := range etcdEndpoints {
if endpoint == "" {
continue
}
u, err := xnet.ParseURL(endpoint)
if err != nil {
return nil, err
return cfg, err
}
// If one of the endpoint is https, we will use https directly.
etcdSecure = etcdSecure || u.Scheme == "https"
}
var err error
var etcdClnt *clientv3.Client
cfg.DialTimeout = defaultDialTimeout
cfg.DialKeepAliveTime = defaultDialKeepAlive
cfg.Endpoints = etcdEndpoints
cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kv.Get(CoreDNSPath))
if etcdSecure {
cfg.TLS = &tls.Config{
RootCAs: rootCAs,
}
// This is only to support client side certificate authentication
// https://coreos.com/etcd/docs/latest/op-guide/security.html
etcdClientCertFile, ok1 := env.Lookup(EnvEtcdClientCert)
etcdClientCertKey, ok2 := env.Lookup(EnvEtcdClientCertKey)
var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
if ok1 && ok2 {
getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
return &cert, terr
etcdClientCertFile := env.Get(EnvEtcdClientCert, kv.Get(ClientCert))
etcdClientCertKey := env.Get(EnvEtcdClientCertKey, kv.Get(ClientCertKey))
if etcdClientCertFile != "" && etcdClientCertKey != "" {
cfg.TLS.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert, err := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
return &cert, err
}
}
etcdClnt, err = clientv3.New(clientv3.Config{
Endpoints: etcdEndpoints,
DialTimeout: defaultDialTimeout,
DialKeepAliveTime: defaultDialKeepAlive,
TLS: &tls.Config{
RootCAs: rootCAs,
GetClientCertificate: getClientCertificate,
},
})
} else {
etcdClnt, err = clientv3.New(clientv3.Config{
Endpoints: etcdEndpoints,
DialTimeout: defaultDialTimeout,
DialKeepAliveTime: defaultDialKeepAlive,
})
}
return etcdClnt, err
return cfg, nil
}

31
cmd/config/etcd/help.go Normal file
View File

@@ -0,0 +1,31 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package etcd
import "github.com/minio/minio/cmd/config"
// etcd config documented in default config
var (
Help = config.HelpKV{
Endpoints: `(required) Comma separated list of etcd endpoints eg: "http://localhost:2379"`,
CoreDNSPath: `(optional) CoreDNS etcd path location to populate DNS srv records eg: "/skydns"`,
ClientCert: `(optional) Etcd client cert for mTLS authentication`,
ClientCertKey: `(optional) Etcd client cert key for mTLS authentication`,
config.State: "Indicates if etcd config is on or off",
config.Comment: "A comment to describe the etcd settings",
}
)

22
cmd/config/help.go
View File

@@ -16,32 +16,10 @@
package config
import (
"text/template"
"github.com/minio/minio/pkg/color"
)
// HelpKV - implements help messages for keys
// with value as description of the keys.
type HelpKV map[string]string
// Help template used by all sub-systems
const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}}
{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
{{end}}`
var funcMap = template.FuncMap{
"colorBlueBold": color.BlueBold,
"colorYellowBold": color.YellowBold,
"colorCyanBold": color.CyanBold,
"colorGreenBold": color.GreenBold,
}
// HelpTemplate - captures config help template
var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help))
// Region and Worm help is documented in default config
var (
RegionHelp = HelpKV{

8
cmd/config/identity/ldap/config.go
View File

@@ -117,10 +117,12 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
if err != nil {
return l, err
}
if !stateBool {
return l, nil
}
ldapServer := env.Get(EnvServerAddr, kvs.Get(ServerAddr))
if stateBool {
if ldapServer == "" {
return l, config.Error("'serveraddr' cannot be empty if you wish to enable AD/LDAP support")
}
}
if ldapServer == "" {
return l, nil
}

43
cmd/config/identity/openid/jwt.go
View File

@@ -207,6 +207,7 @@ const (
ConfigURL = "config_url"
ClaimPrefix = "claim_prefix"
EnvIdentityOpenIDState = "MINIO_IDENTITY_OPENID_STATE"
EnvIdentityOpenIDJWKSURL = "MINIO_IDENTITY_OPENID_JWKS_URL"
EnvIdentityOpenIDURL = "MINIO_IDENTITY_OPENID_CONFIG_URL"
EnvIdentityOpenIDClaimPrefix = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
@@ -271,20 +272,10 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.
return c, err
}
stateBool, err := config.ParseBool(kv.Get(config.State))
stateBool, err := config.ParseBool(env.Get(EnvIdentityOpenIDState, kv.Get(config.State)))
if err != nil {
return c, err
}
if !stateBool {
return c, nil
}
c = Config{
ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kv.Get(ClaimPrefix)),
publicKeys: make(map[string]crypto.PublicKey),
transport: transport,
closeRespFn: closeRespFn,
}
jwksURL := env.Get(EnvIamJwksURL, "") // Legacy
if jwksURL == "" {
@@ -306,15 +297,31 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.
// Fallback to discovery document jwksURL
jwksURL = c.DiscoveryDoc.JwksURI
}
if jwksURL != "" {
c.JWKS.URL, err = xnet.ParseURL(jwksURL)
if err != nil {
return c, err
}
if err = c.PopulatePublicKey(); err != nil {
return c, err
if stateBool {
// This check is needed to ensure that empty Jwks urls are not allowed.
if jwksURL == "" {
return c, config.Error("'config_url' must be set to a proper OpenID discovery document URL")
}
}
if jwksURL == "" {
return c, nil
}
c = Config{
ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kv.Get(ClaimPrefix)),
publicKeys: make(map[string]crypto.PublicKey),
transport: transport,
closeRespFn: closeRespFn,
}
c.JWKS.URL, err = xnet.ParseURL(jwksURL)
if err != nil {
return c, err
}
if err = c.PopulatePublicKey(); err != nil {
return c, err
}
return c, nil
}

2
cmd/config/notify/legacy.go
View File

@@ -27,7 +27,7 @@ func SetNotifyKafka(s config.Config, kName string, cfg target.KafkaArgs) error {
for _, broker := range cfg.Brokers {
brokers = append(brokers, broker.String())
}
return strings.Join(brokers, ",")
return strings.Join(brokers, config.ValueSeparator)
}(),
config.Comment: "Settings for Kafka notification, after migrating config",
target.KafkaTopic: cfg.Topic,

9
cmd/config/storageclass/storage-class.go
View File

@@ -224,8 +224,13 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
if err != nil {
return cfg, err
}
if !stateBool {
return cfg, nil
if stateBool {
if ssc := env.Get(StandardEnv, kvs.Get(ClassStandard)); ssc == "" {
return cfg, config.Error("'standard' key cannot be empty if you wish to enable storage class")
}
if rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS)); rrsc == "" {
return cfg, config.Error("'rrs' key cannot be empty if you wish to enable storage class")
}
}
// Check for environment variables and parse into storageClass struct

5
cmd/crypto/legacy.go
View File

@@ -122,7 +122,10 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) {
cfg := KMSConfig{
AutoEncryption: autoBool,
}
stateBool, err := config.ParseBool(kvs.Get(config.State))
// Assume default as "on" for legacy config since we didn't have a _STATE
// flag to turn it off, but we should honor it nonetheless to turn it off
// if the vault endpoint is down and there is no way to start the server.
stateBool, err := config.ParseBool(env.Get(EnvKMSVaultState, config.StateOn))
if err != nil {
return cfg, err
}

6
cmd/crypto/vault.go
View File

@@ -69,7 +69,11 @@ type vaultService struct {
var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
// empty/default vault configuration used to check whether a particular is empty.
var emptyVaultConfig = VaultConfig{}
var emptyVaultConfig = VaultConfig{
Auth: VaultAuth{
Type: "approle",
},
}
// IsEmpty returns true if the vault config struct is an
// empty configuration.

2
cmd/crypto/vault_test.go
View File

@@ -25,7 +25,7 @@ var verifyVaultConfigTests = []struct {
}{
{
ShouldFail: false, // 0
Config: VaultConfig{},
Config: emptyVaultConfig,
},
{
ShouldFail: true,

3
cmd/globals.go
View File

@@ -184,7 +184,8 @@ var (
// Time when object layer was initialized on start up.
globalBootTime time.Time
globalActiveCred auth.Credentials
globalActiveCred auth.Credentials
globalPublicCerts []*x509.Certificate
globalDomainNames []string // Root domains for virtual host style requests

1
cmd/iam-etcd-store.go
View File

@@ -117,6 +117,7 @@ func (ies *IAMEtcdStore) loadIAMConfig(item interface{}, path string) error {
if err != nil {
return err
}
return json.Unmarshal(pdata, item)
}

4
cmd/object-api-utils.go
View File

@@ -328,7 +328,7 @@ func isMinioReservedBucket(bucketName string) bool {
func getHostsSlice(records []dns.SrvRecord) []string {
var hosts []string
for _, r := range records {
hosts = append(hosts, net.JoinHostPort(r.Host, fmt.Sprintf("%d", r.Port)))
hosts = append(hosts, net.JoinHostPort(r.Host, r.Port))
}
return hosts
}
@@ -337,7 +337,7 @@ func getHostsSlice(records []dns.SrvRecord) []string {
func getHostFromSrv(records []dns.SrvRecord) string {
rand.Seed(time.Now().Unix())
srvRecord := records[rand.Intn(len(records))]
return net.JoinHostPort(srvRecord.Host, fmt.Sprintf("%d", srvRecord.Port))
return net.JoinHostPort(srvRecord.Host, srvRecord.Port)
}
// IsCompressed returns true if the object is marked as compressed.

4
cmd/prepare-storage.go
View File

@@ -20,6 +20,7 @@ import (
"context"
"fmt"
"os"
"sync"
"time"
"github.com/minio/minio/cmd/logger"
@@ -27,11 +28,14 @@ import (
)
var printEndpointError = func() func(Endpoint, error) {
var mutex sync.Mutex
printOnce := make(map[Endpoint]map[string]bool)
return func(endpoint Endpoint, err error) {
reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.String())
ctx := logger.SetReqInfo(context.Background(), reqInfo)
mutex.Lock()
defer mutex.Unlock()
m, ok := printOnce[endpoint]
if !ok {
m = make(map[string]bool)

1
cmd/server-main.go
View File

@@ -192,6 +192,7 @@ func serverHandleEnvVars() {
}
globalActiveCred = cred
}
}
// serverMain handler called for 'minio server' command.

2
cmd/storage-rest-common.go
View File

@@ -1,5 +1,5 @@
/*
* MinIO Cloud Storage, (C) 2018 MinIO, Inc.
* MinIO Cloud Storage, (C) 2018-2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.