mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Add etcd part of config support, add noColor/json support (#8439)
- Add color/json mode support for get/help commands - Support ENV help for all sub-systems - Add support for etcd as part of config
This commit is contained in:
parent
51456e6adc
commit
47b13cdb80
@ -22,12 +22,10 @@ import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/color"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
@ -68,6 +66,9 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
oldCfg := cfg.Clone()
|
||||
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
|
||||
for scanner.Scan() {
|
||||
if scanner.Text() == "" {
|
||||
continue
|
||||
}
|
||||
if err = cfg.DelKVS(scanner.Text()); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@ -121,20 +122,14 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
defaultKVS := configDefaultKVS()
|
||||
oldCfg := cfg.Clone()
|
||||
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
|
||||
var comment string
|
||||
for scanner.Scan() {
|
||||
if strings.HasPrefix(scanner.Text(), config.KvComment) {
|
||||
// Join multiple comments for each newline, separated by ","
|
||||
comments := []string{comment, strings.TrimPrefix(scanner.Text(), config.KvComment)}
|
||||
comment = strings.Join(comments, config.KvNewline)
|
||||
if scanner.Text() == "" {
|
||||
continue
|
||||
}
|
||||
if err = cfg.SetKVS(scanner.Text(), comment, defaultKVS); err != nil {
|
||||
if err = cfg.SetKVS(scanner.Text(), defaultKVS); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Empty the comment for the next sub-system
|
||||
comment = ""
|
||||
}
|
||||
if err = scanner.Err(); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@ -169,39 +164,28 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
var body strings.Builder
|
||||
if vars["key"] != "" {
|
||||
kvs, err := globalServerConfig.GetKVS(vars["key"])
|
||||
var buf = &bytes.Buffer{}
|
||||
key := vars["key"]
|
||||
if key != "" {
|
||||
kvs, err := globalServerConfig.GetKVS(key)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
for k, kv := range kvs {
|
||||
c, ok := kv[config.Comment]
|
||||
if ok {
|
||||
// For multiple comments split it correctly.
|
||||
for _, c1 := range strings.Split(c, config.KvNewline) {
|
||||
if c1 == "" {
|
||||
continue
|
||||
}
|
||||
body.WriteString(color.YellowBold(config.KvComment))
|
||||
body.WriteString(config.KvSpaceSeparator)
|
||||
body.WriteString(color.BlueBold(strings.TrimSpace(c1)))
|
||||
body.WriteString(config.KvNewline)
|
||||
}
|
||||
}
|
||||
body.WriteString(color.CyanBold(k))
|
||||
body.WriteString(config.KvSpaceSeparator)
|
||||
body.WriteString(kv.String())
|
||||
buf.WriteString(k)
|
||||
buf.WriteString(config.KvSpaceSeparator)
|
||||
buf.WriteString(kv.String())
|
||||
if len(kvs) > 1 {
|
||||
body.WriteString(config.KvNewline)
|
||||
buf.WriteString(config.KvNewline)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
body.WriteString(globalServerConfig.String())
|
||||
buf.WriteString(globalServerConfig.String())
|
||||
}
|
||||
|
||||
password := globalActiveCred.SecretKey
|
||||
econfigData, err := madmin.EncryptData(password, []byte(body.String()))
|
||||
econfigData, err := madmin.EncryptData(password, buf.Bytes())
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@ -275,18 +259,14 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
|
||||
defaultKVS := configDefaultKVS()
|
||||
oldCfg := cfg.Clone()
|
||||
scanner := bufio.NewScanner(bytes.NewReader(kvBytes))
|
||||
var comment string
|
||||
for scanner.Scan() {
|
||||
if strings.HasPrefix(scanner.Text(), config.KvComment) {
|
||||
// Join multiple comments for each newline, separated by "\n"
|
||||
comment = strings.Join([]string{comment, scanner.Text()}, config.KvNewline)
|
||||
if scanner.Text() == "" {
|
||||
continue
|
||||
}
|
||||
if err = cfg.SetKVS(scanner.Text(), comment, defaultKVS); err != nil {
|
||||
if err = cfg.SetKVS(scanner.Text(), defaultKVS); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
comment = ""
|
||||
}
|
||||
if err = scanner.Err(); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@ -340,20 +320,23 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
|
||||
subSys := vars["subSys"]
|
||||
key := vars["key"]
|
||||
|
||||
rd, err := GetHelp(subSys, key)
|
||||
_, envOnly := r.URL.Query()["env"]
|
||||
|
||||
rd, err := GetHelp(subSys, key, envOnly)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
io.Copy(w, rd)
|
||||
json.NewEncoder(w).Encode(rd)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
// SetConfigHandler - PUT /minio/admin/v1/config
|
||||
// SetConfigHandler - PUT /minio/admin/v2/config
|
||||
func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "SetConfigHandler")
|
||||
|
||||
@ -403,7 +386,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetConfigHandler - GET /minio/admin/v1/config
|
||||
// GetConfigHandler - GET /minio/admin/v2/config
|
||||
// Get config.json of this minio setup.
|
||||
func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetConfigHandler")
|
||||
|
@ -982,22 +982,23 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
||||
if err == nil {
|
||||
return noError
|
||||
}
|
||||
apiErr := errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
|
||||
if apiErr.Code == "InternalError" {
|
||||
switch e := err.(type) {
|
||||
case config.Error:
|
||||
apiErr = APIError{
|
||||
Code: "XMinioConfigError",
|
||||
Description: e.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case AdminError:
|
||||
apiErr = APIError{
|
||||
Code: e.Code,
|
||||
Description: e.Message,
|
||||
HTTPStatusCode: e.StatusCode,
|
||||
}
|
||||
|
||||
var apiErr APIError
|
||||
switch e := err.(type) {
|
||||
case config.Error:
|
||||
apiErr = APIError{
|
||||
Code: "XMinioConfigError",
|
||||
Description: e.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
case AdminError:
|
||||
apiErr = APIError{
|
||||
Code: e.Code,
|
||||
Description: e.Message,
|
||||
HTTPStatusCode: e.StatusCode,
|
||||
}
|
||||
default:
|
||||
apiErr = errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
|
@ -106,7 +106,7 @@ func initFederatorBackend(objLayer ObjectLayer) {
|
||||
}
|
||||
|
||||
// This is not for our server, so we can continue
|
||||
hostPort := net.JoinHostPort(dnsBuckets[index].Host, fmt.Sprintf("%d", dnsBuckets[index].Port))
|
||||
hostPort := net.JoinHostPort(dnsBuckets[index].Host, dnsBuckets[index].Port)
|
||||
if globalDomainIPs.Intersection(set.CreateStringSet(hostPort)).IsEmpty() {
|
||||
return nil
|
||||
}
|
||||
|
@ -28,10 +28,8 @@ import (
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/config/etcd"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/certs"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
)
|
||||
|
||||
@ -159,17 +157,12 @@ func handleCommonCmdArgs(ctx *cli.Context) {
|
||||
|
||||
func handleCommonEnvVars() {
|
||||
var err error
|
||||
globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, "on"))
|
||||
globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.StateOn))
|
||||
if err != nil {
|
||||
logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
|
||||
}
|
||||
|
||||
globalEtcdClient, err = etcd.New(globalRootCAs)
|
||||
if err != nil {
|
||||
logger.FatalIf(err, "Unable to initialize etcd config")
|
||||
}
|
||||
|
||||
for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), ",") {
|
||||
for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), config.ValueSeparator) {
|
||||
if domainName != "" {
|
||||
if _, ok := dns2.IsDomainName(domainName); !ok {
|
||||
logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
|
||||
@ -181,7 +174,7 @@ func handleCommonEnvVars() {
|
||||
|
||||
minioEndpointsEnv, ok := env.Lookup(config.EnvPublicIPs)
|
||||
if ok {
|
||||
minioEndpoints := strings.Split(minioEndpointsEnv, ",")
|
||||
minioEndpoints := strings.Split(minioEndpointsEnv, config.ValueSeparator)
|
||||
var domainIPs = set.NewStringSet()
|
||||
for _, endpoint := range minioEndpoints {
|
||||
if net.ParseIP(endpoint) == nil {
|
||||
@ -204,12 +197,6 @@ func handleCommonEnvVars() {
|
||||
updateDomainIPs(localIP4)
|
||||
}
|
||||
|
||||
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
|
||||
var err error
|
||||
globalDNSConfig, err = dns.NewCoreDNS(globalDomainNames, globalDomainIPs, globalMinioPort, globalEtcdClient)
|
||||
logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainNames)
|
||||
}
|
||||
|
||||
// In place update is true by default if the MINIO_UPDATE is not set
|
||||
// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
|
||||
// in-place update is off.
|
||||
|
@ -19,14 +19,13 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/config/cache"
|
||||
"github.com/minio/minio/cmd/config/compress"
|
||||
"github.com/minio/minio/cmd/config/etcd"
|
||||
xldap "github.com/minio/minio/cmd/config/identity/ldap"
|
||||
"github.com/minio/minio/cmd/config/identity/openid"
|
||||
"github.com/minio/minio/cmd/config/notify"
|
||||
@ -36,6 +35,7 @@ import (
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/logger/target/http"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
)
|
||||
|
||||
@ -61,6 +61,9 @@ func validateConfig(s config.Config) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -92,14 +95,35 @@ func lookupConfigs(s config.Config) {
|
||||
var err error
|
||||
|
||||
if !globalActiveCred.IsValid() {
|
||||
// Env doesn't seem to be set, we fallback to lookup
|
||||
// creds from the config.
|
||||
// Env doesn't seem to be set, we fallback to lookup creds from the config.
|
||||
globalActiveCred, err = config.LookupCreds(s[config.CredentialsSubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Invalid credentials configuration")
|
||||
}
|
||||
}
|
||||
|
||||
etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize etcd config")
|
||||
}
|
||||
|
||||
globalEtcdClient, err = etcd.New(etcdCfg)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize etcd config")
|
||||
}
|
||||
|
||||
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
|
||||
globalDNSConfig, err = dns.NewCoreDNS(globalEtcdClient,
|
||||
dns.DomainNames(globalDomainNames),
|
||||
dns.DomainIPs(globalDomainIPs),
|
||||
dns.DomainPort(globalMinioPort),
|
||||
dns.CoreDNSPath(etcdCfg.CoreDNSPath),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize DNS config for %s.", globalDomainNames)
|
||||
}
|
||||
}
|
||||
|
||||
globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Invalid region configuration")
|
||||
@ -202,6 +226,7 @@ func lookupConfigs(s config.Config) {
|
||||
var helpMap = map[string]config.HelpKV{
|
||||
config.RegionSubSys: config.RegionHelp,
|
||||
config.WormSubSys: config.WormHelp,
|
||||
config.EtcdSubSys: etcd.Help,
|
||||
config.CacheSubSys: cache.Help,
|
||||
config.CompressionSubSys: compress.Help,
|
||||
config.StorageClassSubSys: storageclass.Help,
|
||||
@ -224,29 +249,42 @@ var helpMap = map[string]config.HelpKV{
|
||||
}
|
||||
|
||||
// GetHelp - returns help for sub-sys, a key for a sub-system or all the help.
|
||||
func GetHelp(subSys, key string) (io.Reader, error) {
|
||||
func GetHelp(subSys, key string, envOnly bool) (config.HelpKV, error) {
|
||||
if len(subSys) == 0 {
|
||||
return nil, config.Error("no help available for empty sub-system inputs")
|
||||
}
|
||||
help, ok := helpMap[subSys]
|
||||
if !ok {
|
||||
subSystemValue := strings.SplitN(subSys, config.SubSystemSeparator, 2)
|
||||
if len(subSystemValue) == 0 {
|
||||
return nil, config.Error(fmt.Sprintf("invalid number of arguments %s", subSys))
|
||||
}
|
||||
|
||||
if !config.SubSystems.Contains(subSystemValue[0]) {
|
||||
return nil, config.Error(fmt.Sprintf("unknown sub-system %s", subSys))
|
||||
}
|
||||
|
||||
help := helpMap[subSystemValue[0]]
|
||||
if key != "" {
|
||||
value, ok := help[key]
|
||||
if !ok {
|
||||
return nil, config.Error(fmt.Sprintf("unknown key %s for sub-system %s", key, subSys))
|
||||
}
|
||||
return strings.NewReader(value), nil
|
||||
help = config.HelpKV{
|
||||
key: value,
|
||||
}
|
||||
}
|
||||
|
||||
var s strings.Builder
|
||||
w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0)
|
||||
if err := config.HelpTemplate.Execute(w, help); err != nil {
|
||||
return nil, config.Error(err.Error())
|
||||
envHelp := config.HelpKV{}
|
||||
if envOnly {
|
||||
for k, v := range help {
|
||||
envK := config.EnvPrefix + strings.Join([]string{
|
||||
strings.ToTitle(subSys), strings.ToTitle(k),
|
||||
}, config.EnvWordDelimiter)
|
||||
envHelp[envK] = v
|
||||
}
|
||||
help = envHelp
|
||||
}
|
||||
w.Flush()
|
||||
return strings.NewReader(s.String()), nil
|
||||
|
||||
return help, nil
|
||||
}
|
||||
|
||||
func configDefaultKVS() map[string]config.KVS {
|
||||
@ -262,6 +300,8 @@ func newServerConfig() config.Config {
|
||||
for k := range srvCfg {
|
||||
// Initialize with default KVS
|
||||
switch k {
|
||||
case config.EtcdSubSys:
|
||||
srvCfg[k][config.Default] = etcd.DefaultKVS
|
||||
case config.CacheSubSys:
|
||||
srvCfg[k][config.Default] = cache.DefaultKVS
|
||||
case config.CompressionSubSys:
|
||||
@ -312,22 +352,29 @@ func newSrvConfig(objAPI ObjectLayer) error {
|
||||
return saveServerConfig(context.Background(), objAPI, globalServerConfig, nil)
|
||||
}
|
||||
|
||||
// getValidConfig - returns valid server configuration
|
||||
func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
|
||||
srvCfg, err := readServerConfig(context.Background(), objAPI)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defaultKVS := configDefaultKVS()
|
||||
for _, k := range config.SubSystems.ToSlice() {
|
||||
_, ok := srvCfg[k][config.Default]
|
||||
if !ok {
|
||||
// Populate default configs for any new
|
||||
// sub-systems added automatically.
|
||||
srvCfg[k][config.Default] = defaultKVS[k]
|
||||
}
|
||||
}
|
||||
return srvCfg, nil
|
||||
}
|
||||
|
||||
// loadConfig - loads a new config from disk, overrides params from env
|
||||
// if found and valid
|
||||
// loadConfig - loads a new config from disk, overrides params
|
||||
// from env if found and valid
|
||||
func loadConfig(objAPI ObjectLayer) error {
|
||||
srvCfg, err := getValidConfig(objAPI)
|
||||
if err != nil {
|
||||
return config.ErrInvalidConfig(err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Override any values from ENVs.
|
||||
|
@ -2563,12 +2563,7 @@ func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
|
||||
cfg.Version = "28"
|
||||
cfg.KMS = crypto.KMSConfig{}
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
|
||||
}
|
||||
|
||||
@ -2595,12 +2590,7 @@ func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
|
||||
}
|
||||
|
||||
cfg.Version = "29"
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
|
||||
}
|
||||
|
||||
@ -2632,12 +2622,7 @@ func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
|
||||
cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
|
||||
cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %v", err)
|
||||
}
|
||||
|
||||
@ -2672,12 +2657,7 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
|
||||
AuthToken: "",
|
||||
}
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %v", err)
|
||||
}
|
||||
|
||||
@ -2707,12 +2687,7 @@ func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
|
||||
cfg.Notify.NSQ = make(map[string]target.NSQArgs)
|
||||
cfg.Notify.NSQ["1"] = target.NSQArgs{}
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %v", err)
|
||||
}
|
||||
|
||||
@ -2740,12 +2715,7 @@ func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
|
||||
|
||||
cfg.Version = "33"
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
if err = saveServerConfig(context.Background(), objAPI, cfg, nil); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from 32 to 33 . %v", err)
|
||||
}
|
||||
|
||||
|
@ -17,12 +17,10 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@ -80,7 +78,11 @@ func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV stri
|
||||
|
||||
func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) {
|
||||
historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
|
||||
return readConfig(ctx, objAPI, historyFile)
|
||||
data, err := readConfig(ctx, objAPI, historyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
|
||||
func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
|
||||
@ -108,8 +110,12 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config interface{
|
||||
if err != nil && err != errConfigNotFound {
|
||||
return err
|
||||
}
|
||||
// Current config not found, so nothing to backup.
|
||||
freshConfig = true
|
||||
if err == errConfigNotFound {
|
||||
// Current config not found, so nothing to backup.
|
||||
freshConfig = true
|
||||
}
|
||||
// Do not need to decrypt oldData since we are going to
|
||||
// save it anyway if freshConfig is false.
|
||||
} else {
|
||||
oldData, err = json.Marshal(oldConfig)
|
||||
if err != nil {
|
||||
@ -135,10 +141,6 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
configData = bytes.Replace(configData, []byte("\r\n"), []byte("\n"), -1)
|
||||
}
|
||||
|
||||
var config = config.New()
|
||||
if err = json.Unmarshal(configData, &config); err != nil {
|
||||
return nil, err
|
||||
|
8
cmd/config/cache/lookup.go
vendored
8
cmd/config/cache/lookup.go
vendored
@ -75,11 +75,13 @@ func LookupConfig(kvs config.KVS) (Config, error) {
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
if !stateBool {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
drives := env.Get(EnvCacheDrives, kvs.Get(Drives))
|
||||
if stateBool {
|
||||
if len(drives) == 0 {
|
||||
return cfg, config.Error("'drives' key cannot be empty if you wish to enable caching")
|
||||
}
|
||||
}
|
||||
if len(drives) == 0 {
|
||||
return cfg, nil
|
||||
}
|
||||
|
@ -38,7 +38,7 @@ func SetCompressionConfig(s config.Config, cfg Config) {
|
||||
return config.StateOff
|
||||
}(),
|
||||
config.Comment: "Settings for Compression, after migrating config",
|
||||
Extensions: strings.Join(cfg.Extensions, ","),
|
||||
MimeTypes: strings.Join(cfg.MimeTypes, ","),
|
||||
Extensions: strings.Join(cfg.Extensions, config.ValueSeparator),
|
||||
MimeTypes: strings.Join(cfg.MimeTypes, config.ValueSeparator),
|
||||
}
|
||||
}
|
||||
|
@ -23,7 +23,6 @@ import (
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/color"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
)
|
||||
|
||||
@ -36,7 +35,7 @@ func (e Error) Error() string {
|
||||
|
||||
// Default keys
|
||||
const (
|
||||
Default = "_"
|
||||
Default = `_`
|
||||
State = "state"
|
||||
Comment = "comment"
|
||||
|
||||
@ -58,6 +57,7 @@ const (
|
||||
WormSubSys = "worm"
|
||||
CacheSubSys = "cache"
|
||||
RegionSubSys = "region"
|
||||
EtcdSubSys = "etcd"
|
||||
StorageClassSubSys = "storageclass"
|
||||
CompressionSubSys = "compression"
|
||||
KmsVaultSubSys = "kms_vault"
|
||||
@ -88,6 +88,7 @@ var SubSystems = set.CreateStringSet([]string{
|
||||
CredentialsSubSys,
|
||||
WormSubSys,
|
||||
RegionSubSys,
|
||||
EtcdSubSys,
|
||||
CacheSubSys,
|
||||
StorageClassSubSys,
|
||||
CompressionSubSys,
|
||||
@ -114,6 +115,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
|
||||
CredentialsSubSys,
|
||||
WormSubSys,
|
||||
RegionSubSys,
|
||||
EtcdSubSys,
|
||||
CacheSubSys,
|
||||
StorageClassSubSys,
|
||||
CompressionSubSys,
|
||||
@ -132,6 +134,10 @@ const (
|
||||
KvNewline = "\n"
|
||||
KvDoubleQuote = `"`
|
||||
KvSingleQuote = `'`
|
||||
|
||||
// Env prefix used for all envs in MinIO
|
||||
EnvPrefix = "MINIO_"
|
||||
EnvWordDelimiter = `_`
|
||||
)
|
||||
|
||||
// KVS - is a shorthand for some wrapper functions
|
||||
@ -141,10 +147,6 @@ type KVS map[string]string
|
||||
func (kvs KVS) String() string {
|
||||
var s strings.Builder
|
||||
for k, v := range kvs {
|
||||
if k == Comment {
|
||||
// Skip the comment, comment will be printed elsewhere.
|
||||
continue
|
||||
}
|
||||
s.WriteString(k)
|
||||
s.WriteString(KvSeparator)
|
||||
s.WriteString(KvDoubleQuote)
|
||||
@ -167,20 +169,7 @@ func (c Config) String() string {
|
||||
var s strings.Builder
|
||||
for k, v := range c {
|
||||
for target, kv := range v {
|
||||
c, ok := kv[Comment]
|
||||
if ok {
|
||||
// For multiple comments split it correctly.
|
||||
for _, c1 := range strings.Split(c, KvNewline) {
|
||||
if c1 == "" {
|
||||
continue
|
||||
}
|
||||
s.WriteString(color.YellowBold(KvComment))
|
||||
s.WriteString(KvSpaceSeparator)
|
||||
s.WriteString(color.BlueBold(strings.TrimSpace(c1)))
|
||||
s.WriteString(KvNewline)
|
||||
}
|
||||
}
|
||||
s.WriteString(color.CyanBold(k))
|
||||
s.WriteString(k)
|
||||
if target != Default {
|
||||
s.WriteString(SubSystemSeparator)
|
||||
s.WriteString(target)
|
||||
@ -307,7 +296,7 @@ func (c Config) GetKVS(s string) (map[string]KVS, error) {
|
||||
}
|
||||
kvs[inputs[0]], ok = c[subSystemValue[0]][subSystemValue[1]]
|
||||
if !ok {
|
||||
err := fmt.Sprintf("sub-system target '%s' doesn't exist, proceed to create a new one", s)
|
||||
err := fmt.Sprintf("sub-system target '%s' doesn't exist", s)
|
||||
return nil, Error(err)
|
||||
}
|
||||
return kvs, nil
|
||||
@ -377,7 +366,7 @@ func (c Config) Clone() Config {
|
||||
}
|
||||
|
||||
// SetKVS - set specific key values per sub-system.
|
||||
func (c Config) SetKVS(s string, comment string, defaultKVS map[string]KVS) error {
|
||||
func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
|
||||
if len(s) == 0 {
|
||||
return Error("input arguments cannot be empty")
|
||||
}
|
||||
@ -422,27 +411,18 @@ func (c Config) SetKVS(s string, comment string, defaultKVS map[string]KVS) erro
|
||||
kvs[kv[0]] = sanitizeValue(kv[1])
|
||||
}
|
||||
|
||||
tgt := Default
|
||||
if len(subSystemValue) == 2 {
|
||||
_, ok := c[subSystemValue[0]][subSystemValue[1]]
|
||||
if !ok {
|
||||
c[subSystemValue[0]][subSystemValue[1]] = defaultKVS[subSystemValue[0]]
|
||||
// Add a comment since its a new target, this comment may be
|
||||
// overridden if client supplied it.
|
||||
if comment == "" {
|
||||
comment = fmt.Sprintf("Settings for sub-system target %s:%s",
|
||||
subSystemValue[0], subSystemValue[1])
|
||||
}
|
||||
c[subSystemValue[0]][subSystemValue[1]][Comment] = comment
|
||||
}
|
||||
tgt = subSystemValue[1]
|
||||
}
|
||||
_, ok := c[subSystemValue[0]][tgt]
|
||||
if !ok {
|
||||
c[subSystemValue[0]][tgt] = defaultKVS[subSystemValue[0]]
|
||||
comment := fmt.Sprintf("Settings for sub-system target %s:%s", subSystemValue[0], tgt)
|
||||
c[subSystemValue[0]][tgt][Comment] = comment
|
||||
}
|
||||
|
||||
var commentKv bool
|
||||
for k, v := range kvs {
|
||||
if k == Comment {
|
||||
// Set this to true to indicate comment was
|
||||
// supplied by client and is going to be preserved.
|
||||
commentKv = true
|
||||
}
|
||||
if len(subSystemValue) == 2 {
|
||||
c[subSystemValue[0]][subSystemValue[1]][k] = v
|
||||
} else {
|
||||
@ -450,15 +430,5 @@ func (c Config) SetKVS(s string, comment string, defaultKVS map[string]KVS) erro
|
||||
}
|
||||
}
|
||||
|
||||
// if client didn't supply the comment try to preserve
|
||||
// the comment if any we found while parsing the incoming
|
||||
// stream, if not preserve the default.
|
||||
if !commentKv && comment != "" {
|
||||
if len(subSystemValue) == 2 {
|
||||
c[subSystemValue[0]][subSystemValue[1]][Comment] = comment
|
||||
} else {
|
||||
c[subSystemValue[0]][Default][Comment] = comment
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
@ -18,12 +18,6 @@ package config
|
||||
|
||||
// UI errors
|
||||
var (
|
||||
ErrInvalidConfig = newErrFn(
|
||||
"Invalid value found in the configuration file",
|
||||
"Please ensure a valid value in the configuration file",
|
||||
"For more details, refer to https://docs.min.io/docs/minio-server-configuration-guide",
|
||||
)
|
||||
|
||||
ErrInvalidBrowserValue = newErrFn(
|
||||
"Invalid browser value",
|
||||
"Please check the passed value",
|
||||
|
@ -36,61 +36,100 @@ const (
|
||||
|
||||
// etcd environment values
|
||||
const (
|
||||
Endpoints = "endpoints"
|
||||
CoreDNSPath = "coredns_path"
|
||||
ClientCert = "client_cert"
|
||||
ClientCertKey = "client_cert_key"
|
||||
|
||||
EnvEtcdState = "MINIO_ETCD_STATE"
|
||||
EnvEtcdEndpoints = "MINIO_ETCD_ENDPOINTS"
|
||||
EnvEtcdCoreDNSPath = "MINIO_ETCD_COREDNS_PATH"
|
||||
EnvEtcdClientCert = "MINIO_ETCD_CLIENT_CERT"
|
||||
EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
|
||||
)
|
||||
|
||||
// New - Initialize new etcd client
|
||||
func New(rootCAs *x509.CertPool) (*clientv3.Client, error) {
|
||||
envEndpoints := env.Get(EnvEtcdEndpoints, "")
|
||||
if envEndpoints == "" {
|
||||
// etcd is not configured, nothing to do.
|
||||
// DefaultKVS - default KV settings for etcd.
|
||||
var (
|
||||
DefaultKVS = config.KVS{
|
||||
config.State: config.StateOff,
|
||||
config.Comment: "This is a default etcd configuration",
|
||||
Endpoints: "",
|
||||
CoreDNSPath: "/skydns",
|
||||
ClientCert: "",
|
||||
ClientCertKey: "",
|
||||
}
|
||||
)
|
||||
|
||||
// Config - server etcd config.
|
||||
type Config struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
CoreDNSPath string `json:"coreDNSPath"`
|
||||
clientv3.Config
|
||||
}
|
||||
|
||||
// New - initialize new etcd client.
|
||||
func New(cfg Config) (*clientv3.Client, error) {
|
||||
if !cfg.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
return clientv3.New(cfg.Config)
|
||||
}
|
||||
|
||||
etcdEndpoints := strings.Split(envEndpoints, config.ValueSeparator)
|
||||
// LookupConfig - Initialize new etcd config.
|
||||
func LookupConfig(kv config.KVS, rootCAs *x509.CertPool) (Config, error) {
|
||||
cfg := Config{}
|
||||
if err := config.CheckValidKeys(config.EtcdSubSys, kv, DefaultKVS); err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
stateBool, err := config.ParseBool(env.Get(EnvEtcdState, kv.Get(config.State)))
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
endpoints := env.Get(EnvEtcdEndpoints, kv.Get(Endpoints))
|
||||
if stateBool && len(endpoints) == 0 {
|
||||
return cfg, config.Error("'endpoints' key cannot be empty if you wish to enable etcd")
|
||||
}
|
||||
|
||||
if len(endpoints) == 0 {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
cfg.Enabled = true
|
||||
etcdEndpoints := strings.Split(endpoints, config.ValueSeparator)
|
||||
|
||||
var etcdSecure bool
|
||||
for _, endpoint := range etcdEndpoints {
|
||||
if endpoint == "" {
|
||||
continue
|
||||
}
|
||||
u, err := xnet.ParseURL(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return cfg, err
|
||||
}
|
||||
// If one of the endpoint is https, we will use https directly.
|
||||
etcdSecure = etcdSecure || u.Scheme == "https"
|
||||
}
|
||||
|
||||
var err error
|
||||
var etcdClnt *clientv3.Client
|
||||
cfg.DialTimeout = defaultDialTimeout
|
||||
cfg.DialKeepAliveTime = defaultDialKeepAlive
|
||||
cfg.Endpoints = etcdEndpoints
|
||||
cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kv.Get(CoreDNSPath))
|
||||
if etcdSecure {
|
||||
cfg.TLS = &tls.Config{
|
||||
RootCAs: rootCAs,
|
||||
}
|
||||
// This is only to support client side certificate authentication
|
||||
// https://coreos.com/etcd/docs/latest/op-guide/security.html
|
||||
etcdClientCertFile, ok1 := env.Lookup(EnvEtcdClientCert)
|
||||
etcdClientCertKey, ok2 := env.Lookup(EnvEtcdClientCertKey)
|
||||
var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
|
||||
if ok1 && ok2 {
|
||||
getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
|
||||
return &cert, terr
|
||||
etcdClientCertFile := env.Get(EnvEtcdClientCert, kv.Get(ClientCert))
|
||||
etcdClientCertKey := env.Get(EnvEtcdClientCertKey, kv.Get(ClientCertKey))
|
||||
if etcdClientCertFile != "" && etcdClientCertKey != "" {
|
||||
cfg.TLS.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
cert, err := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
|
||||
return &cert, err
|
||||
}
|
||||
}
|
||||
|
||||
etcdClnt, err = clientv3.New(clientv3.Config{
|
||||
Endpoints: etcdEndpoints,
|
||||
DialTimeout: defaultDialTimeout,
|
||||
DialKeepAliveTime: defaultDialKeepAlive,
|
||||
TLS: &tls.Config{
|
||||
RootCAs: rootCAs,
|
||||
GetClientCertificate: getClientCertificate,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
etcdClnt, err = clientv3.New(clientv3.Config{
|
||||
Endpoints: etcdEndpoints,
|
||||
DialTimeout: defaultDialTimeout,
|
||||
DialKeepAliveTime: defaultDialKeepAlive,
|
||||
})
|
||||
}
|
||||
return etcdClnt, err
|
||||
return cfg, nil
|
||||
}
|
||||
|
31
cmd/config/etcd/help.go
Normal file
31
cmd/config/etcd/help.go
Normal file
@ -0,0 +1,31 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package etcd
|
||||
|
||||
import "github.com/minio/minio/cmd/config"
|
||||
|
||||
// etcd config documented in default config
|
||||
var (
|
||||
Help = config.HelpKV{
|
||||
Endpoints: `(required) Comma separated list of etcd endpoints eg: "http://localhost:2379"`,
|
||||
CoreDNSPath: `(optional) CoreDNS etcd path location to populate DNS srv records eg: "/skydns"`,
|
||||
ClientCert: `(optional) Etcd client cert for mTLS authentication`,
|
||||
ClientCertKey: `(optional) Etcd client cert key for mTLS authentication`,
|
||||
config.State: "Indicates if etcd config is on or off",
|
||||
config.Comment: "A comment to describe the etcd settings",
|
||||
}
|
||||
)
|
@ -16,32 +16,10 @@
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"text/template"
|
||||
|
||||
"github.com/minio/minio/pkg/color"
|
||||
)
|
||||
|
||||
// HelpKV - implements help messages for keys
|
||||
// with value as description of the keys.
|
||||
type HelpKV map[string]string
|
||||
|
||||
// Help template used by all sub-systems
|
||||
const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}}
|
||||
{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
|
||||
{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
|
||||
{{end}}`
|
||||
|
||||
var funcMap = template.FuncMap{
|
||||
"colorBlueBold": color.BlueBold,
|
||||
"colorYellowBold": color.YellowBold,
|
||||
"colorCyanBold": color.CyanBold,
|
||||
"colorGreenBold": color.GreenBold,
|
||||
}
|
||||
|
||||
// HelpTemplate - captures config help template
|
||||
var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help))
|
||||
|
||||
// Region and Worm help is documented in default config
|
||||
var (
|
||||
RegionHelp = HelpKV{
|
||||
|
@ -117,10 +117,12 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
|
||||
if err != nil {
|
||||
return l, err
|
||||
}
|
||||
if !stateBool {
|
||||
return l, nil
|
||||
}
|
||||
ldapServer := env.Get(EnvServerAddr, kvs.Get(ServerAddr))
|
||||
if stateBool {
|
||||
if ldapServer == "" {
|
||||
return l, config.Error("'serveraddr' cannot be empty if you wish to enable AD/LDAP support")
|
||||
}
|
||||
}
|
||||
if ldapServer == "" {
|
||||
return l, nil
|
||||
}
|
||||
|
@ -207,6 +207,7 @@ const (
|
||||
ConfigURL = "config_url"
|
||||
ClaimPrefix = "claim_prefix"
|
||||
|
||||
EnvIdentityOpenIDState = "MINIO_IDENTITY_OPENID_STATE"
|
||||
EnvIdentityOpenIDJWKSURL = "MINIO_IDENTITY_OPENID_JWKS_URL"
|
||||
EnvIdentityOpenIDURL = "MINIO_IDENTITY_OPENID_CONFIG_URL"
|
||||
EnvIdentityOpenIDClaimPrefix = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
|
||||
@ -271,20 +272,10 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.
|
||||
return c, err
|
||||
}
|
||||
|
||||
stateBool, err := config.ParseBool(kv.Get(config.State))
|
||||
stateBool, err := config.ParseBool(env.Get(EnvIdentityOpenIDState, kv.Get(config.State)))
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
if !stateBool {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
c = Config{
|
||||
ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kv.Get(ClaimPrefix)),
|
||||
publicKeys: make(map[string]crypto.PublicKey),
|
||||
transport: transport,
|
||||
closeRespFn: closeRespFn,
|
||||
}
|
||||
|
||||
jwksURL := env.Get(EnvIamJwksURL, "") // Legacy
|
||||
if jwksURL == "" {
|
||||
@ -306,15 +297,31 @@ func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.
|
||||
// Fallback to discovery document jwksURL
|
||||
jwksURL = c.DiscoveryDoc.JwksURI
|
||||
}
|
||||
if jwksURL != "" {
|
||||
c.JWKS.URL, err = xnet.ParseURL(jwksURL)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
if err = c.PopulatePublicKey(); err != nil {
|
||||
return c, err
|
||||
|
||||
if stateBool {
|
||||
// This check is needed to ensure that empty Jwks urls are not allowed.
|
||||
if jwksURL == "" {
|
||||
return c, config.Error("'config_url' must be set to a proper OpenID discovery document URL")
|
||||
}
|
||||
}
|
||||
if jwksURL == "" {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
c = Config{
|
||||
ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kv.Get(ClaimPrefix)),
|
||||
publicKeys: make(map[string]crypto.PublicKey),
|
||||
transport: transport,
|
||||
closeRespFn: closeRespFn,
|
||||
}
|
||||
|
||||
c.JWKS.URL, err = xnet.ParseURL(jwksURL)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
if err = c.PopulatePublicKey(); err != nil {
|
||||
return c, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
|
@ -27,7 +27,7 @@ func SetNotifyKafka(s config.Config, kName string, cfg target.KafkaArgs) error {
|
||||
for _, broker := range cfg.Brokers {
|
||||
brokers = append(brokers, broker.String())
|
||||
}
|
||||
return strings.Join(brokers, ",")
|
||||
return strings.Join(brokers, config.ValueSeparator)
|
||||
}(),
|
||||
config.Comment: "Settings for Kafka notification, after migrating config",
|
||||
target.KafkaTopic: cfg.Topic,
|
||||
|
@ -224,8 +224,13 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
if !stateBool {
|
||||
return cfg, nil
|
||||
if stateBool {
|
||||
if ssc := env.Get(StandardEnv, kvs.Get(ClassStandard)); ssc == "" {
|
||||
return cfg, config.Error("'standard' key cannot be empty if you wish to enable storage class")
|
||||
}
|
||||
if rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS)); rrsc == "" {
|
||||
return cfg, config.Error("'rrs' key cannot be empty if you wish to enable storage class")
|
||||
}
|
||||
}
|
||||
|
||||
// Check for environment variables and parse into storageClass struct
|
||||
|
@ -122,7 +122,10 @@ func lookupConfigLegacy(kvs config.KVS) (KMSConfig, error) {
|
||||
cfg := KMSConfig{
|
||||
AutoEncryption: autoBool,
|
||||
}
|
||||
stateBool, err := config.ParseBool(kvs.Get(config.State))
|
||||
// Assume default as "on" for legacy config since we didn't have a _STATE
|
||||
// flag to turn it off, but we should honor it nonetheless to turn it off
|
||||
// if the vault endpoint is down and there is no way to start the server.
|
||||
stateBool, err := config.ParseBool(env.Get(EnvKMSVaultState, config.StateOn))
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
|
@ -69,7 +69,11 @@ type vaultService struct {
|
||||
var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
|
||||
|
||||
// empty/default vault configuration used to check whether a particular is empty.
|
||||
var emptyVaultConfig = VaultConfig{}
|
||||
var emptyVaultConfig = VaultConfig{
|
||||
Auth: VaultAuth{
|
||||
Type: "approle",
|
||||
},
|
||||
}
|
||||
|
||||
// IsEmpty returns true if the vault config struct is an
|
||||
// empty configuration.
|
||||
|
@ -25,7 +25,7 @@ var verifyVaultConfigTests = []struct {
|
||||
}{
|
||||
{
|
||||
ShouldFail: false, // 0
|
||||
Config: VaultConfig{},
|
||||
Config: emptyVaultConfig,
|
||||
},
|
||||
{
|
||||
ShouldFail: true,
|
||||
|
@ -184,7 +184,8 @@ var (
|
||||
// Time when object layer was initialized on start up.
|
||||
globalBootTime time.Time
|
||||
|
||||
globalActiveCred auth.Credentials
|
||||
globalActiveCred auth.Credentials
|
||||
|
||||
globalPublicCerts []*x509.Certificate
|
||||
|
||||
globalDomainNames []string // Root domains for virtual host style requests
|
||||
|
@ -117,6 +117,7 @@ func (ies *IAMEtcdStore) loadIAMConfig(item interface{}, path string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return json.Unmarshal(pdata, item)
|
||||
}
|
||||
|
||||
|
@ -328,7 +328,7 @@ func isMinioReservedBucket(bucketName string) bool {
|
||||
func getHostsSlice(records []dns.SrvRecord) []string {
|
||||
var hosts []string
|
||||
for _, r := range records {
|
||||
hosts = append(hosts, net.JoinHostPort(r.Host, fmt.Sprintf("%d", r.Port)))
|
||||
hosts = append(hosts, net.JoinHostPort(r.Host, r.Port))
|
||||
}
|
||||
return hosts
|
||||
}
|
||||
@ -337,7 +337,7 @@ func getHostsSlice(records []dns.SrvRecord) []string {
|
||||
func getHostFromSrv(records []dns.SrvRecord) string {
|
||||
rand.Seed(time.Now().Unix())
|
||||
srvRecord := records[rand.Intn(len(records))]
|
||||
return net.JoinHostPort(srvRecord.Host, fmt.Sprintf("%d", srvRecord.Port))
|
||||
return net.JoinHostPort(srvRecord.Host, srvRecord.Port)
|
||||
}
|
||||
|
||||
// IsCompressed returns true if the object is marked as compressed.
|
||||
|
@ -20,6 +20,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@ -27,11 +28,14 @@ import (
|
||||
)
|
||||
|
||||
var printEndpointError = func() func(Endpoint, error) {
|
||||
var mutex sync.Mutex
|
||||
printOnce := make(map[Endpoint]map[string]bool)
|
||||
|
||||
return func(endpoint Endpoint, err error) {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.String())
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
m, ok := printOnce[endpoint]
|
||||
if !ok {
|
||||
m = make(map[string]bool)
|
||||
|
@ -192,6 +192,7 @@ func serverHandleEnvVars() {
|
||||
}
|
||||
globalActiveCred = cred
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// serverMain handler called for 'minio server' command.
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2018 MinIO, Inc.
|
||||
* MinIO Cloud Storage, (C) 2018-2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
@ -12,6 +12,8 @@ Additionally `--config-dir` is now a legacy option which will is scheduled for r
|
||||
minio server /data
|
||||
```
|
||||
|
||||
MinIO also encrypts all the config, IAM and policies content with admin credentials.
|
||||
|
||||
### Certificate Directory
|
||||
|
||||
TLS certificates by default are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to MinIO server with TLS](https://docs.min.io/docs/how-to-secure-access-to-minio-server-with-tls).
|
||||
@ -29,6 +31,15 @@ $ mc tree --files ~/.minio
|
||||
|
||||
You can provide a custom certs directory using `--certs-dir` command line option.
|
||||
|
||||
#### Credentials
|
||||
On MinIO admin credentials or root credentials are only allowed to be changed using ENVs `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY`.
|
||||
|
||||
```
|
||||
export MINIO_ACCESS_KEY=minio
|
||||
export MINIO_SECRET_KEY=minio13
|
||||
minio server /data
|
||||
```
|
||||
|
||||
#### Region
|
||||
| Field | Type | Description |
|
||||
|:--------------------------|:---------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
|
@ -62,6 +62,29 @@ Minimum permissions required if you wish to provide restricted access with your
|
||||
## Run MinIO Gateway for AWS S3 compatible services
|
||||
As a prerequisite to run MinIO S3 gateway on an AWS S3 compatible service, you need valid access key, secret key and service endpoint.
|
||||
|
||||
## Run MinIO Gateway with double-encryption
|
||||
MinIO gateway to S3 supports encryption of data at rest. Three types of encryption modes are supported
|
||||
|
||||
- encryption can be set to ``pass-through`` to backend
|
||||
- ``single encryption`` (at the gateway)
|
||||
- ``double encryption`` (single encryption at gateway and pass through to backend).
|
||||
|
||||
This can be specified by setting MINIO_GATEWAY_SSE environment variable. If MINIO_GATEWAY_SSE and KMS are not setup, all encryption headers are passed through to the backend. If KMS environment variables are set up, ``single encryption`` is automatically performed at the gateway and encrypted object is saved at the backend.
|
||||
|
||||
To specify ``double encryption``, MINIO_GATEWAY_SSE environment variable needs to be set to "s3" for sse-s3
|
||||
and "c" for sse-c encryption. More than one encryption option can be set, delimited by ";". Objects are encrypted at the gateway and the gateway also does a pass-through to backend. Note that in the case of SSE-C encryption, gateway derives a unique SSE-C key for pass through from the SSE-C client key using a key derivation function (KDF).
|
||||
|
||||
```sh
|
||||
export MINIO_GATEWAY_SSE="s3;c"
|
||||
export MINIO_KMS_VAULT_STATE=on
|
||||
export MINIO_KMS_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
|
||||
export MINIO_KMS_VAULT_APPROLE_SECRET=4e30c52f-13e4-a6f5-0763-d50e8cb4321f
|
||||
export MINIO_KMS_VAULT_ENDPOINT=https://vault-endpoint-ip:8200
|
||||
export MINIO_KMS_VAULT_KEY_NAME=my-minio-key
|
||||
export MINIO_KMS_VAULT_AUTH_TYPE=approle
|
||||
minio gateway s3
|
||||
```
|
||||
|
||||
### Using Docker
|
||||
```
|
||||
docker run -p 9000:9000 --name minio-s3 \
|
||||
|
@ -20,10 +20,7 @@ MinIO supports two different KMS concepts:
|
||||
Further if the MinIO server machine is ever compromised, then the master key must also be treated as compromised.
|
||||
|
||||
**Important:**
|
||||
If multiple MinIO servers are configured as [gateways](https://github.com/minio/minio/blob/master/docs/gateway/README.md)
|
||||
pointing to the *same* backend - for example the same NAS storage - then the KMS configuration **must** be the same for
|
||||
all gateways. Otherwise one gateway may not be able to decrypt objects created by another gateway. It is the operators'
|
||||
responsibility to ensure consistency.
|
||||
If multiple MinIO servers are configured as [gateways](https://github.com/minio/minio/blob/master/docs/gateway/README.md) pointing to the *same* backend - for example the same NAS storage - then the KMS configuration **must** be the same for all gateways. Otherwise one gateway may not be able to decrypt objects created by another gateway. It is the operator responsibility to ensure consistency.
|
||||
|
||||
## Get started
|
||||
|
||||
@ -197,24 +194,6 @@ export MINIO_KMS_VAULT_NAMESPACE=ns1
|
||||
|
||||
Note: If [Vault Namespaces](https://learn.hashicorp.com/vault/operations/namespaces) are in use, MINIO_KMS_VAULT_VAULT_NAMESPACE variable needs to be set before setting approle and transit secrets engine.
|
||||
|
||||
MinIO gateway to S3 supports encryption. Three encryption modes are possible - encryption can be set to ``pass-through`` to backend, ``single encryption`` (at the gateway) or ``double encryption`` (single encryption at gateway and pass through to backend). This can be specified by setting MINIO_GATEWAY_SSE and KMS environment variables set in Step 2.1.2.
|
||||
|
||||
If MINIO_GATEWAY_SSE and KMS are not setup, all encryption headers are passed through to the backend. If KMS environment variables are set up, ``single encryption`` is automatically performed at the gateway and encrypted object is saved at the backend.
|
||||
|
||||
To specify ``double encryption``, MINIO_GATEWAY_SSE environment variable needs to be set to "s3" for sse-s3
|
||||
and "c" for sse-c encryption. More than one encryption option can be set, delimited by ";". Objects are encrypted at the gateway and the gateway also does a pass-through to backend. Note that in the case of SSE-C encryption, gateway derives a unique SSE-C key for pass through from the SSE-C client key using a KDF.
|
||||
|
||||
```sh
|
||||
export MINIO_GATEWAY_SSE="s3;c"
|
||||
export MINIO_KMS_VAULT_STATE=on
|
||||
export MINIO_KMS_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
|
||||
export MINIO_KMS_VAULT_APPROLE_SECRET=4e30c52f-13e4-a6f5-0763-d50e8cb4321f
|
||||
export MINIO_KMS_VAULT_ENDPOINT=https://vault-endpoint-ip:8200
|
||||
export MINIO_KMS_VAULT_KEY_NAME=my-minio-key
|
||||
export MINIO_KMS_VAULT_AUTH_TYPE=approle
|
||||
minio gateway s3
|
||||
```
|
||||
|
||||
#### 2.2 Specify a master key
|
||||
|
||||
**2.2.1 KMS master key from environment variables**
|
||||
|
@ -34,6 +34,7 @@ Make sure we have followed the previous step and configured each software indepe
|
||||
```
|
||||
export MINIO_ACCESS_KEY=minio
|
||||
export MINIO_SECRET_KEY=minio123
|
||||
export MINIO_IDENTITY_OPENID_STATE="on"
|
||||
export MINIO_IDENTITY_OPENID_CONFIG_URL=https://localhost:9443/oauth2/oidcdiscovery/.well-known/openid-configuration
|
||||
minio server /mnt/data
|
||||
```
|
||||
@ -46,6 +47,7 @@ Make sure we have followed the previous step and configured each software indepe
|
||||
```
|
||||
export MINIO_ACCESS_KEY=aws_access_key
|
||||
export MINIO_SECRET_KEY=aws_secret_key
|
||||
export MINIO_IDENTITY_OPENID_STATE="on"
|
||||
export MINIO_IDENTITY_OPENID_CONFIG_URL=https://localhost:9443/oauth2/oidcdiscovery/.well-known/openid-configuration
|
||||
export MINIO_ETCD_ENDPOINTS=http://localhost:2379
|
||||
minio gateway s3
|
||||
|
@ -1,11 +1,25 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package color
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/mattn/go-isatty"
|
||||
)
|
||||
|
||||
// global colors.
|
||||
@ -13,7 +27,7 @@ var (
|
||||
// Check if we stderr, stdout are dumb terminals, we do not apply
|
||||
// ansi coloring on dumb terminals.
|
||||
IsTerminal = func() bool {
|
||||
return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
|
||||
return !color.NoColor
|
||||
}
|
||||
|
||||
Bold = func() func(a ...interface{}) string {
|
||||
@ -22,78 +36,91 @@ var (
|
||||
}
|
||||
return fmt.Sprint
|
||||
}()
|
||||
|
||||
Red = func() func(format string, a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgRed).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
|
||||
Blue = func() func(format string, a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgBlue).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
|
||||
Yellow = func() func(format string, a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgYellow).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
|
||||
Green = func() func(a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgGreen).SprintFunc()
|
||||
}
|
||||
return fmt.Sprint
|
||||
}()
|
||||
|
||||
GreenBold = func() func(a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgGreen, color.Bold).SprintFunc()
|
||||
}
|
||||
return fmt.Sprint
|
||||
}()
|
||||
|
||||
CyanBold = func() func(a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgCyan, color.Bold).SprintFunc()
|
||||
}
|
||||
return fmt.Sprint
|
||||
}()
|
||||
|
||||
YellowBold = func() func(format string, a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgYellow, color.Bold).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
|
||||
BlueBold = func() func(format string, a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgBlue, color.Bold).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
|
||||
BgYellow = func() func(format string, a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.BgYellow).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
|
||||
Black = func() func(format string, a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgBlack).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
|
||||
FgRed = func() func(a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgRed).SprintFunc()
|
||||
}
|
||||
return fmt.Sprint
|
||||
}()
|
||||
|
||||
BgRed = func() func(format string, a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.BgRed).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
|
||||
FgWhite = func() func(format string, a ...interface{}) string {
|
||||
if IsTerminal() {
|
||||
return color.New(color.FgWhite).SprintfFunc()
|
||||
|
@ -23,7 +23,6 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -39,7 +38,7 @@ var ErrNoEntriesFound = errors.New("No entries found for this key")
|
||||
const etcdPathSeparator = "/"
|
||||
|
||||
// create a new coredns service record for the bucket.
|
||||
func newCoreDNSMsg(ip string, port int, ttl uint32) ([]byte, error) {
|
||||
func newCoreDNSMsg(ip string, port string, ttl uint32) ([]byte, error) {
|
||||
return json.Marshal(&SrvRecord{
|
||||
Host: ip,
|
||||
Port: port,
|
||||
@ -48,11 +47,11 @@ func newCoreDNSMsg(ip string, port int, ttl uint32) ([]byte, error) {
|
||||
})
|
||||
}
|
||||
|
||||
// Retrieves list of DNS entries for the domain.
|
||||
func (c *coreDNS) List() ([]SrvRecord, error) {
|
||||
// List - Retrieves list of DNS entries for the domain.
|
||||
func (c *CoreDNS) List() ([]SrvRecord, error) {
|
||||
var srvRecords []SrvRecord
|
||||
for _, domainName := range c.domainNames {
|
||||
key := msg.Path(fmt.Sprintf("%s.", domainName), defaultPrefixPath)
|
||||
key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
|
||||
records, err := c.list(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -67,11 +66,11 @@ func (c *coreDNS) List() ([]SrvRecord, error) {
|
||||
return srvRecords, nil
|
||||
}
|
||||
|
||||
// Retrieves DNS records for a bucket.
|
||||
func (c *coreDNS) Get(bucket string) ([]SrvRecord, error) {
|
||||
// Get - Retrieves DNS records for a bucket.
|
||||
func (c *CoreDNS) Get(bucket string) ([]SrvRecord, error) {
|
||||
var srvRecords []SrvRecord
|
||||
for _, domainName := range c.domainNames {
|
||||
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), defaultPrefixPath)
|
||||
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
|
||||
records, err := c.list(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -105,7 +104,7 @@ func msgUnPath(s string) string {
|
||||
|
||||
// Retrieves list of entries under the key passed.
|
||||
// Note that this method fetches entries upto only two levels deep.
|
||||
func (c *coreDNS) list(key string) ([]SrvRecord, error) {
|
||||
func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
|
||||
r, err := c.etcdClient.Get(ctx, key, etcd.WithPrefix())
|
||||
defer cancel()
|
||||
@ -153,15 +152,15 @@ func (c *coreDNS) list(key string) ([]SrvRecord, error) {
|
||||
return srvRecords, nil
|
||||
}
|
||||
|
||||
// Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
|
||||
func (c *coreDNS) Put(bucket string) error {
|
||||
// Put - Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
|
||||
func (c *CoreDNS) Put(bucket string) error {
|
||||
for ip := range c.domainIPs {
|
||||
bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, domainName := range c.domainNames {
|
||||
key := msg.Path(fmt.Sprintf("%s.%s", bucket, domainName), defaultPrefixPath)
|
||||
key := msg.Path(fmt.Sprintf("%s.%s", bucket, domainName), c.prefixPath)
|
||||
key = key + etcdPathSeparator + ip
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
|
||||
_, err = c.etcdClient.Put(ctx, key, string(bucketMsg))
|
||||
@ -177,10 +176,10 @@ func (c *coreDNS) Put(bucket string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Removes DNS entries added in Put().
|
||||
func (c *coreDNS) Delete(bucket string) error {
|
||||
// Delete - Removes DNS entries added in Put().
|
||||
func (c *CoreDNS) Delete(bucket string) error {
|
||||
for _, domainName := range c.domainNames {
|
||||
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), defaultPrefixPath)
|
||||
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
|
||||
srvRecords, err := c.list(key)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -197,10 +196,10 @@ func (c *coreDNS) Delete(bucket string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Removes a specific DNS entry
|
||||
func (c *coreDNS) DeleteRecord(record SrvRecord) error {
|
||||
// DeleteRecord - Removes a specific DNS entry
|
||||
func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
|
||||
for _, domainName := range c.domainNames {
|
||||
key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), defaultPrefixPath)
|
||||
key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), c.prefixPath)
|
||||
|
||||
dctx, dcancel := context.WithTimeout(context.Background(), defaultContextTimeout)
|
||||
if _, err := c.etcdClient.Delete(dctx, key+etcdPathSeparator+record.Host); err != nil {
|
||||
@ -213,26 +212,71 @@ func (c *coreDNS) DeleteRecord(record SrvRecord) error {
|
||||
}
|
||||
|
||||
// CoreDNS - represents dns config for coredns server.
|
||||
type coreDNS struct {
|
||||
type CoreDNS struct {
|
||||
domainNames []string
|
||||
domainIPs set.StringSet
|
||||
domainPort int
|
||||
domainPort string
|
||||
prefixPath string
|
||||
etcdClient *etcd.Client
|
||||
}
|
||||
|
||||
// Option - functional options pattern style
|
||||
type Option func(*CoreDNS)
|
||||
|
||||
// DomainNames set a list of domain names used by this CoreDNS
|
||||
// client setting, note this will fail if set to empty when
|
||||
// constructor initializes.
|
||||
func DomainNames(domainNames []string) Option {
|
||||
return func(args *CoreDNS) {
|
||||
args.domainNames = domainNames
|
||||
}
|
||||
}
|
||||
|
||||
// DomainIPs set a list of custom domain IPs, note this will
|
||||
// fail if set to empty when constructor initializes.
|
||||
func DomainIPs(domainIPs set.StringSet) Option {
|
||||
return func(args *CoreDNS) {
|
||||
args.domainIPs = domainIPs
|
||||
}
|
||||
}
|
||||
|
||||
// DomainPort - is a string version of server port
|
||||
func DomainPort(domainPort string) Option {
|
||||
return func(args *CoreDNS) {
|
||||
args.domainPort = domainPort
|
||||
}
|
||||
}
|
||||
|
||||
// CoreDNSPath - custom prefix on etcd to populate DNS
|
||||
// service records, optional and can be empty.
|
||||
// if empty then c.prefixPath is used i.e "/skydns"
|
||||
func CoreDNSPath(prefix string) Option {
|
||||
return func(args *CoreDNS) {
|
||||
args.prefixPath = prefix
|
||||
}
|
||||
}
|
||||
|
||||
// NewCoreDNS - initialize a new coreDNS set/unset values.
|
||||
func NewCoreDNS(domainNames []string, domainIPs set.StringSet, domainPort string, etcdClient *etcd.Client) (Config, error) {
|
||||
if len(domainNames) == 0 || domainIPs.IsEmpty() {
|
||||
func NewCoreDNS(etcdClient *etcd.Client, setters ...Option) (Config, error) {
|
||||
if etcdClient == nil {
|
||||
return nil, errors.New("invalid argument")
|
||||
}
|
||||
|
||||
port, err := strconv.Atoi(domainPort)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
args := &CoreDNS{
|
||||
etcdClient: etcdClient,
|
||||
prefixPath: defaultPrefixPath,
|
||||
}
|
||||
|
||||
for _, setter := range setters {
|
||||
setter(args)
|
||||
}
|
||||
|
||||
if len(args.domainNames) == 0 || args.domainIPs.IsEmpty() {
|
||||
return nil, errors.New("invalid argument")
|
||||
}
|
||||
|
||||
// strip ports off of domainIPs
|
||||
domainIPsWithoutPorts := domainIPs.ApplyFunc(func(ip string) string {
|
||||
domainIPsWithoutPorts := args.domainIPs.ApplyFunc(func(ip string) string {
|
||||
host, _, err := net.SplitHostPort(ip)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "missing port in address") {
|
||||
@ -241,11 +285,7 @@ func NewCoreDNS(domainNames []string, domainIPs set.StringSet, domainPort string
|
||||
}
|
||||
return host
|
||||
})
|
||||
args.domainIPs = domainIPsWithoutPorts
|
||||
|
||||
return &coreDNS{
|
||||
domainNames: domainNames,
|
||||
domainIPs: domainIPsWithoutPorts,
|
||||
domainPort: port,
|
||||
etcdClient: etcdClient,
|
||||
}, nil
|
||||
return args, nil
|
||||
}
|
||||
|
@ -29,7 +29,7 @@ const (
|
||||
// SrvRecord - represents a DNS service record
|
||||
type SrvRecord struct {
|
||||
Host string `json:"host,omitempty"`
|
||||
Port int `json:"port,omitempty"`
|
||||
Port string `json:"port,omitempty"`
|
||||
Priority int `json:"priority,omitempty"`
|
||||
Weight int `json:"weight,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
|
@ -18,16 +18,50 @@
|
||||
package madmin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"text/template"
|
||||
|
||||
"github.com/minio/minio/pkg/color"
|
||||
)
|
||||
|
||||
// Help template used by all sub-systems
|
||||
const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}}
|
||||
{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
|
||||
{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
|
||||
{{end}}`
|
||||
|
||||
// HelpEnv template used by all sub-systems
|
||||
const HelpEnv = `{{colorBlueBold "KeyEnv"}}{{"\t"}}{{colorBlueBold "Description"}}
|
||||
{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
|
||||
{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
|
||||
{{end}}`
|
||||
|
||||
var funcMap = template.FuncMap{
|
||||
"colorBlueBold": color.BlueBold,
|
||||
"colorYellowBold": color.YellowBold,
|
||||
"colorCyanBold": color.CyanBold,
|
||||
}
|
||||
|
||||
// HelpTemplate - captures config help template
|
||||
var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help))
|
||||
|
||||
// HelpEnvTemplate - captures config help template
|
||||
var HelpEnvTemplate = template.Must(template.New("config-help-env").Funcs(funcMap).Parse(HelpEnv))
|
||||
|
||||
// HelpConfigKV - return help for a given sub-system.
|
||||
func (adm *AdminClient) HelpConfigKV(subSys, key string) (io.ReadCloser, error) {
|
||||
func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reader, error) {
|
||||
v := url.Values{}
|
||||
v.Set("subSys", subSys)
|
||||
v.Set("key", key)
|
||||
if envOnly {
|
||||
v.Set("env", "")
|
||||
}
|
||||
|
||||
reqData := requestData{
|
||||
relPath: adminAPIPrefix + "/help-config-kv",
|
||||
queryValues: v,
|
||||
@ -38,12 +72,29 @@ func (adm *AdminClient) HelpConfigKV(subSys, key string) (io.ReadCloser, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer closeResponse(resp)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
defer closeResponse(resp)
|
||||
return nil, httpRespToErrorResponse(resp)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
var help = make(map[string]string)
|
||||
d := json.NewDecoder(resp.Body)
|
||||
if err = d.Decode(&help); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var s strings.Builder
|
||||
w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0)
|
||||
if !envOnly {
|
||||
err = HelpTemplate.Execute(w, help)
|
||||
} else {
|
||||
err = HelpEnvTemplate.Execute(w, help)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
w.Flush()
|
||||
return strings.NewReader(s.String()), nil
|
||||
}
|
||||
|
@ -18,8 +18,11 @@
|
||||
package madmin
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DelConfigKV - delete key from server config.
|
||||
@ -51,7 +54,32 @@ func (adm *AdminClient) DelConfigKV(k string) (err error) {
|
||||
|
||||
// SetConfigKV - set key value config to server.
|
||||
func (adm *AdminClient) SetConfigKV(kv string) (err error) {
|
||||
econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(kv))
|
||||
bio := bufio.NewScanner(strings.NewReader(kv))
|
||||
var s strings.Builder
|
||||
var comment string
|
||||
for bio.Scan() {
|
||||
if bio.Text() == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(bio.Text(), KvComment) {
|
||||
// Join multiple comments for each newline, separated by "\n"
|
||||
comments := []string{comment, strings.TrimPrefix(bio.Text(), KvComment)}
|
||||
comment = strings.Join(comments, KvNewline)
|
||||
continue
|
||||
}
|
||||
s.WriteString(bio.Text())
|
||||
if comment != "" {
|
||||
s.WriteString(KvSpaceSeparator)
|
||||
s.WriteString(commentKey)
|
||||
s.WriteString(KvSeparator)
|
||||
s.WriteString(KvDoubleQuote)
|
||||
s.WriteString(base64.RawStdEncoding.EncodeToString([]byte(comment)))
|
||||
s.WriteString(KvDoubleQuote)
|
||||
}
|
||||
comment = ""
|
||||
}
|
||||
|
||||
econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(s.String()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -77,7 +105,7 @@ func (adm *AdminClient) SetConfigKV(kv string) (err error) {
|
||||
}
|
||||
|
||||
// GetConfigKV - returns the key, value of the requested key, incoming data is encrypted.
|
||||
func (adm *AdminClient) GetConfigKV(key string) ([]byte, error) {
|
||||
func (adm *AdminClient) GetConfigKV(key string) (Targets, error) {
|
||||
v := url.Values{}
|
||||
v.Set("key", key)
|
||||
|
||||
@ -92,9 +120,16 @@ func (adm *AdminClient) GetConfigKV(key string) ([]byte, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer closeResponse(resp)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, httpRespToErrorResponse(resp)
|
||||
}
|
||||
|
||||
return DecryptData(adm.secretAccessKey, resp.Body)
|
||||
data, err := DecryptData(adm.secretAccessKey, resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return parseSubSysTarget(data)
|
||||
}
|
||||
|
156
pkg/madmin/parse-kv.go
Normal file
156
pkg/madmin/parse-kv.go
Normal file
@ -0,0 +1,156 @@
|
||||
/*
|
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package madmin
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/pkg/color"
|
||||
)
|
||||
|
||||
// KVS each sub-system key, value
|
||||
type KVS map[string]string
|
||||
|
||||
// Targets sub-system targets
|
||||
type Targets map[string]map[string]KVS
|
||||
|
||||
const (
|
||||
commentKey = "comment"
|
||||
)
|
||||
|
||||
func (t Targets) String() string {
|
||||
var s strings.Builder
|
||||
for subSys, targetKV := range t {
|
||||
for target, kv := range targetKV {
|
||||
c := kv[commentKey]
|
||||
data, err := base64.RawStdEncoding.DecodeString(c)
|
||||
if err == nil {
|
||||
c = string(data)
|
||||
}
|
||||
for _, c1 := range strings.Split(c, KvNewline) {
|
||||
if c1 == "" {
|
||||
continue
|
||||
}
|
||||
s.WriteString(color.YellowBold(KvComment))
|
||||
s.WriteString(KvSpaceSeparator)
|
||||
s.WriteString(color.BlueBold(strings.TrimSpace(c1)))
|
||||
s.WriteString(KvNewline)
|
||||
}
|
||||
s.WriteString(subSys)
|
||||
if target != Default {
|
||||
s.WriteString(SubSystemSeparator)
|
||||
s.WriteString(target)
|
||||
}
|
||||
s.WriteString(KvSpaceSeparator)
|
||||
for k, v := range kv {
|
||||
// Comment is already printed, do not print it here.
|
||||
if k == commentKey {
|
||||
continue
|
||||
}
|
||||
s.WriteString(k)
|
||||
s.WriteString(KvSeparator)
|
||||
s.WriteString(KvDoubleQuote)
|
||||
s.WriteString(v)
|
||||
s.WriteString(KvDoubleQuote)
|
||||
s.WriteString(KvSpaceSeparator)
|
||||
}
|
||||
if len(t) > 1 {
|
||||
s.WriteString(KvNewline)
|
||||
s.WriteString(KvNewline)
|
||||
}
|
||||
}
|
||||
}
|
||||
return s.String()
|
||||
}
|
||||
|
||||
// Constant separators
|
||||
const (
|
||||
SubSystemSeparator = `:`
|
||||
KvSeparator = `=`
|
||||
KvSpaceSeparator = ` `
|
||||
KvComment = `#`
|
||||
KvDoubleQuote = `"`
|
||||
KvSingleQuote = `'`
|
||||
|
||||
KvNewline = "\n"
|
||||
Default = `_`
|
||||
)
|
||||
|
||||
// This function is needed, to trim off single or double quotes, creeping into the values.
|
||||
func sanitizeValue(v string) string {
|
||||
v = strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(v), KvDoubleQuote), KvDoubleQuote)
|
||||
return strings.TrimSuffix(strings.TrimPrefix(v, KvSingleQuote), KvSingleQuote)
|
||||
}
|
||||
|
||||
func convertTargets(s string, targets Targets) error {
|
||||
inputs := strings.SplitN(s, KvSpaceSeparator, 2)
|
||||
if len(inputs) <= 1 {
|
||||
return fmt.Errorf("invalid number of arguments '%s'", s)
|
||||
}
|
||||
subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
|
||||
if len(subSystemValue) == 0 {
|
||||
return fmt.Errorf("invalid number of arguments %s", s)
|
||||
}
|
||||
var kvs = KVS{}
|
||||
var prevK string
|
||||
for _, v := range strings.Fields(inputs[1]) {
|
||||
kv := strings.SplitN(v, KvSeparator, 2)
|
||||
if len(kv) == 0 {
|
||||
continue
|
||||
}
|
||||
if len(kv) == 1 && prevK != "" {
|
||||
kvs[prevK] = strings.Join([]string{kvs[prevK], sanitizeValue(kv[0])}, KvSpaceSeparator)
|
||||
continue
|
||||
}
|
||||
if len(kv[1]) == 0 {
|
||||
return fmt.Errorf("value for key '%s' cannot be empty", kv[0])
|
||||
}
|
||||
prevK = kv[0]
|
||||
kvs[kv[0]] = sanitizeValue(kv[1])
|
||||
}
|
||||
|
||||
_, ok := targets[subSystemValue[0]]
|
||||
if !ok {
|
||||
targets[subSystemValue[0]] = map[string]KVS{}
|
||||
}
|
||||
if len(subSystemValue) == 2 {
|
||||
targets[subSystemValue[0]][subSystemValue[1]] = kvs
|
||||
} else {
|
||||
targets[subSystemValue[0]][Default] = kvs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseSubSysTarget - parse sub-system target
|
||||
func parseSubSysTarget(buf []byte) (Targets, error) {
|
||||
targets := make(map[string]map[string]KVS)
|
||||
bio := bufio.NewScanner(bytes.NewReader(buf))
|
||||
for bio.Scan() {
|
||||
if err := convertTargets(bio.Text(), targets); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := bio.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return targets, nil
|
||||
}
|
Loading…
Reference in New Issue
Block a user