Add etcd part of config support, add noColor/json support (#8439)

- Add color/json mode support for get/help commands
- Support ENV help for all sub-systems
- Add support for etcd as part of config

pkg/color/color.go

@@ -1,11 +1,25 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package color
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/fatih/color"
-	"github.com/mattn/go-isatty"
 )
 
 // global colors.
@@ -13,7 +27,7 @@ var (
 	// Check if we stderr, stdout are dumb terminals, we do not apply
 	// ansi coloring on dumb terminals.
 	IsTerminal = func() bool {
-		return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
+		return !color.NoColor
 	}
 
 	Bold = func() func(a ...interface{}) string {
@@ -22,78 +36,91 @@ var (
 		}
 		return fmt.Sprint
 	}()
+
 	Red = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgRed).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
+
 	Blue = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgBlue).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
+
 	Yellow = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgYellow).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
+
 	Green = func() func(a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgGreen).SprintFunc()
 		}
 		return fmt.Sprint
 	}()
+
 	GreenBold = func() func(a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgGreen, color.Bold).SprintFunc()
 		}
 		return fmt.Sprint
 	}()
+
 	CyanBold = func() func(a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgCyan, color.Bold).SprintFunc()
 		}
 		return fmt.Sprint
 	}()
+
 	YellowBold = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgYellow, color.Bold).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
+
 	BlueBold = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgBlue, color.Bold).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
+
 	BgYellow = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.BgYellow).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
+
 	Black = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgBlack).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
+
 	FgRed = func() func(a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgRed).SprintFunc()
 		}
 		return fmt.Sprint
 	}()
+
 	BgRed = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.BgRed).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
+
 	FgWhite = func() func(format string, a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgWhite).SprintfFunc()

pkg/dns/etcd_dns.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"net"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
@@ -39,7 +38,7 @@ var ErrNoEntriesFound = errors.New("No entries found for this key")
 const etcdPathSeparator = "/"
 
 // create a new coredns service record for the bucket.
-func newCoreDNSMsg(ip string, port int, ttl uint32) ([]byte, error) {
+func newCoreDNSMsg(ip string, port string, ttl uint32) ([]byte, error) {
 	return json.Marshal(&SrvRecord{
 		Host:         ip,
 		Port:         port,
@@ -48,11 +47,11 @@ func newCoreDNSMsg(ip string, port int, ttl uint32) ([]byte, error) {
 	})
 }
 
-// Retrieves list of DNS entries for the domain.
-func (c *coreDNS) List() ([]SrvRecord, error) {
+// List - Retrieves list of DNS entries for the domain.
+func (c *CoreDNS) List() ([]SrvRecord, error) {
 	var srvRecords []SrvRecord
 	for _, domainName := range c.domainNames {
-		key := msg.Path(fmt.Sprintf("%s.", domainName), defaultPrefixPath)
+		key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
 		records, err := c.list(key)
 		if err != nil {
 			return nil, err
@@ -67,11 +66,11 @@ func (c *coreDNS) List() ([]SrvRecord, error) {
 	return srvRecords, nil
 }
 
-// Retrieves DNS records for a bucket.
-func (c *coreDNS) Get(bucket string) ([]SrvRecord, error) {
+// Get - Retrieves DNS records for a bucket.
+func (c *CoreDNS) Get(bucket string) ([]SrvRecord, error) {
 	var srvRecords []SrvRecord
 	for _, domainName := range c.domainNames {
-		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), defaultPrefixPath)
+		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
 		records, err := c.list(key)
 		if err != nil {
 			return nil, err
@@ -105,7 +104,7 @@ func msgUnPath(s string) string {
 
 // Retrieves list of entries under the key passed.
 // Note that this method fetches entries upto only two levels deep.
-func (c *coreDNS) list(key string) ([]SrvRecord, error) {
+func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 	r, err := c.etcdClient.Get(ctx, key, etcd.WithPrefix())
 	defer cancel()
@@ -153,15 +152,15 @@ func (c *coreDNS) list(key string) ([]SrvRecord, error) {
 	return srvRecords, nil
 }
 
-// Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
-func (c *coreDNS) Put(bucket string) error {
+// Put - Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
+func (c *CoreDNS) Put(bucket string) error {
 	for ip := range c.domainIPs {
 		bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL)
 		if err != nil {
 			return err
 		}
 		for _, domainName := range c.domainNames {
-			key := msg.Path(fmt.Sprintf("%s.%s", bucket, domainName), defaultPrefixPath)
+			key := msg.Path(fmt.Sprintf("%s.%s", bucket, domainName), c.prefixPath)
 			key = key + etcdPathSeparator + ip
 			ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 			_, err = c.etcdClient.Put(ctx, key, string(bucketMsg))
@@ -177,10 +176,10 @@ func (c *coreDNS) Put(bucket string) error {
 	return nil
 }
 
-// Removes DNS entries added in Put().
-func (c *coreDNS) Delete(bucket string) error {
+// Delete - Removes DNS entries added in Put().
+func (c *CoreDNS) Delete(bucket string) error {
 	for _, domainName := range c.domainNames {
-		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), defaultPrefixPath)
+		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
 		srvRecords, err := c.list(key)
 		if err != nil {
 			return err
@@ -197,10 +196,10 @@ func (c *coreDNS) Delete(bucket string) error {
 	return nil
 }
 
-// Removes a specific DNS entry
-func (c *coreDNS) DeleteRecord(record SrvRecord) error {
+// DeleteRecord - Removes a specific DNS entry
+func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
 	for _, domainName := range c.domainNames {
-		key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), defaultPrefixPath)
+		key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), c.prefixPath)
 
 		dctx, dcancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 		if _, err := c.etcdClient.Delete(dctx, key+etcdPathSeparator+record.Host); err != nil {
@@ -213,26 +212,71 @@ func (c *coreDNS) DeleteRecord(record SrvRecord) error {
 }
 
 // CoreDNS - represents dns config for coredns server.
-type coreDNS struct {
+type CoreDNS struct {
 	domainNames []string
 	domainIPs   set.StringSet
-	domainPort  int
+	domainPort  string
+	prefixPath  string
 	etcdClient  *etcd.Client
 }
 
+// Option - functional options pattern style
+type Option func(*CoreDNS)
+
+// DomainNames set a list of domain names used by this CoreDNS
+// client setting, note this will fail if set to empty when
+// constructor initializes.
+func DomainNames(domainNames []string) Option {
+	return func(args *CoreDNS) {
+		args.domainNames = domainNames
+	}
+}
+
+// DomainIPs set a list of custom domain IPs, note this will
+// fail if set to empty when constructor initializes.
+func DomainIPs(domainIPs set.StringSet) Option {
+	return func(args *CoreDNS) {
+		args.domainIPs = domainIPs
+	}
+}
+
+// DomainPort - is a string version of server port
+func DomainPort(domainPort string) Option {
+	return func(args *CoreDNS) {
+		args.domainPort = domainPort
+	}
+}
+
+// CoreDNSPath - custom prefix on etcd to populate DNS
+// service records, optional and can be empty.
+// if empty then c.prefixPath is used i.e "/skydns"
+func CoreDNSPath(prefix string) Option {
+	return func(args *CoreDNS) {
+		args.prefixPath = prefix
+	}
+}
+
 // NewCoreDNS - initialize a new coreDNS set/unset values.
-func NewCoreDNS(domainNames []string, domainIPs set.StringSet, domainPort string, etcdClient *etcd.Client) (Config, error) {
-	if len(domainNames) == 0 || domainIPs.IsEmpty() {
+func NewCoreDNS(etcdClient *etcd.Client, setters ...Option) (Config, error) {
+	if etcdClient == nil {
 		return nil, errors.New("invalid argument")
 	}
 
-	port, err := strconv.Atoi(domainPort)
-	if err != nil {
-		return nil, err
+	args := &CoreDNS{
+		etcdClient: etcdClient,
+		prefixPath: defaultPrefixPath,
+	}
+
+	for _, setter := range setters {
+		setter(args)
+	}
+
+	if len(args.domainNames) == 0 || args.domainIPs.IsEmpty() {
+		return nil, errors.New("invalid argument")
 	}
 
 	// strip ports off of domainIPs
-	domainIPsWithoutPorts := domainIPs.ApplyFunc(func(ip string) string {
+	domainIPsWithoutPorts := args.domainIPs.ApplyFunc(func(ip string) string {
 		host, _, err := net.SplitHostPort(ip)
 		if err != nil {
 			if strings.Contains(err.Error(), "missing port in address") {
@@ -241,11 +285,7 @@ func NewCoreDNS(domainNames []string, domainIPs set.StringSet, domainPort string
 		}
 		return host
 	})
+	args.domainIPs = domainIPsWithoutPorts
 
-	return &coreDNS{
-		domainNames: domainNames,
-		domainIPs:   domainIPsWithoutPorts,
-		domainPort:  port,
-		etcdClient:  etcdClient,
-	}, nil
+	return args, nil
 }

pkg/dns/types.go

@@ -29,7 +29,7 @@ const (
 // SrvRecord - represents a DNS service record
 type SrvRecord struct {
 	Host     string `json:"host,omitempty"`
-	Port     int    `json:"port,omitempty"`
+	Port     string `json:"port,omitempty"`
 	Priority int    `json:"priority,omitempty"`
 	Weight   int    `json:"weight,omitempty"`
 	Text     string `json:"text,omitempty"`

pkg/madmin/config-help-commands.go

@@ -18,16 +18,50 @@
 package madmin
 
 import (
+	"encoding/json"
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
+	"text/tabwriter"
+	"text/template"
+
+	"github.com/minio/minio/pkg/color"
 )
 
+// Help template used by all sub-systems
+const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}}
+{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
+{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
+{{end}}`
+
+// HelpEnv template used by all sub-systems
+const HelpEnv = `{{colorBlueBold "KeyEnv"}}{{"\t"}}{{colorBlueBold "Description"}}
+{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
+{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
+{{end}}`
+
+var funcMap = template.FuncMap{
+	"colorBlueBold":   color.BlueBold,
+	"colorYellowBold": color.YellowBold,
+	"colorCyanBold":   color.CyanBold,
+}
+
+// HelpTemplate - captures config help template
+var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help))
+
+// HelpEnvTemplate - captures config help template
+var HelpEnvTemplate = template.Must(template.New("config-help-env").Funcs(funcMap).Parse(HelpEnv))
+
 // HelpConfigKV - return help for a given sub-system.
-func (adm *AdminClient) HelpConfigKV(subSys, key string) (io.ReadCloser, error) {
+func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reader, error) {
 	v := url.Values{}
 	v.Set("subSys", subSys)
 	v.Set("key", key)
+	if envOnly {
+		v.Set("env", "")
+	}
+
 	reqData := requestData{
 		relPath:     adminAPIPrefix + "/help-config-kv",
 		queryValues: v,
@@ -38,12 +72,29 @@ func (adm *AdminClient) HelpConfigKV(subSys, key string) (io.ReadCloser, error)
 	if err != nil {
 		return nil, err
 	}
+	defer closeResponse(resp)
 
 	if resp.StatusCode != http.StatusOK {
-		defer closeResponse(resp)
 		return nil, httpRespToErrorResponse(resp)
 	}
 
-	return resp.Body, nil
+	var help = make(map[string]string)
+	d := json.NewDecoder(resp.Body)
+	if err = d.Decode(&help); err != nil {
+		return nil, err
+	}
 
+	var s strings.Builder
+	w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0)
+	if !envOnly {
+		err = HelpTemplate.Execute(w, help)
+	} else {
+		err = HelpEnvTemplate.Execute(w, help)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	w.Flush()
+	return strings.NewReader(s.String()), nil
 }

pkg/madmin/config-kv-commands.go

@@ -18,8 +18,11 @@
 package madmin
 
 import (
+	"bufio"
+	"encoding/base64"
 	"net/http"
 	"net/url"
+	"strings"
 )
 
 // DelConfigKV - delete key from server config.
@@ -51,7 +54,32 @@ func (adm *AdminClient) DelConfigKV(k string) (err error) {
 
 // SetConfigKV - set key value config to server.
 func (adm *AdminClient) SetConfigKV(kv string) (err error) {
-	econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(kv))
+	bio := bufio.NewScanner(strings.NewReader(kv))
+	var s strings.Builder
+	var comment string
+	for bio.Scan() {
+		if bio.Text() == "" {
+			continue
+		}
+		if strings.HasPrefix(bio.Text(), KvComment) {
+			// Join multiple comments for each newline, separated by "\n"
+			comments := []string{comment, strings.TrimPrefix(bio.Text(), KvComment)}
+			comment = strings.Join(comments, KvNewline)
+			continue
+		}
+		s.WriteString(bio.Text())
+		if comment != "" {
+			s.WriteString(KvSpaceSeparator)
+			s.WriteString(commentKey)
+			s.WriteString(KvSeparator)
+			s.WriteString(KvDoubleQuote)
+			s.WriteString(base64.RawStdEncoding.EncodeToString([]byte(comment)))
+			s.WriteString(KvDoubleQuote)
+		}
+		comment = ""
+	}
+
+	econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(s.String()))
 	if err != nil {
 		return err
 	}
@@ -77,7 +105,7 @@ func (adm *AdminClient) SetConfigKV(kv string) (err error) {
 }
 
 // GetConfigKV - returns the key, value of the requested key, incoming data is encrypted.
-func (adm *AdminClient) GetConfigKV(key string) ([]byte, error) {
+func (adm *AdminClient) GetConfigKV(key string) (Targets, error) {
 	v := url.Values{}
 	v.Set("key", key)
 
@@ -92,9 +120,16 @@ func (adm *AdminClient) GetConfigKV(key string) ([]byte, error) {
 		return nil, err
 	}
 
+	defer closeResponse(resp)
+
 	if resp.StatusCode != http.StatusOK {
 		return nil, httpRespToErrorResponse(resp)
 	}
 
-	return DecryptData(adm.secretAccessKey, resp.Body)
+	data, err := DecryptData(adm.secretAccessKey, resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return parseSubSysTarget(data)
 }

pkg/madmin/parse-kv.go

@@ -0,0 +1,156 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/pkg/color"
+)
+
+// KVS each sub-system key, value
+type KVS map[string]string
+
+// Targets sub-system targets
+type Targets map[string]map[string]KVS
+
+const (
+	commentKey = "comment"
+)
+
+func (t Targets) String() string {
+	var s strings.Builder
+	for subSys, targetKV := range t {
+		for target, kv := range targetKV {
+			c := kv[commentKey]
+			data, err := base64.RawStdEncoding.DecodeString(c)
+			if err == nil {
+				c = string(data)
+			}
+			for _, c1 := range strings.Split(c, KvNewline) {
+				if c1 == "" {
+					continue
+				}
+				s.WriteString(color.YellowBold(KvComment))
+				s.WriteString(KvSpaceSeparator)
+				s.WriteString(color.BlueBold(strings.TrimSpace(c1)))
+				s.WriteString(KvNewline)
+			}
+			s.WriteString(subSys)
+			if target != Default {
+				s.WriteString(SubSystemSeparator)
+				s.WriteString(target)
+			}
+			s.WriteString(KvSpaceSeparator)
+			for k, v := range kv {
+				// Comment is already printed, do not print it here.
+				if k == commentKey {
+					continue
+				}
+				s.WriteString(k)
+				s.WriteString(KvSeparator)
+				s.WriteString(KvDoubleQuote)
+				s.WriteString(v)
+				s.WriteString(KvDoubleQuote)
+				s.WriteString(KvSpaceSeparator)
+			}
+			if len(t) > 1 {
+				s.WriteString(KvNewline)
+				s.WriteString(KvNewline)
+			}
+		}
+	}
+	return s.String()
+}
+
+// Constant separators
+const (
+	SubSystemSeparator = `:`
+	KvSeparator        = `=`
+	KvSpaceSeparator   = ` `
+	KvComment          = `#`
+	KvDoubleQuote      = `"`
+	KvSingleQuote      = `'`
+
+	KvNewline = "\n"
+	Default   = `_`
+)
+
+// This function is needed, to trim off single or double quotes, creeping into the values.
+func sanitizeValue(v string) string {
+	v = strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(v), KvDoubleQuote), KvDoubleQuote)
+	return strings.TrimSuffix(strings.TrimPrefix(v, KvSingleQuote), KvSingleQuote)
+}
+
+func convertTargets(s string, targets Targets) error {
+	inputs := strings.SplitN(s, KvSpaceSeparator, 2)
+	if len(inputs) <= 1 {
+		return fmt.Errorf("invalid number of arguments '%s'", s)
+	}
+	subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
+	if len(subSystemValue) == 0 {
+		return fmt.Errorf("invalid number of arguments %s", s)
+	}
+	var kvs = KVS{}
+	var prevK string
+	for _, v := range strings.Fields(inputs[1]) {
+		kv := strings.SplitN(v, KvSeparator, 2)
+		if len(kv) == 0 {
+			continue
+		}
+		if len(kv) == 1 && prevK != "" {
+			kvs[prevK] = strings.Join([]string{kvs[prevK], sanitizeValue(kv[0])}, KvSpaceSeparator)
+			continue
+		}
+		if len(kv[1]) == 0 {
+			return fmt.Errorf("value for key '%s' cannot be empty", kv[0])
+		}
+		prevK = kv[0]
+		kvs[kv[0]] = sanitizeValue(kv[1])
+	}
+
+	_, ok := targets[subSystemValue[0]]
+	if !ok {
+		targets[subSystemValue[0]] = map[string]KVS{}
+	}
+	if len(subSystemValue) == 2 {
+		targets[subSystemValue[0]][subSystemValue[1]] = kvs
+	} else {
+		targets[subSystemValue[0]][Default] = kvs
+	}
+	return nil
+}
+
+// parseSubSysTarget - parse sub-system target
+func parseSubSysTarget(buf []byte) (Targets, error) {
+	targets := make(map[string]map[string]KVS)
+	bio := bufio.NewScanner(bytes.NewReader(buf))
+	for bio.Scan() {
+		if err := convertTargets(bio.Text(), targets); err != nil {
+			return nil, err
+		}
+	}
+	if err := bio.Err(); err != nil {
+		return nil, err
+	}
+	return targets, nil
+}