Enhance config restore to carry previous set content as well (#8483)

This PR brings support for `history` list to
list in the following agreed format

```
~ mc admin config history list -n 2 myminio
RestoreId: df0ebb1e-69b0-4043-b9dd-ab54508f2897
Date: Mon, 04 Nov 2019 17:27:27 GMT

region name="us-east-1" state="on"
region name="us-east-1" state="on"
region name="us-east-1" state="on"
region name="us-east-1" state="on"

RestoreId: ecc6873a-0ed3-41f9-b03e-a2a1bab48b5f
Date: Mon, 04 Nov 2019 17:28:23 GMT

region name=us-east-1 state=off
```

This PR also moves the help templating and coloring to
fully `mc` side instead than `madmin` API.

cmd/admin-handlers-config-kv.go

@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"strconv"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/config"
@@ -236,7 +237,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 		return
 	}
 	if restoreID == "all" {
-		chEntries, err := listServerConfigHistory(ctx, objectAPI)
+		chEntries, err := listServerConfigHistory(ctx, objectAPI, false, -1)
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -323,7 +324,14 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	chEntries, err := listServerConfigHistory(ctx, objectAPI)
+	vars := mux.Vars(r)
+	count, err := strconv.Atoi(vars["count"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	chEntries, err := listServerConfigHistory(ctx, objectAPI, true, count)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -335,7 +343,14 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	writeSuccessResponseJSON(w, data)
+	password := globalActiveCred.SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
 }
 
 // HelpConfigKVHandler - GET /minio/admin/v2/help-config-kv?subSys={subSys}&key={key}

cmd/admin-router.go

@@ -81,7 +81,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler))
 		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler))
 		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}")
 		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
 		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
 	}

cmd/config.go

@@ -34,6 +34,8 @@ import (
 const (
 	minioConfigPrefix = "config"
 
+	kvPrefix = ".kv"
+
 	// Captures all the previous SetKV operations and allows rollback.
 	minioConfigHistoryPrefix = minioConfigPrefix + "/history"
 
@@ -44,7 +46,9 @@ const (
 	minioConfigBackupFile = minioConfigFile + ".backup"
 )
 
-func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer) ([]madmin.ConfigHistoryEntry, error) {
+func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData bool, count int) (
+	[]madmin.ConfigHistoryEntry, error) {
+
 	var configHistory []madmin.ConfigHistoryEntry
 
 	// List all kvs
@@ -55,10 +59,28 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer) ([]madmin.
 			return nil, err
 		}
 		for _, obj := range res.Objects {
-			configHistory = append(configHistory, madmin.ConfigHistoryEntry{
+			cfgEntry := madmin.ConfigHistoryEntry{
-				RestoreID:  path.Base(obj.Name),
+				RestoreID:  strings.TrimSuffix(path.Base(obj.Name), kvPrefix),
 				CreateTime: obj.ModTime, // ModTime is createTime for config history entries.
-			})
+			}
+			if withData {
+				data, err := readConfig(ctx, objAPI, obj.Name)
+				if err != nil {
+					return nil, err
+				}
+				if globalConfigEncrypted {
+					data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
+					if err != nil {
+						return nil, err
+					}
+				}
+				cfgEntry.Data = string(data)
+			}
+			configHistory = append(configHistory, cfgEntry)
+			count--
+			if count == 0 {
+				break
+			}
 		}
 		if !res.IsTruncated {
 			// We are done here
@@ -73,12 +95,12 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer) ([]madmin.
 }
 
 func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error {
-	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
+	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
 	return objAPI.DeleteObject(ctx, minioMetaBucket, historyFile)
 }
 
 func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) {
-	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
+	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
 	data, err := readConfig(ctx, objAPI, historyFile)
 	if err != nil {
 		return nil, err
@@ -92,7 +114,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str
 }
 
 func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
-	uuidKV := mustGetUUID() + ".kv"
+	uuidKV := mustGetUUID() + kvPrefix
 	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
 
 	var err error

cmd/config/config.go

@@ -24,6 +24,7 @@ import (
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // Error config error type
@@ -41,7 +42,7 @@ func (e Error) Error() string {
 
 // Default keys
 const (
-	Default = `_`
+	Default = madmin.Default
 	State   = "state"
 	Comment = "comment"
 
@@ -133,13 +134,13 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 
 // Constant separators
 const (
-	SubSystemSeparator = `:`
+	SubSystemSeparator = madmin.SubSystemSeparator
-	KvSeparator        = `=`
+	KvSeparator        = madmin.KvSeparator
-	KvSpaceSeparator   = ` `
+	KvSpaceSeparator   = madmin.KvSpaceSeparator
-	KvComment          = `#`
+	KvComment          = madmin.KvComment
-	KvNewline          = "\n"
+	KvNewline          = madmin.KvNewline
-	KvDoubleQuote      = `"`
+	KvDoubleQuote      = madmin.KvDoubleQuote
-	KvSingleQuote      = `'`
+	KvSingleQuote      = madmin.KvSingleQuote
 
 	// Env prefix used for all envs in MinIO
 	EnvPrefix        = "MINIO_"

pkg/madmin/config-help-commands.go

@@ -19,42 +19,12 @@ package madmin
 
 import (
 	"encoding/json"
-	"io"
 	"net/http"
 	"net/url"
-	"strings"
-	"text/tabwriter"
-	"text/template"
-
-	"github.com/minio/minio/pkg/color"
 )
 
-// Help template used by all sub-systems
-const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}}
-{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
-{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
-{{end}}`
-
-// HelpEnv template used by all sub-systems
-const HelpEnv = `{{colorBlueBold "KeyEnv"}}{{"\t"}}{{colorBlueBold "Description"}}
-{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
-{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
-{{end}}`
-
-var funcMap = template.FuncMap{
-	"colorBlueBold":   color.BlueBold,
-	"colorYellowBold": color.YellowBold,
-	"colorCyanBold":   color.CyanBold,
-}
-
-// HelpTemplate - captures config help template
-var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help))
-
-// HelpEnvTemplate - captures config help template
-var HelpEnvTemplate = template.Must(template.New("config-help-env").Funcs(funcMap).Parse(HelpEnv))
-
 // HelpConfigKV - return help for a given sub-system.
-func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reader, error) {
+func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (map[string]string, error) {
 	v := url.Values{}
 	v.Set("subSys", subSys)
 	v.Set("key", key)
@@ -84,17 +54,5 @@ func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reade
 		return nil, err
 	}
 
-	var s strings.Builder
+	return help, nil
-	w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0)
-	if !envOnly {
-		err = HelpTemplate.Execute(w, help)
-	} else {
-		err = HelpEnvTemplate.Execute(w, help)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	w.Flush()
-	return strings.NewReader(s.String()), nil
 }

pkg/madmin/config-history-commands.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/url"
+	"strconv"
 	"time"
 )
 
@@ -80,14 +81,27 @@ func (adm *AdminClient) RestoreConfigHistoryKV(restoreID string) (err error) {
 type ConfigHistoryEntry struct {
 	RestoreID  string    `json:"restoreId"`
 	CreateTime time.Time `json:"createTime"`
+	Data       string    `json:"data"`
+}
+
+// CreateTimeFormatted is used to print formatted time for CreateTime.
+func (ch ConfigHistoryEntry) CreateTimeFormatted() string {
+	return ch.CreateTime.Format(http.TimeFormat)
 }
 
 // ListConfigHistoryKV - lists a slice of ConfigHistoryEntries sorted by createTime.
-func (adm *AdminClient) ListConfigHistoryKV() ([]ConfigHistoryEntry, error) {
+func (adm *AdminClient) ListConfigHistoryKV(count int) ([]ConfigHistoryEntry, error) {
+	if count == 0 {
+		count = 10
+	}
+	v := url.Values{}
+	v.Set("count", strconv.Itoa(count))
+
 	// Execute GET on /minio/admin/v2/list-config-history-kv
 	resp, err := adm.executeMethod(http.MethodGet,
 		requestData{
 			relPath:     adminAPIPrefix + "/list-config-history-kv",
+			queryValues: v,
 		})
 	defer closeResponse(resp)
 	if err != nil {
@@ -97,11 +111,15 @@ func (adm *AdminClient) ListConfigHistoryKV() ([]ConfigHistoryEntry, error) {
 		return nil, httpRespToErrorResponse(resp)
 	}
 
+	data, err := DecryptData(adm.secretAccessKey, resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
 	var chEntries []ConfigHistoryEntry
-	d := json.NewDecoder(resp.Body)
+	if err = json.Unmarshal(data, &chEntries); err != nil {
-	d.DisallowUnknownFields()
-	if err = d.Decode(&chEntries); err != nil {
 		return chEntries, err
 	}
+
 	return chEntries, nil
 }

pkg/madmin/config-kv-commands.go

@@ -76,6 +76,7 @@ func (adm *AdminClient) SetConfigKV(kv string) (err error) {
 			s.WriteString(base64.RawStdEncoding.EncodeToString([]byte(comment)))
 			s.WriteString(KvDoubleQuote)
 		}
+		s.WriteString(KvNewline)
 		comment = ""
 	}
 
@@ -131,5 +132,5 @@ func (adm *AdminClient) GetConfigKV(key string) (Targets, error) {
 		return nil, err
 	}
 
-	return parseSubSysTarget(data)
+	return ParseSubSysTarget(data)
 }

pkg/madmin/parse-kv.go

@@ -88,10 +88,10 @@ const (
 	KvSeparator        = `=`
 	KvSpaceSeparator   = ` `
 	KvComment          = `#`
+	KvNewline          = "\n"
 	KvDoubleQuote      = `"`
 	KvSingleQuote      = `'`
 
-	KvNewline = "\n"
 	Default = `_`
 )
 
@@ -140,8 +140,8 @@ func convertTargets(s string, targets Targets) error {
 	return nil
 }
 
-// parseSubSysTarget - parse sub-system target
+// ParseSubSysTarget - parse sub-system target
-func parseSubSysTarget(buf []byte) (Targets, error) {
+func ParseSubSysTarget(buf []byte) (Targets, error) {
 	targets := make(map[string]map[string]KVS)
 	bio := bufio.NewScanner(bytes.NewReader(buf))
 	for bio.Scan() {