From 1e457dda7eed1169005ee47a30e8121785eccdbf Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 5 Nov 2019 06:18:26 -0800 Subject: [PATCH] Enhance config restore to carry previous set content as well (#8483) This PR brings support for `history` list to list in the following agreed format ``` ~ mc admin config history list -n 2 myminio RestoreId: df0ebb1e-69b0-4043-b9dd-ab54508f2897 Date: Mon, 04 Nov 2019 17:27:27 GMT region name="us-east-1" state="on" region name="us-east-1" state="on" region name="us-east-1" state="on" region name="us-east-1" state="on" RestoreId: ecc6873a-0ed3-41f9-b03e-a2a1bab48b5f Date: Mon, 04 Nov 2019 17:28:23 GMT region name=us-east-1 state=off ``` This PR also moves the help templating and coloring to fully `mc` side instead than `madmin` API. --- cmd/admin-handlers-config-kv.go | 21 ++++++++++-- cmd/admin-router.go | 2 +- cmd/config.go | 36 +++++++++++++++++---- cmd/config/config.go | 17 +++++----- pkg/madmin/config-help-commands.go | 46 ++------------------------- pkg/madmin/config-history-commands.go | 28 +++++++++++++--- pkg/madmin/config-kv-commands.go | 3 +- pkg/madmin/parse-kv.go | 8 ++--- 8 files changed, 88 insertions(+), 73 deletions(-) diff --git a/cmd/admin-handlers-config-kv.go b/cmd/admin-handlers-config-kv.go index ab9921ed0..3fa4eaf5a 100644 --- a/cmd/admin-handlers-config-kv.go +++ b/cmd/admin-handlers-config-kv.go @@ -23,6 +23,7 @@ import ( "encoding/json" "io" "net/http" + "strconv" "github.com/gorilla/mux" "github.com/minio/minio/cmd/config" @@ -236,7 +237,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r * return } if restoreID == "all" { - chEntries, err := listServerConfigHistory(ctx, objectAPI) + chEntries, err := listServerConfigHistory(ctx, objectAPI, false, -1) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -323,7 +324,14 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h return } - chEntries, err := listServerConfigHistory(ctx, objectAPI) + vars := mux.Vars(r) + count, err := strconv.Atoi(vars["count"]) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + chEntries, err := listServerConfigHistory(ctx, objectAPI, true, count) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -335,7 +343,14 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h return } - writeSuccessResponseJSON(w, data) + password := globalActiveCred.SecretKey + econfigData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, econfigData) } // HelpConfigKVHandler - GET /minio/admin/v2/help-config-kv?subSys={subSys}&key={key} diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 973f0bbd7..470d4f369 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -81,7 +81,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler)) adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler)) adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}") - adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)) + adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}") adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}") adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}") } diff --git a/cmd/config.go b/cmd/config.go index 2d1d93bad..d3279886d 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -34,6 +34,8 @@ import ( const ( minioConfigPrefix = "config" + kvPrefix = ".kv" + // Captures all the previous SetKV operations and allows rollback. minioConfigHistoryPrefix = minioConfigPrefix + "/history" @@ -44,7 +46,9 @@ const ( minioConfigBackupFile = minioConfigFile + ".backup" ) -func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer) ([]madmin.ConfigHistoryEntry, error) { +func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData bool, count int) ( + []madmin.ConfigHistoryEntry, error) { + var configHistory []madmin.ConfigHistoryEntry // List all kvs @@ -55,10 +59,28 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer) ([]madmin. return nil, err } for _, obj := range res.Objects { - configHistory = append(configHistory, madmin.ConfigHistoryEntry{ - RestoreID: path.Base(obj.Name), + cfgEntry := madmin.ConfigHistoryEntry{ + RestoreID: strings.TrimSuffix(path.Base(obj.Name), kvPrefix), CreateTime: obj.ModTime, // ModTime is createTime for config history entries. - }) + } + if withData { + data, err := readConfig(ctx, objAPI, obj.Name) + if err != nil { + return nil, err + } + if globalConfigEncrypted { + data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data)) + if err != nil { + return nil, err + } + } + cfgEntry.Data = string(data) + } + configHistory = append(configHistory, cfgEntry) + count-- + if count == 0 { + break + } } if !res.IsTruncated { // We are done here @@ -73,12 +95,12 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer) ([]madmin. } func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error { - historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV) + historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix) return objAPI.DeleteObject(ctx, minioMetaBucket, historyFile) } func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) { - historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV) + historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix) data, err := readConfig(ctx, objAPI, historyFile) if err != nil { return nil, err @@ -92,7 +114,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str } func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error { - uuidKV := mustGetUUID() + ".kv" + uuidKV := mustGetUUID() + kvPrefix historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV) var err error diff --git a/cmd/config/config.go b/cmd/config/config.go index c1dba84fb..6b1ac7c30 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -24,6 +24,7 @@ import ( "github.com/minio/minio-go/pkg/set" "github.com/minio/minio/pkg/auth" "github.com/minio/minio/pkg/env" + "github.com/minio/minio/pkg/madmin" ) // Error config error type @@ -41,7 +42,7 @@ func (e Error) Error() string { // Default keys const ( - Default = `_` + Default = madmin.Default State = "state" Comment = "comment" @@ -133,13 +134,13 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{ // Constant separators const ( - SubSystemSeparator = `:` - KvSeparator = `=` - KvSpaceSeparator = ` ` - KvComment = `#` - KvNewline = "\n" - KvDoubleQuote = `"` - KvSingleQuote = `'` + SubSystemSeparator = madmin.SubSystemSeparator + KvSeparator = madmin.KvSeparator + KvSpaceSeparator = madmin.KvSpaceSeparator + KvComment = madmin.KvComment + KvNewline = madmin.KvNewline + KvDoubleQuote = madmin.KvDoubleQuote + KvSingleQuote = madmin.KvSingleQuote // Env prefix used for all envs in MinIO EnvPrefix = "MINIO_" diff --git a/pkg/madmin/config-help-commands.go b/pkg/madmin/config-help-commands.go index c4569cc9f..db4f5ed0f 100644 --- a/pkg/madmin/config-help-commands.go +++ b/pkg/madmin/config-help-commands.go @@ -19,42 +19,12 @@ package madmin import ( "encoding/json" - "io" "net/http" "net/url" - "strings" - "text/tabwriter" - "text/template" - - "github.com/minio/minio/pkg/color" ) -// Help template used by all sub-systems -const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}} -{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}} -{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}} -{{end}}` - -// HelpEnv template used by all sub-systems -const HelpEnv = `{{colorBlueBold "KeyEnv"}}{{"\t"}}{{colorBlueBold "Description"}} -{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}} -{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}} -{{end}}` - -var funcMap = template.FuncMap{ - "colorBlueBold": color.BlueBold, - "colorYellowBold": color.YellowBold, - "colorCyanBold": color.CyanBold, -} - -// HelpTemplate - captures config help template -var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help)) - -// HelpEnvTemplate - captures config help template -var HelpEnvTemplate = template.Must(template.New("config-help-env").Funcs(funcMap).Parse(HelpEnv)) - // HelpConfigKV - return help for a given sub-system. -func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reader, error) { +func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (map[string]string, error) { v := url.Values{} v.Set("subSys", subSys) v.Set("key", key) @@ -84,17 +54,5 @@ func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reade return nil, err } - var s strings.Builder - w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0) - if !envOnly { - err = HelpTemplate.Execute(w, help) - } else { - err = HelpEnvTemplate.Execute(w, help) - } - if err != nil { - return nil, err - } - - w.Flush() - return strings.NewReader(s.String()), nil + return help, nil } diff --git a/pkg/madmin/config-history-commands.go b/pkg/madmin/config-history-commands.go index a8ca48baf..4d4522944 100644 --- a/pkg/madmin/config-history-commands.go +++ b/pkg/madmin/config-history-commands.go @@ -21,6 +21,7 @@ import ( "encoding/json" "net/http" "net/url" + "strconv" "time" ) @@ -80,14 +81,27 @@ func (adm *AdminClient) RestoreConfigHistoryKV(restoreID string) (err error) { type ConfigHistoryEntry struct { RestoreID string `json:"restoreId"` CreateTime time.Time `json:"createTime"` + Data string `json:"data"` +} + +// CreateTimeFormatted is used to print formatted time for CreateTime. +func (ch ConfigHistoryEntry) CreateTimeFormatted() string { + return ch.CreateTime.Format(http.TimeFormat) } // ListConfigHistoryKV - lists a slice of ConfigHistoryEntries sorted by createTime. -func (adm *AdminClient) ListConfigHistoryKV() ([]ConfigHistoryEntry, error) { +func (adm *AdminClient) ListConfigHistoryKV(count int) ([]ConfigHistoryEntry, error) { + if count == 0 { + count = 10 + } + v := url.Values{} + v.Set("count", strconv.Itoa(count)) + // Execute GET on /minio/admin/v2/list-config-history-kv resp, err := adm.executeMethod(http.MethodGet, requestData{ - relPath: adminAPIPrefix + "/list-config-history-kv", + relPath: adminAPIPrefix + "/list-config-history-kv", + queryValues: v, }) defer closeResponse(resp) if err != nil { @@ -97,11 +111,15 @@ func (adm *AdminClient) ListConfigHistoryKV() ([]ConfigHistoryEntry, error) { return nil, httpRespToErrorResponse(resp) } + data, err := DecryptData(adm.secretAccessKey, resp.Body) + if err != nil { + return nil, err + } + var chEntries []ConfigHistoryEntry - d := json.NewDecoder(resp.Body) - d.DisallowUnknownFields() - if err = d.Decode(&chEntries); err != nil { + if err = json.Unmarshal(data, &chEntries); err != nil { return chEntries, err } + return chEntries, nil } diff --git a/pkg/madmin/config-kv-commands.go b/pkg/madmin/config-kv-commands.go index 70d9cf115..3f630c91a 100644 --- a/pkg/madmin/config-kv-commands.go +++ b/pkg/madmin/config-kv-commands.go @@ -76,6 +76,7 @@ func (adm *AdminClient) SetConfigKV(kv string) (err error) { s.WriteString(base64.RawStdEncoding.EncodeToString([]byte(comment))) s.WriteString(KvDoubleQuote) } + s.WriteString(KvNewline) comment = "" } @@ -131,5 +132,5 @@ func (adm *AdminClient) GetConfigKV(key string) (Targets, error) { return nil, err } - return parseSubSysTarget(data) + return ParseSubSysTarget(data) } diff --git a/pkg/madmin/parse-kv.go b/pkg/madmin/parse-kv.go index 0f0d35fce..97c6152c2 100644 --- a/pkg/madmin/parse-kv.go +++ b/pkg/madmin/parse-kv.go @@ -88,11 +88,11 @@ const ( KvSeparator = `=` KvSpaceSeparator = ` ` KvComment = `#` + KvNewline = "\n" KvDoubleQuote = `"` KvSingleQuote = `'` - KvNewline = "\n" - Default = `_` + Default = `_` ) // This function is needed, to trim off single or double quotes, creeping into the values. @@ -140,8 +140,8 @@ func convertTargets(s string, targets Targets) error { return nil } -// parseSubSysTarget - parse sub-system target -func parseSubSysTarget(buf []byte) (Targets, error) { +// ParseSubSysTarget - parse sub-system target +func ParseSubSysTarget(buf []byte) (Targets, error) { targets := make(map[string]map[string]KVS) bio := bufio.NewScanner(bytes.NewReader(buf)) for bio.Scan() {