From 3d94c38ec4d4d5383ae303457f30475a12d12ad2 Mon Sep 17 00:00:00 2001 From: Aditya Manthramurthy Date: Thu, 4 Aug 2022 22:21:52 -0700 Subject: [PATCH] Add env variables to configuration APIs output (#15465) Config export and config get APIs now include environment variables set on the server --- cmd/admin-handlers-config-kv.go | 51 ++++++------- cmd/common-main.go | 3 +- internal/config/config.go | 128 ++++++++++++++++++++++++++++++-- 3 files changed, 144 insertions(+), 38 deletions(-) diff --git a/cmd/admin-handlers-config-kv.go b/cmd/admin-handlers-config-kv.go index d22ae7b73..974c4070b 100644 --- a/cmd/admin-handlers-config-kv.go +++ b/cmd/admin-handlers-config-kv.go @@ -189,15 +189,20 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ cfg := globalServerConfig.Clone() vars := mux.Vars(r) - buf := &bytes.Buffer{} - cw := config.NewConfigWriteTo(cfg, vars["key"]) - if _, err := cw.WriteTo(buf); err != nil { + subSys := vars["key"] + subSysConfigs, err := cfg.GetSubsysInfo(subSys) + if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return } + var s strings.Builder + for _, subSysConfig := range subSysConfigs { + subSysConfig.AddString(&s, false) + } + password := cred.SecretKey - econfigData, err := madmin.EncryptData(password, buf.Bytes()) + econfigData, err := madmin.EncryptData(password, []byte(s.String())) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -427,42 +432,30 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques count += len(cfg[hkv.Key]) } for _, hkv := range hkvs { - v := cfg[hkv.Key] - for target, kv := range v { - off := kv.Get(config.Enable) == config.EnableOff + // We ignore the error below, as we cannot get one. + cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key) + + for _, item := range cfgSubsysItems { + off := item.Params.Get(config.Enable) == config.EnableOff switch hkv.Key { case config.EtcdSubSys: - off = !etcd.Enabled(kv) + off = !etcd.Enabled(item.Params) case config.CacheSubSys: - off = !cache.Enabled(kv) + off = !cache.Enabled(item.Params) case config.StorageClassSubSys: - off = !storageclass.Enabled(kv) + off = !storageclass.Enabled(item.Params) case config.PolicyPluginSubSys: - off = !polplugin.Enabled(kv) + off = !polplugin.Enabled(item.Params) case config.IdentityOpenIDSubSys: - off = !openid.Enabled(kv) + off = !openid.Enabled(item.Params) case config.IdentityLDAPSubSys: - off = !xldap.Enabled(kv) + off = !xldap.Enabled(item.Params) case config.IdentityTLSSubSys: off = !globalSTSTLSConfig.Enabled case config.IdentityPluginSubSys: - off = !idplugin.Enabled(kv) - } - if off { - s.WriteString(config.KvComment) - s.WriteString(config.KvSpaceSeparator) - } - s.WriteString(hkv.Key) - if target != config.Default { - s.WriteString(config.SubSystemSeparator) - s.WriteString(target) - } - s.WriteString(config.KvSpaceSeparator) - s.WriteString(kv.String()) - count-- - if count > 0 { - s.WriteString(config.KvNewline) + off = !idplugin.Enabled(item.Params) } + item.AddString(&s, off) } } diff --git a/cmd/common-main.go b/cmd/common-main.go index 40fe8e367..db18c91c3 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -508,8 +508,7 @@ func parsEnvEntry(envEntry string) (envKV, error) { Skip: true, }, nil } - const envSeparator = "=" - envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2) + envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2) if len(envTokens) != 2 { return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry) } diff --git a/internal/config/config.go b/internal/config/config.go index 8765f69fe..bb24382a3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "regexp" + "sort" "strings" "github.com/minio/madmin-go" @@ -51,6 +52,8 @@ const ( Enable = madmin.EnableKey Comment = madmin.CommentKey + EnvSeparator = "=" + // Enable values EnableOn = madmin.EnableOn EnableOff = madmin.EnableOff @@ -1020,7 +1023,8 @@ func (c Config) CheckValidKeys(subSys string, deprecatedKeys []string) error { // GetAvailableTargets - returns a list of targets configured for the given // subsystem (whether they are enabled or not). A target could be configured via // environment variables or via the configuration store. The default target is -// `_` and is always returned. +// `_` and is always returned. The result is sorted so that the default target +// is the first one and the remaining entries are sorted in ascending order. func (c Config) GetAvailableTargets(subSys string) ([]string, error) { if SubSystemsSingleTargets.Contains(subSys) { return []string{Default}, nil @@ -1032,11 +1036,11 @@ func (c Config) GetAvailableTargets(subSys string) ([]string, error) { } kvsMap := c[subSys] - s := set.NewStringSet() + seen := set.NewStringSet() // Add all targets that are configured in the config store. for k := range kvsMap { - s.Add(k) + seen.Add(k) } // Add targets that are configured via environment variables. @@ -1046,12 +1050,17 @@ func (c Config) GetAvailableTargets(subSys string) ([]string, error) { for _, k := range envsWithPrefix { tgtName := strings.TrimPrefix(k, envVarPrefix) if tgtName != "" { - s.Add(tgtName) + seen.Add(tgtName) } } } - return s.ToSlice(), nil + seen.Remove(Default) + targets := seen.ToSlice() + sort.Strings(targets) + targets = append([]string{Default}, targets...) + + return targets, nil } func getEnvVarName(subSys, target, param string) string { @@ -1059,7 +1068,8 @@ func getEnvVarName(subSys, target, param string) string { return fmt.Sprintf("%s%s%s%s", EnvPrefix, strings.ToUpper(subSys), Default, strings.ToUpper(param)) } - return fmt.Sprintf("%s%s%s%s%s%s", EnvPrefix, strings.ToUpper(subSys), Default, strings.ToUpper(param), Default, target) + return fmt.Sprintf("%s%s%s%s%s%s", EnvPrefix, strings.ToUpper(subSys), Default, strings.ToUpper(param), + Default, target) } var resolvableSubsystems = set.CreateStringSet(IdentityOpenIDSubSys) @@ -1102,7 +1112,7 @@ func (c Config) ResolveConfigParam(subSys, target, cfgParam string) (value strin defValue, isFound := defKVS.Lookup(cfgParam) // Comments usually are absent from `defKVS`, so we handle it specially. - if cfgParam == Comment { + if !isFound && cfgParam == Comment { defValue, isFound = "", true } if !isFound { @@ -1190,3 +1200,107 @@ func (c Config) GetResolvedConfigParams(subSys, target string) ([]KVSrc, error) return r, nil } + +func (c Config) getTargetKVS(subSys, target string) KVS { + store, ok := c[subSys] + if !ok { + return nil + } + return store[target] +} + +// EnvPair represents an environment variable and its value. +type EnvPair struct { + Name, Value string +} + +// SubsysInfo holds config info for a subsystem target. +type SubsysInfo struct { + SubSys, Target string + Params KVS + + // map of config parameter name to EnvPair. + EnvMap map[string]EnvPair +} + +// GetSubsysInfo returns `SubsysInfo`s for all targets for the subsystem. +func (c Config) GetSubsysInfo(subSys string) ([]SubsysInfo, error) { + // Check if config param requested is valid. + defKVS1, ok := DefaultKVS[subSys] + if !ok { + return nil, fmt.Errorf("unknown subsystem: %s", subSys) + } + + targets, err := c.GetAvailableTargets(subSys) + if err != nil { + return nil, err + } + + // The `Comment` configuration variable is optional but is available to be + // set for all sub-systems. It is not present in the `DefaultKVS` map's + // values. To enable fetching a configured comment value from the + // environment we add it to the list of default keys for the subsystem. + defKVS := make([]KV, len(defKVS1), len(defKVS1)+1) + copy(defKVS, defKVS1) + defKVS = append(defKVS, KV{Key: Comment}) + + r := make([]SubsysInfo, 0, len(targets)) + for _, target := range targets { + kvs := c.getTargetKVS(subSys, target) + cs := SubsysInfo{ + SubSys: subSys, + Target: target, + Params: kvs, + EnvMap: make(map[string]EnvPair), + } + + // Add all env vars that are set. + for _, kv := range defKVS { + envName := getEnvVarName(subSys, target, kv.Key) + envPair := EnvPair{ + Name: envName, + Value: env.Get(envName, ""), + } + if envPair.Value != "" { + cs.EnvMap[kv.Key] = envPair + } + } + + r = append(r, cs) + } + + return r, nil +} + +// AddEnvString adds env vars to the given string builder. +func (cs *SubsysInfo) AddEnvString(b *strings.Builder) { + for _, v := range cs.Params { + if ep, ok := cs.EnvMap[v.Key]; ok { + b.WriteString(KvComment) + b.WriteString(KvSpaceSeparator) + b.WriteString(ep.Name) + b.WriteString(EnvSeparator) + b.WriteString(ep.Value) + b.WriteString(KvNewline) + } + } +} + +// AddString adds the string representation of the configuration to the given +// builder. When off is true, adds a comment character before the config system +// output. +func (cs *SubsysInfo) AddString(b *strings.Builder, off bool) { + cs.AddEnvString(b) + if off { + b.WriteString(KvComment) + b.WriteString(KvSpaceSeparator) + } + b.WriteString(cs.SubSys) + if cs.Target != Default { + b.WriteString(SubSystemSeparator) + b.WriteString(cs.Target) + } + b.WriteString(KvSpaceSeparator) + b.WriteString(cs.Params.String()) + b.WriteString(KvNewline) +}