MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-10-29 15:55:00 -04:00
Code Issues Packages Projects Releases Wiki Activity

config: Deprecated delay/max_wait/scanner and introduce speed (#15941)

Browse Source
This commit is contained in:
Anis Elleuch 2022-11-01 16:04:07 +01:00 committed by GitHub
parent fd6f6fc8df
commit 7721595aa9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 101 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/admin-handlers-config-kv.go
View File

@ -225,7 +225,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
var s strings.Builder
for _, subSysConfig := range subSysConfigs {
subSysConfig.AddString(&s, false)
subSysConfig.WriteTo(&s, false)
}
password := cred.SecretKey
@ -454,10 +454,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
var s strings.Builder
hkvs := config.HelpSubSysMap[""]
var count int
for _, hkv := range hkvs {
count += len(cfg[hkv.Key])
}
for _, hkv := range hkvs {
// We ignore the error below, as we cannot get one.
cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
@ -482,7 +478,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
case config.IdentityPluginSubSys:
off = !idplugin.Enabled(item.Config)
}
item.AddString(&s, off)
item.WriteTo(&s, off)
}
}

99
internal/config/config.go
View File

@ -232,6 +232,23 @@ func RegisterHelpDeprecatedSubSys(helpDeprecatedKVMap map[string]HelpKV) {
type KV struct {
Key string `json:"key"`
Value string `json:"value"`
Deprecated bool `json:"-"`
}
func (kv KV) String() string {
var s strings.Builder
s.WriteString(kv.Key)
s.WriteString(KvSeparator)
spc := madmin.HasSpace(kv.Value)
if spc {
s.WriteString(KvDoubleQuote)
}
s.WriteString(kv.Value)
if spc {
s.WriteString(KvDoubleQuote)
}
return s.String()
}
// KVS - is a shorthand for some wrapper functions
@ -277,20 +294,7 @@ func (kvs KVS) Keys() []string {
func (kvs KVS) String() string {
var s strings.Builder
for _, kv := range kvs {
// Do not need to print if state is on
if kv.Key == Enable && kv.Value == EnableOn {
continue
}
s.WriteString(kv.Key)
s.WriteString(KvSeparator)
spc := madmin.HasSpace(kv.Value)
if spc {
s.WriteString(KvDoubleQuote)
}
s.WriteString(kv.Value)
if spc {
s.WriteString(KvDoubleQuote)
}
s.WriteString(kv.String())
s.WriteString(KvSpaceSeparator)
}
return s.String()
@ -349,6 +353,16 @@ func (kvs *KVS) Delete(key string) {
}
}
// LookupKV returns the KV by its key
func (kvs KVS) LookupKV(key string) (KV, bool) {
for _, kv := range kvs {
if kv.Key == key {
return kv, true
}
}
return KV{}, false
}
// Lookup - lookup a key in a list of KVS
func (kvs KVS) Lookup(key string) (string, bool) {
for _, kv := range kvs {
@ -429,39 +443,6 @@ func (c Config) RedactSensitiveInfo() Config {
return nc
}
type configWriteTo struct {
Config
filterByKey string
}
// NewConfigWriteTo - returns a struct which
// allows for serializing the config/kv struct
// to a io.WriterTo
func NewConfigWriteTo(cfg Config, key string) io.WriterTo {
return &configWriteTo{Config: cfg, filterByKey: key}
}
// WriteTo - implements io.WriterTo interface implementation for config.
func (c *configWriteTo) WriteTo(w io.Writer) (int64, error) {
kvsTargets, err := c.GetKVS(c.filterByKey, DefaultKVS)
if err != nil {
return 0, err
}
var n int
for _, target := range kvsTargets {
m1, _ := w.Write([]byte(target.SubSystem))
m2, _ := w.Write([]byte(KvSpaceSeparator))
m3, _ := w.Write([]byte(target.KVS.String()))
if len(kvsTargets) > 1 {
m4, _ := w.Write([]byte(KvNewline))
n += m1 + m2 + m3 + m4
} else {
n += m1 + m2 + m3
}
}
return int64(n), nil
}
// Default KV configs for worm and region
var (
DefaultCredentialKVS = KVS{
@ -1301,10 +1282,10 @@ func (cs *SubsysInfo) AddEnvString(b *strings.Builder) {
}
}
// AddString adds the string representation of the configuration to the given
// WriteTo writes the string representation of the configuration to the given
// builder. When off is true, adds a comment character before the config system
// output.
func (cs *SubsysInfo) AddString(b *strings.Builder, off bool) {
// output. It also ignores values when empty and deprecated.
func (cs *SubsysInfo) WriteTo(b *strings.Builder, off bool) {
cs.AddEnvString(b)
if off {
b.WriteString(KvComment)
@ -1316,6 +1297,22 @@ func (cs *SubsysInfo) AddString(b *strings.Builder, off bool) {
b.WriteString(cs.Target)
}
b.WriteString(KvSpaceSeparator)
b.WriteString(cs.Config.String())
for _, kv := range cs.Config {
dkv, ok := cs.Defaults.LookupKV(kv.Key)
if !ok {
continue
}
// Ignore empty and deprecated values
if dkv.Deprecated && kv.Value == "" {
continue
}
// Do not need to print if state is on
if kv.Key == Enable && kv.Value == EnableOn {
continue
}
b.WriteString(kv.String())
b.WriteString(KvSpaceSeparator)
}
b.WriteString(KvNewline)
}

18
internal/config/scanner/help.go
View File

@ -27,22 +27,10 @@ var (
// Help provides help for config values
Help = config.HelpKVS{
config.HelpKV{
Key: Delay,
Description: `scanner delay multiplier` + defaultHelpPostfix(Delay),
Key: Speed,
Description: `scanner speed` + defaultHelpPostfix(Speed),
Optional: true,
Type: "float",
},
config.HelpKV{
Key: MaxWait,
Description: `maximum wait time between operations` + defaultHelpPostfix(MaxWait),
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: Cycle,
Description: `time duration between scanner cycles` + defaultHelpPostfix(Cycle),
Optional: true,
Type: "duration",
Type: "default|slowest|slow|fast|fastest",
},
}
)

58
internal/config/scanner/scanner.go
View File

@ -18,6 +18,7 @@
package scanner
import (
"fmt"
"strconv"
"time"
@ -27,10 +28,14 @@ import (
// Compression environment variables
const (
Delay = "delay"
MaxWait = "max_wait"
Cycle = "cycle"
Speed = "speed"
EnvSpeed = "MINIO_SCANNER_SPEED"
// All below are deprecated in October 2022 and
// replaced them with a single speed parameter
Delay = "delay"
MaxWait = "max_wait"
Cycle = "cycle"
EnvDelay = "MINIO_SCANNER_DELAY"
EnvCycle = "MINIO_SCANNER_CYCLE"
EnvDelayLegacy = "MINIO_CRAWLER_DELAY"
@ -51,16 +56,26 @@ type Config struct {
// DefaultKVS - default KV config for heal settings
var DefaultKVS = config.KVS{
config.KV{
Key: Delay,
Value: "2",
Key: Speed,
Value: "default",
},
// Deprecated Oct 2022
config.KV{
Key: MaxWait,
Value: "5s",
Key: Delay,
Value: "",
Deprecated: true,
},
// Deprecated Oct 2022
config.KV{
Key: Cycle,
Value: "1m",
Key: MaxWait,
Value: "",
Deprecated: true,
},
// Deprecated Oct 2022
config.KV{
Key: Cycle,
Value: "",
Deprecated: true,
},
}
@ -69,6 +84,30 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
if err = config.CheckValidKeys(config.ScannerSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
// Stick to loading deprecated config/env if they are already set
if kvs.Get(Delay) != "" && kvs.Get(MaxWait) != "" && kvs.Get(Cycle) != "" {
return lookupDeprecatedScannerConfig(kvs)
}
switch speed := env.Get(EnvSpeed, kvs.GetWithDefault(Speed, DefaultKVS)); speed {
case "fastest":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 0, 0, 0
case "fast":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 1, 100*time.Millisecond, time.Minute
case "default":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 2, 5*time.Second, time.Minute
case "slow":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 10, 15*time.Second, time.Minute
case "slowest":
cfg.Delay, cfg.MaxWait, cfg.Cycle = 100, 15*time.Second, 30*time.Minute
default:
return cfg, fmt.Errorf("unknown '%s' value", speed)
}
return
}
func lookupDeprecatedScannerConfig(kvs config.KVS) (cfg Config, err error) {
delay := env.Get(EnvDelayLegacy, "")
if delay == "" {
delay = env.Get(EnvDelay, kvs.GetWithDefault(Delay, DefaultKVS))
@ -85,7 +124,6 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
if err != nil {
return cfg, err
}
cfg.Cycle, err = time.ParseDuration(env.Get(EnvCycle, kvs.GetWithDefault(Cycle, DefaultKVS)))
if err != nil {
return cfg, err