config: Deprecated delay/max_wait/scanner and introduce speed (#15941)

cmd/admin-handlers-config-kv.go

@@ -225,7 +225,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	var s strings.Builder
 	for _, subSysConfig := range subSysConfigs {
-		subSysConfig.AddString(&s, false)
+		subSysConfig.WriteTo(&s, false)
 	}
 
 	password := cred.SecretKey
@@ -454,10 +454,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	var s strings.Builder
 	hkvs := config.HelpSubSysMap[""]
-	var count int
-	for _, hkv := range hkvs {
-		count += len(cfg[hkv.Key])
-	}
 	for _, hkv := range hkvs {
 		// We ignore the error below, as we cannot get one.
 		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
@@ -482,7 +478,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 			case config.IdentityPluginSubSys:
 				off = !idplugin.Enabled(item.Config)
 			}
-			item.AddString(&s, off)
+			item.WriteTo(&s, off)
 		}
 	}
 

internal/config/config.go

@@ -232,6 +232,23 @@ func RegisterHelpDeprecatedSubSys(helpDeprecatedKVMap map[string]HelpKV) {
 type KV struct {
 	Key   string `json:"key"`
 	Value string `json:"value"`
+
+	Deprecated bool `json:"-"`
+}
+
+func (kv KV) String() string {
+	var s strings.Builder
+	s.WriteString(kv.Key)
+	s.WriteString(KvSeparator)
+	spc := madmin.HasSpace(kv.Value)
+	if spc {
+		s.WriteString(KvDoubleQuote)
+	}
+	s.WriteString(kv.Value)
+	if spc {
+		s.WriteString(KvDoubleQuote)
+	}
+	return s.String()
 }
 
 // KVS - is a shorthand for some wrapper functions
@@ -277,20 +294,7 @@ func (kvs KVS) Keys() []string {
 func (kvs KVS) String() string {
 	var s strings.Builder
 	for _, kv := range kvs {
-		// Do not need to print if state is on
+		s.WriteString(kv.String())
-		if kv.Key == Enable && kv.Value == EnableOn {
-			continue
-		}
-		s.WriteString(kv.Key)
-		s.WriteString(KvSeparator)
-		spc := madmin.HasSpace(kv.Value)
-		if spc {
-			s.WriteString(KvDoubleQuote)
-		}
-		s.WriteString(kv.Value)
-		if spc {
-			s.WriteString(KvDoubleQuote)
-		}
 		s.WriteString(KvSpaceSeparator)
 	}
 	return s.String()
@@ -349,6 +353,16 @@ func (kvs *KVS) Delete(key string) {
 	}
 }
 
+// LookupKV returns the KV by its key
+func (kvs KVS) LookupKV(key string) (KV, bool) {
+	for _, kv := range kvs {
+		if kv.Key == key {
+			return kv, true
+		}
+	}
+	return KV{}, false
+}
+
 // Lookup - lookup a key in a list of KVS
 func (kvs KVS) Lookup(key string) (string, bool) {
 	for _, kv := range kvs {
@@ -429,39 +443,6 @@ func (c Config) RedactSensitiveInfo() Config {
 	return nc
 }
 
-type configWriteTo struct {
-	Config
-	filterByKey string
-}
-
-// NewConfigWriteTo - returns a struct which
-// allows for serializing the config/kv struct
-// to a io.WriterTo
-func NewConfigWriteTo(cfg Config, key string) io.WriterTo {
-	return &configWriteTo{Config: cfg, filterByKey: key}
-}
-
-// WriteTo - implements io.WriterTo interface implementation for config.
-func (c *configWriteTo) WriteTo(w io.Writer) (int64, error) {
-	kvsTargets, err := c.GetKVS(c.filterByKey, DefaultKVS)
-	if err != nil {
-		return 0, err
-	}
-	var n int
-	for _, target := range kvsTargets {
-		m1, _ := w.Write([]byte(target.SubSystem))
-		m2, _ := w.Write([]byte(KvSpaceSeparator))
-		m3, _ := w.Write([]byte(target.KVS.String()))
-		if len(kvsTargets) > 1 {
-			m4, _ := w.Write([]byte(KvNewline))
-			n += m1 + m2 + m3 + m4
-		} else {
-			n += m1 + m2 + m3
-		}
-	}
-	return int64(n), nil
-}
-
 // Default KV configs for worm and region
 var (
 	DefaultCredentialKVS = KVS{
@@ -1301,10 +1282,10 @@ func (cs *SubsysInfo) AddEnvString(b *strings.Builder) {
 	}
 }
 
-// AddString adds the string representation of the configuration to the given
+// WriteTo writes the string representation of the configuration to the given
 // builder. When off is true, adds a comment character before the config system
-// output.
+// output. It also ignores values when empty and deprecated.
-func (cs *SubsysInfo) AddString(b *strings.Builder, off bool) {
+func (cs *SubsysInfo) WriteTo(b *strings.Builder, off bool) {
 	cs.AddEnvString(b)
 	if off {
 		b.WriteString(KvComment)
@@ -1316,6 +1297,22 @@ func (cs *SubsysInfo) AddString(b *strings.Builder, off bool) {
 		b.WriteString(cs.Target)
 	}
 	b.WriteString(KvSpaceSeparator)
-	b.WriteString(cs.Config.String())
+	for _, kv := range cs.Config {
+		dkv, ok := cs.Defaults.LookupKV(kv.Key)
+		if !ok {
+			continue
+		}
+		// Ignore empty and deprecated values
+		if dkv.Deprecated && kv.Value == "" {
+			continue
+		}
+		// Do not need to print if state is on
+		if kv.Key == Enable && kv.Value == EnableOn {
+			continue
+		}
+		b.WriteString(kv.String())
+		b.WriteString(KvSpaceSeparator)
+	}
+
 	b.WriteString(KvNewline)
 }

internal/config/scanner/help.go

@@ -27,22 +27,10 @@ var (
 	// Help provides help for config values
 	Help = config.HelpKVS{
 		config.HelpKV{
-			Key:         Delay,
+			Key:         Speed,
-			Description: `scanner delay multiplier` + defaultHelpPostfix(Delay),
+			Description: `scanner speed` + defaultHelpPostfix(Speed),
 			Optional:    true,
-			Type:        "float",
+			Type:        "default|slowest|slow|fast|fastest",
-		},
-		config.HelpKV{
-			Key:         MaxWait,
-			Description: `maximum wait time between operations` + defaultHelpPostfix(MaxWait),
-			Optional:    true,
-			Type:        "duration",
-		},
-		config.HelpKV{
-			Key:         Cycle,
-			Description: `time duration between scanner cycles` + defaultHelpPostfix(Cycle),
-			Optional:    true,
-			Type:        "duration",
 		},
 	}
 )

internal/config/scanner/scanner.go

@@ -18,6 +18,7 @@
 package scanner
 
 import (
+	"fmt"
 	"strconv"
 	"time"
 
@@ -27,10 +28,14 @@ import (
 
 // Compression environment variables
 const (
-	Delay   = "delay"
+	Speed    = "speed"
-	MaxWait = "max_wait"
+	EnvSpeed = "MINIO_SCANNER_SPEED"
-	Cycle   = "cycle"
 
+	// All below are deprecated in October 2022 and
+	// replaced them with a single speed parameter
+	Delay            = "delay"
+	MaxWait          = "max_wait"
+	Cycle            = "cycle"
 	EnvDelay         = "MINIO_SCANNER_DELAY"
 	EnvCycle         = "MINIO_SCANNER_CYCLE"
 	EnvDelayLegacy   = "MINIO_CRAWLER_DELAY"
@@ -51,16 +56,26 @@ type Config struct {
 // DefaultKVS - default KV config for heal settings
 var DefaultKVS = config.KVS{
 	config.KV{
-		Key:   Delay,
+		Key:   Speed,
-		Value: "2",
+		Value: "default",
 	},
+	// Deprecated Oct 2022
 	config.KV{
-		Key:   MaxWait,
+		Key:        Delay,
-		Value: "5s",
+		Value:      "",
+		Deprecated: true,
 	},
+	// Deprecated Oct 2022
 	config.KV{
-		Key:   Cycle,
+		Key:        MaxWait,
-		Value: "1m",
+		Value:      "",
+		Deprecated: true,
+	},
+	// Deprecated Oct 2022
+	config.KV{
+		Key:        Cycle,
+		Value:      "",
+		Deprecated: true,
 	},
 }
 
@@ -69,6 +84,30 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	if err = config.CheckValidKeys(config.ScannerSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
+
+	// Stick to loading deprecated config/env if they are already set
+	if kvs.Get(Delay) != "" && kvs.Get(MaxWait) != "" && kvs.Get(Cycle) != "" {
+		return lookupDeprecatedScannerConfig(kvs)
+	}
+
+	switch speed := env.Get(EnvSpeed, kvs.GetWithDefault(Speed, DefaultKVS)); speed {
+	case "fastest":
+		cfg.Delay, cfg.MaxWait, cfg.Cycle = 0, 0, 0
+	case "fast":
+		cfg.Delay, cfg.MaxWait, cfg.Cycle = 1, 100*time.Millisecond, time.Minute
+	case "default":
+		cfg.Delay, cfg.MaxWait, cfg.Cycle = 2, 5*time.Second, time.Minute
+	case "slow":
+		cfg.Delay, cfg.MaxWait, cfg.Cycle = 10, 15*time.Second, time.Minute
+	case "slowest":
+		cfg.Delay, cfg.MaxWait, cfg.Cycle = 100, 15*time.Second, 30*time.Minute
+	default:
+		return cfg, fmt.Errorf("unknown '%s' value", speed)
+	}
+	return
+}
+
+func lookupDeprecatedScannerConfig(kvs config.KVS) (cfg Config, err error) {
 	delay := env.Get(EnvDelayLegacy, "")
 	if delay == "" {
 		delay = env.Get(EnvDelay, kvs.GetWithDefault(Delay, DefaultKVS))
@@ -85,7 +124,6 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	if err != nil {
 		return cfg, err
 	}
-
 	cfg.Cycle, err = time.ParseDuration(env.Get(EnvCycle, kvs.GetWithDefault(Cycle, DefaultKVS)))
 	if err != nil {
 		return cfg, err