fix: allow configuring excess versions alerting (#19028)

Bonus: enable audit alerts for object versions
beyond the configured value, default is '100'
versions per object beyond which scanner will
alert for each such objects.
This commit is contained in:
Harshavardhana
2024-02-11 23:41:53 -08:00
committed by GitHub
parent e3fbac9e24
commit afd19de5a9
7 changed files with 114 additions and 20 deletions

View File

@@ -28,9 +28,21 @@ var (
Help = config.HelpKVS{
config.HelpKV{
Key: Speed,
Description: `scanner speed` + defaultHelpPostfix(Speed),
Description: `customize scanner speed (default|slowest|slow|fast|fastest)` + defaultHelpPostfix(Speed),
Optional: true,
Type: "default|slowest|slow|fast|fastest",
Type: "string",
},
config.HelpKV{
Key: ExcessVersions,
Description: `alert per object beyond this many versions` + defaultHelpPostfix(ExcessVersions),
Optional: true,
Type: "int",
},
config.HelpKV{
Key: ExcessFolders,
Description: `alert beyond this many sub-folders per folder in an erasure set` + defaultHelpPostfix(ExcessFolders),
Optional: true,
Type: "int",
},
}
)

View File

@@ -34,6 +34,12 @@ const (
IdleSpeed = "idle_speed"
EnvIdleSpeed = "MINIO_SCANNER_IDLE_SPEED"
ExcessVersions = "alert_excess_versions"
EnvExcessVersions = "MINIO_SCANNER_ALERT_EXCESS_VERSIONS"
ExcessFolders = "alert_excess_folders"
EnvExcessFolders = "MINIO_SCANNER_ALERT_EXCESS_FOLDERS"
// All below are deprecated in October 2022 and
// replaced them with a single speed parameter
Delay = "delay"
@@ -50,8 +56,16 @@ const (
type Config struct {
// Delay is the sleep multiplier.
Delay float64 `json:"delay"`
// Behavior of the scanner when there is no other parallel S3 requests
IdleMode int32 // 0 => throttling, 1 => full speed
// Sleep always or based on incoming S3 requests.
IdleMode int32 // 0 => on, 1 => off
// Alert upon this many excess object versions
ExcessVersions int64 // 100
// Alert upon this many excess sub-folders per folder in an erasure set.
ExcessFolders int64 // 50000
// MaxWait is maximum wait time between operations
MaxWait time.Duration
// Cycle is the time.Duration between each scanner cycles
@@ -69,6 +83,15 @@ var DefaultKVS = config.KVS{
Value: "",
HiddenIfEmpty: true,
},
config.KV{
Key: ExcessVersions,
Value: "100",
},
config.KV{
Key: ExcessFolders,
Value: "50000",
},
// Deprecated Oct 2022
config.KV{
Key: Delay,
@@ -119,15 +142,27 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
}
switch idleSpeed := env.Get(EnvIdleSpeed, kvs.GetWithDefault(IdleSpeed, DefaultKVS)); idleSpeed {
case "", "throttled": // Empty is the default mode;
case "", config.EnableOn:
cfg.IdleMode = 0
case "full":
case config.EnableOff:
cfg.IdleMode = 1
default:
return cfg, fmt.Errorf("unknown value: '%s'", idleSpeed)
}
return
excessVersions, err := strconv.ParseInt(env.Get(EnvExcessVersions, kvs.GetWithDefault(ExcessVersions, DefaultKVS)), 10, 64)
if err != nil {
return cfg, err
}
cfg.ExcessVersions = excessVersions
excessFolders, err := strconv.ParseInt(env.Get(EnvExcessFolders, kvs.GetWithDefault(ExcessFolders, DefaultKVS)), 10, 64)
if err != nil {
return cfg, err
}
cfg.ExcessFolders = excessFolders
return cfg, nil
}
func lookupDeprecatedScannerConfig(kvs config.KVS) (cfg Config, err error) {