2021-04-18 15:41:13 -04:00
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
2019-10-23 01:59:13 -04:00
package config
import (
2019-11-27 12:36:08 -05:00
"bufio"
2019-10-23 01:59:13 -04:00
"fmt"
2019-11-27 12:36:08 -05:00
"io"
2019-11-14 17:19:57 -05:00
"regexp"
2019-10-23 01:59:13 -04:00
"strings"
2021-05-06 11:52:02 -04:00
"github.com/minio/madmin-go"
2020-07-14 12:38:05 -04:00
"github.com/minio/minio-go/v7/pkg/set"
2021-06-01 17:59:40 -04:00
"github.com/minio/minio/internal/auth"
2021-05-28 18:17:01 -04:00
"github.com/minio/pkg/env"
2019-10-23 01:59:13 -04:00
)
// Error config error type
2019-12-04 18:32:37 -05:00
type Error struct {
2019-12-14 20:27:57 -05:00
Err string
2019-12-04 18:32:37 -05:00
}
2019-10-31 02:39:09 -04:00
// Errorf - formats according to a format specifier and returns
// the string as a value that satisfies error of type config.Error
2019-12-14 20:27:57 -05:00
func Errorf ( format string , a ... interface { } ) error {
return Error { Err : fmt . Sprintf ( format , a ... ) }
2019-10-31 02:39:09 -04:00
}
2019-10-23 01:59:13 -04:00
func ( e Error ) Error ( ) string {
2019-12-04 18:32:37 -05:00
return e . Err
2019-10-23 01:59:13 -04:00
}
// Default keys
const (
2019-11-05 09:18:26 -05:00
Default = madmin . Default
2019-12-04 18:32:37 -05:00
Enable = madmin . EnableKey
2019-12-03 13:50:20 -05:00
Comment = madmin . CommentKey
2019-10-23 01:59:13 -04:00
2019-12-04 18:32:37 -05:00
// Enable values
EnableOn = madmin . EnableOn
EnableOff = madmin . EnableOff
2019-10-23 01:59:13 -04:00
2021-11-25 16:06:25 -05:00
RegionKey = "region"
NameKey = "name"
2019-10-23 01:59:13 -04:00
RegionName = "name"
AccessKey = "access_key"
SecretKey = "secret_key"
2021-12-03 12:32:11 -05:00
License = "license" // Deprecated Dec 2021
APIKey = "api_key"
2022-02-01 12:52:38 -05:00
Proxy = "proxy"
2019-10-23 01:59:13 -04:00
)
// Top level config constants.
const (
2019-11-13 20:38:05 -05:00
CredentialsSubSys = "credentials"
PolicyOPASubSys = "policy_opa"
IdentityOpenIDSubSys = "identity_openid"
IdentityLDAPSubSys = "identity_ldap"
2021-09-07 22:03:48 -04:00
IdentityTLSSubSys = "identity_tls"
2019-11-13 20:38:05 -05:00
CacheSubSys = "cache"
2021-11-25 16:06:25 -05:00
SiteSubSys = "site"
2019-11-13 20:38:05 -05:00
RegionSubSys = "region"
EtcdSubSys = "etcd"
2019-12-04 18:32:37 -05:00
StorageClassSubSys = "storage_class"
2020-04-14 15:46:37 -04:00
APISubSys = "api"
2019-11-13 20:38:05 -05:00
CompressionSubSys = "compression"
LoggerWebhookSubSys = "logger_webhook"
AuditWebhookSubSys = "audit_webhook"
2021-07-13 12:39:13 -04:00
AuditKafkaSubSys = "audit_kafka"
2020-10-14 16:51:51 -04:00
HealSubSys = "heal"
2021-02-17 15:04:11 -05:00
ScannerSubSys = "scanner"
2020-12-04 12:32:35 -05:00
CrawlerSubSys = "crawler"
2021-09-15 00:54:25 -04:00
SubnetSubSys = "subnet"
2019-10-23 01:59:13 -04:00
// Add new constants here if you add new fields to config.
)
// Notification config constants.
const (
NotifyKafkaSubSys = "notify_kafka"
NotifyMQTTSubSys = "notify_mqtt"
NotifyMySQLSubSys = "notify_mysql"
NotifyNATSSubSys = "notify_nats"
NotifyNSQSubSys = "notify_nsq"
NotifyESSubSys = "notify_elasticsearch"
NotifyAMQPSubSys = "notify_amqp"
NotifyPostgresSubSys = "notify_postgres"
NotifyRedisSubSys = "notify_redis"
NotifyWebhookSubSys = "notify_webhook"
// Add new constants here if you add new fields to config.
)
2022-02-08 13:36:41 -05:00
// NotifySubSystems - all notification sub-systems
var NotifySubSystems = set . CreateStringSet (
NotifyKafkaSubSys ,
NotifyMQTTSubSys ,
NotifyMySQLSubSys ,
NotifyNATSSubSys ,
NotifyNSQSubSys ,
NotifyESSubSys ,
NotifyAMQPSubSys ,
NotifyPostgresSubSys ,
NotifyRedisSubSys ,
NotifyWebhookSubSys ,
)
// LoggerSubSystems - all sub-systems related to logger
var LoggerSubSystems = set . CreateStringSet (
LoggerWebhookSubSys ,
AuditWebhookSubSys ,
AuditKafkaSubSys ,
)
2019-10-23 01:59:13 -04:00
// SubSystems - all supported sub-systems
2020-12-04 12:32:35 -05:00
var SubSystems = set . CreateStringSet (
2019-10-23 01:59:13 -04:00
CredentialsSubSys ,
2021-11-25 16:06:25 -05:00
SiteSubSys ,
2019-10-23 01:59:13 -04:00
RegionSubSys ,
2019-10-30 03:04:39 -04:00
EtcdSubSys ,
2019-10-23 01:59:13 -04:00
CacheSubSys ,
2020-04-14 15:46:37 -04:00
APISubSys ,
2019-10-23 01:59:13 -04:00
StorageClassSubSys ,
CompressionSubSys ,
2019-11-13 20:38:05 -05:00
LoggerWebhookSubSys ,
AuditWebhookSubSys ,
2021-07-13 12:39:13 -04:00
AuditKafkaSubSys ,
2019-10-23 01:59:13 -04:00
PolicyOPASubSys ,
IdentityLDAPSubSys ,
IdentityOpenIDSubSys ,
2021-09-07 22:03:48 -04:00
IdentityTLSSubSys ,
2021-02-17 15:04:11 -05:00
ScannerSubSys ,
2020-10-14 16:51:51 -04:00
HealSubSys ,
2019-10-23 01:59:13 -04:00
NotifyAMQPSubSys ,
NotifyESSubSys ,
NotifyKafkaSubSys ,
NotifyMQTTSubSys ,
NotifyMySQLSubSys ,
NotifyNATSSubSys ,
NotifyNSQSubSys ,
NotifyPostgresSubSys ,
NotifyRedisSubSys ,
NotifyWebhookSubSys ,
2021-09-15 00:54:25 -04:00
SubnetSubSys ,
2020-12-04 12:32:35 -05:00
)
// SubSystemsDynamic - all sub-systems that have dynamic config.
var SubSystemsDynamic = set . CreateStringSet (
APISubSys ,
CompressionSubSys ,
2021-02-17 15:04:11 -05:00
ScannerSubSys ,
2020-12-04 12:32:35 -05:00
HealSubSys ,
2021-09-15 00:54:25 -04:00
SubnetSubSys ,
2022-02-17 14:11:15 -05:00
LoggerWebhookSubSys ,
2020-12-04 12:32:35 -05:00
)
2019-10-23 01:59:13 -04:00
// SubSystemsSingleTargets - subsystems which only support single target.
var SubSystemsSingleTargets = set . CreateStringSet ( [ ] string {
CredentialsSubSys ,
2021-11-25 16:06:25 -05:00
SiteSubSys ,
2019-10-23 01:59:13 -04:00
RegionSubSys ,
2019-10-30 03:04:39 -04:00
EtcdSubSys ,
2019-10-23 01:59:13 -04:00
CacheSubSys ,
2020-04-14 15:46:37 -04:00
APISubSys ,
2019-10-23 01:59:13 -04:00
StorageClassSubSys ,
CompressionSubSys ,
PolicyOPASubSys ,
IdentityLDAPSubSys ,
IdentityOpenIDSubSys ,
2021-09-07 22:03:48 -04:00
IdentityTLSSubSys ,
2020-10-14 16:51:51 -04:00
HealSubSys ,
2021-02-17 15:04:11 -05:00
ScannerSubSys ,
2019-10-23 01:59:13 -04:00
} ... )
// Constant separators
const (
2019-11-05 09:18:26 -05:00
SubSystemSeparator = madmin . SubSystemSeparator
KvSeparator = madmin . KvSeparator
KvSpaceSeparator = madmin . KvSpaceSeparator
2019-12-03 13:50:20 -05:00
KvComment = madmin . KvComment
2019-11-05 09:18:26 -05:00
KvNewline = madmin . KvNewline
KvDoubleQuote = madmin . KvDoubleQuote
KvSingleQuote = madmin . KvSingleQuote
2019-10-30 03:04:39 -04:00
// Env prefix used for all envs in MinIO
EnvPrefix = "MINIO_"
EnvWordDelimiter = ` _ `
2019-10-23 01:59:13 -04:00
)
2019-11-27 12:36:08 -05:00
// DefaultKVS - default kvs for all sub-systems
var DefaultKVS map [ string ] KVS
// RegisterDefaultKVS - this function saves input kvsMap
// globally, this should be called only once preferably
// during `init()`.
func RegisterDefaultKVS ( kvsMap map [ string ] KVS ) {
DefaultKVS = map [ string ] KVS { }
for subSys , kvs := range kvsMap {
DefaultKVS [ subSys ] = kvs
}
}
// HelpSubSysMap - help for all individual KVS for each sub-systems
// also carries a special empty sub-system which dumps
// help for each sub-system key.
var HelpSubSysMap map [ string ] HelpKVS
// RegisterHelpSubSys - this function saves
// input help KVS for each sub-system globally,
// this function should be called only once
// preferably in during `init()`.
func RegisterHelpSubSys ( helpKVSMap map [ string ] HelpKVS ) {
HelpSubSysMap = map [ string ] HelpKVS { }
for subSys , hkvs := range helpKVSMap {
HelpSubSysMap [ subSys ] = hkvs
}
}
2021-11-25 16:06:25 -05:00
// HelpDeprecatedSubSysMap - help for all deprecated sub-systems, that may be
// removed in the future.
var HelpDeprecatedSubSysMap map [ string ] HelpKV
// RegisterHelpDeprecatedSubSys - saves input help KVS for deprecated
// sub-systems globally. Should be called only once at init.
func RegisterHelpDeprecatedSubSys ( helpDeprecatedKVMap map [ string ] HelpKV ) {
HelpDeprecatedSubSysMap = map [ string ] HelpKV { }
for k , v := range helpDeprecatedKVMap {
HelpDeprecatedSubSysMap [ k ] = v
}
}
2019-11-20 18:10:24 -05:00
// KV - is a shorthand of each key value.
type KV struct {
Key string ` json:"key" `
Value string ` json:"value" `
}
2019-10-23 01:59:13 -04:00
// KVS - is a shorthand for some wrapper functions
// to operate on list of key values.
2019-11-20 18:10:24 -05:00
type KVS [ ] KV
2019-10-23 01:59:13 -04:00
2019-11-13 20:38:05 -05:00
// Empty - return if kv is empty
func ( kvs KVS ) Empty ( ) bool {
return len ( kvs ) == 0
}
2021-11-10 13:01:32 -05:00
// Clone - returns a copy of the KVS
func ( kvs KVS ) Clone ( ) KVS {
return append ( make ( KVS , 0 , len ( kvs ) ) , kvs ... )
}
// GetWithDefault - returns default value if key not set
func ( kvs KVS ) GetWithDefault ( key string , defaultKVS KVS ) string {
v := kvs . Get ( key )
if len ( v ) == 0 {
return defaultKVS . Get ( key )
}
return v
}
2020-04-10 00:45:17 -04:00
// Keys returns the list of keys for the current KVS
func ( kvs KVS ) Keys ( ) [ ] string {
2022-01-02 12:15:06 -05:00
keys := make ( [ ] string , len ( kvs ) )
2020-04-10 14:44:28 -04:00
var foundComment bool
2020-04-10 00:45:17 -04:00
for i := range kvs {
2020-04-10 14:44:28 -04:00
if kvs [ i ] . Key == madmin . CommentKey {
foundComment = true
}
2020-04-10 00:45:17 -04:00
keys [ i ] = kvs [ i ] . Key
}
2020-04-10 14:44:28 -04:00
// Comment KV not found, add it explicitly.
if ! foundComment {
keys = append ( keys , madmin . CommentKey )
}
2020-04-10 00:45:17 -04:00
return keys
}
2019-10-23 01:59:13 -04:00
func ( kvs KVS ) String ( ) string {
var s strings . Builder
2019-11-20 18:10:24 -05:00
for _ , kv := range kvs {
2019-11-13 20:38:05 -05:00
// Do not need to print if state is on
2019-12-04 18:32:37 -05:00
if kv . Key == Enable && kv . Value == EnableOn {
2019-11-13 20:38:05 -05:00
continue
}
2019-11-20 18:10:24 -05:00
s . WriteString ( kv . Key )
2019-10-23 01:59:13 -04:00
s . WriteString ( KvSeparator )
2019-12-04 18:32:37 -05:00
spc := madmin . HasSpace ( kv . Value )
if spc {
s . WriteString ( KvDoubleQuote )
}
2019-11-20 18:10:24 -05:00
s . WriteString ( kv . Value )
2019-12-04 18:32:37 -05:00
if spc {
s . WriteString ( KvDoubleQuote )
}
2019-10-23 01:59:13 -04:00
s . WriteString ( KvSpaceSeparator )
}
return s . String ( )
}
2021-07-13 12:39:13 -04:00
// Merge environment values with on disk KVS, environment values overrides
// anything on the disk.
func Merge ( cfgKVS map [ string ] KVS , envname string , defaultKVS KVS ) map [ string ] KVS {
newCfgKVS := make ( map [ string ] KVS )
for _ , e := range env . List ( envname ) {
tgt := strings . TrimPrefix ( e , envname + Default )
if tgt == envname {
tgt = Default
}
newCfgKVS [ tgt ] = defaultKVS
}
for tgt , kv := range cfgKVS {
newCfgKVS [ tgt ] = kv
}
return newCfgKVS
}
2019-12-04 18:32:37 -05:00
// Set sets a value, if not sets a default value.
func ( kvs * KVS ) Set ( key , value string ) {
for i , kv := range * kvs {
if kv . Key == key {
( * kvs ) [ i ] = KV {
Key : key ,
Value : value ,
}
return
}
}
* kvs = append ( * kvs , KV {
Key : key ,
Value : value ,
} )
}
2019-10-23 01:59:13 -04:00
// Get - returns the value of a key, if not found returns empty.
func ( kvs KVS ) Get ( key string ) string {
2019-11-20 18:10:24 -05:00
v , ok := kvs . Lookup ( key )
if ok {
return v
}
return ""
}
2020-09-23 12:14:33 -04:00
// Delete - deletes the key if present from the KV list.
func ( kvs * KVS ) Delete ( key string ) {
for i , kv := range * kvs {
if kv . Key == key {
* kvs = append ( ( * kvs ) [ : i ] , ( * kvs ) [ i + 1 : ] ... )
return
}
}
}
2019-11-20 18:10:24 -05:00
// Lookup - lookup a key in a list of KVS
func ( kvs KVS ) Lookup ( key string ) ( string , bool ) {
for _ , kv := range kvs {
if kv . Key == key {
return kv . Value , true
}
}
return "" , false
2019-10-23 01:59:13 -04:00
}
// Config - MinIO server config structure.
type Config map [ string ] map [ string ] KVS
2019-11-27 12:36:08 -05:00
// DelFrom - deletes all keys in the input reader.
func ( c Config ) DelFrom ( r io . Reader ) error {
scanner := bufio . NewScanner ( r )
for scanner . Scan ( ) {
// Skip any empty lines, or comment like characters
2019-12-03 13:50:20 -05:00
text := scanner . Text ( )
if text == "" || strings . HasPrefix ( text , KvComment ) {
2019-11-27 12:36:08 -05:00
continue
}
2019-12-03 13:50:20 -05:00
if err := c . DelKVS ( text ) ; err != nil {
2019-11-27 12:36:08 -05:00
return err
}
}
2021-05-25 17:17:33 -04:00
return scanner . Err ( )
2019-11-27 12:36:08 -05:00
}
2020-12-04 12:32:35 -05:00
// ReadConfig - read content from input and write into c.
// Returns whether all parameters were dynamic.
func ( c Config ) ReadConfig ( r io . Reader ) ( dynOnly bool , err error ) {
2019-11-27 12:36:08 -05:00
var n int
scanner := bufio . NewScanner ( r )
2020-12-04 12:32:35 -05:00
dynOnly = true
2019-11-27 12:36:08 -05:00
for scanner . Scan ( ) {
// Skip any empty lines, or comment like characters
2019-12-03 13:50:20 -05:00
text := scanner . Text ( )
if text == "" || strings . HasPrefix ( text , KvComment ) {
2019-11-27 12:36:08 -05:00
continue
}
2020-12-04 12:32:35 -05:00
dynamic , err := c . SetKVS ( text , DefaultKVS )
if err != nil {
return false , err
2019-11-27 12:36:08 -05:00
}
2020-12-04 12:32:35 -05:00
dynOnly = dynOnly && dynamic
2019-12-03 13:50:20 -05:00
n += len ( text )
2019-11-27 12:36:08 -05:00
}
if err := scanner . Err ( ) ; err != nil {
2020-12-04 12:32:35 -05:00
return false , err
2019-11-27 12:36:08 -05:00
}
2020-12-04 12:32:35 -05:00
return dynOnly , nil
2019-11-27 12:36:08 -05:00
}
2021-06-03 11:15:44 -04:00
// RedactSensitiveInfo - removes sensitive information
// like urls and credentials from the configuration
func ( c Config ) RedactSensitiveInfo ( ) Config {
nc := c . Clone ( )
for configName , configVals := range nc {
for _ , helpKV := range HelpSubSysMap [ configName ] {
if helpKV . Sensitive {
for name , kvs := range configVals {
for i := range kvs {
if kvs [ i ] . Key == helpKV . Key && len ( kvs [ i ] . Value ) > 0 {
kvs [ i ] . Value = "*redacted*"
}
}
configVals [ name ] = kvs
}
}
}
}
2021-07-14 03:23:22 -04:00
// Remove the server credentials altogether
nc . DelKVS ( CredentialsSubSys )
2021-06-03 11:15:44 -04:00
return nc
}
2019-11-27 12:36:08 -05:00
type configWriteTo struct {
Config
filterByKey string
}
// NewConfigWriteTo - returns a struct which
// allows for serializing the config/kv struct
// to a io.WriterTo
func NewConfigWriteTo ( cfg Config , key string ) io . WriterTo {
return & configWriteTo { Config : cfg , filterByKey : key }
}
// WriteTo - implements io.WriterTo interface implementation for config.
func ( c * configWriteTo ) WriteTo ( w io . Writer ) ( int64 , error ) {
2019-12-06 05:43:10 -05:00
kvsTargets , err := c . GetKVS ( c . filterByKey , DefaultKVS )
2019-11-27 12:36:08 -05:00
if err != nil {
return 0 , err
}
var n int
2019-12-06 05:43:10 -05:00
for _ , target := range kvsTargets {
m1 , _ := w . Write ( [ ] byte ( target . SubSystem ) )
2019-11-27 12:36:08 -05:00
m2 , _ := w . Write ( [ ] byte ( KvSpaceSeparator ) )
2019-12-06 05:43:10 -05:00
m3 , _ := w . Write ( [ ] byte ( target . KVS . String ( ) ) )
if len ( kvsTargets ) > 1 {
2019-11-27 12:36:08 -05:00
m4 , _ := w . Write ( [ ] byte ( KvNewline ) )
n += m1 + m2 + m3 + m4
} else {
n += m1 + m2 + m3
}
}
return int64 ( n ) , nil
}
2019-10-23 01:59:13 -04:00
// Default KV configs for worm and region
var (
DefaultCredentialKVS = KVS {
2019-11-20 18:10:24 -05:00
KV {
Key : AccessKey ,
Value : auth . DefaultAccessKey ,
} ,
KV {
Key : SecretKey ,
Value : auth . DefaultSecretKey ,
} ,
2019-10-23 01:59:13 -04:00
}
2021-11-25 16:06:25 -05:00
DefaultSiteKVS = KVS {
KV {
Key : NameKey ,
Value : "" ,
} ,
KV {
Key : RegionKey ,
Value : "" ,
} ,
}
2019-10-23 01:59:13 -04:00
DefaultRegionKVS = KVS {
2019-11-20 18:10:24 -05:00
KV {
Key : RegionName ,
Value : "" ,
} ,
2019-10-23 01:59:13 -04:00
}
)
// LookupCreds - lookup credentials from config.
func LookupCreds ( kv KVS ) ( auth . Credentials , error ) {
if err := CheckValidKeys ( CredentialsSubSys , kv , DefaultCredentialKVS ) ; err != nil {
return auth . Credentials { } , err
}
2019-12-14 20:27:57 -05:00
accessKey := kv . Get ( AccessKey )
secretKey := kv . Get ( SecretKey )
if accessKey == "" || secretKey == "" {
2019-11-13 20:38:05 -05:00
accessKey = auth . DefaultAccessKey
secretKey = auth . DefaultSecretKey
}
return auth . CreateCredentials ( accessKey , secretKey )
2019-10-23 01:59:13 -04:00
}
2021-11-25 16:06:25 -05:00
// Site - holds site info - name and region.
type Site struct {
Name string
Region string
}
2019-11-14 17:19:57 -05:00
var validRegionRegex = regexp . MustCompile ( "^[a-zA-Z][a-zA-Z0-9-_-]+$" )
2021-11-25 16:06:25 -05:00
// validSiteNameRegex - allows lowercase letters, digits and '-', starts with
// letter. At least 2 characters long.
var validSiteNameRegex = regexp . MustCompile ( "^[a-z][a-z0-9-]+$" )
// LookupSite - get site related configuration. Loads configuration from legacy
// region sub-system as well.
func LookupSite ( siteKV KVS , regionKV KVS ) ( s Site , err error ) {
if err = CheckValidKeys ( SiteSubSys , siteKV , DefaultSiteKVS ) ; err != nil {
return
2019-10-23 01:59:13 -04:00
}
region := env . Get ( EnvRegion , "" )
if region == "" {
2021-11-25 16:06:25 -05:00
env . Get ( EnvRegionName , "" )
}
if region == "" {
region = env . Get ( EnvSiteRegion , siteKV . Get ( RegionKey ) )
}
if region == "" {
// No region config found in the site-subsystem. So lookup the legacy
// region sub-system.
if err = CheckValidKeys ( RegionSubSys , regionKV , DefaultRegionKVS ) ; err != nil {
// An invalid key was found in the region sub-system.
// Since the region sub-system cannot be (re)set as it
// is legacy, we return an error to tell the user to
// reset the region via the new command.
err = Errorf ( "could not load region from legacy configuration as it was invalid - use 'mc admin config set myminio site region=myregion name=myname' to set a region and name (%v)" , err )
return
}
region = regionKV . Get ( RegionName )
2019-10-23 01:59:13 -04:00
}
2019-11-14 17:19:57 -05:00
if region != "" {
2021-11-25 16:06:25 -05:00
if ! validRegionRegex . MatchString ( region ) {
err = Errorf (
"region '%s' is invalid, expected simple characters such as [us-east-1, myregion...]" ,
region )
return
2019-11-14 17:19:57 -05:00
}
2021-11-25 16:06:25 -05:00
s . Region = region
2019-11-14 17:19:57 -05:00
}
2021-11-25 16:06:25 -05:00
name := env . Get ( EnvSiteName , siteKV . Get ( NameKey ) )
if name != "" {
if ! validSiteNameRegex . MatchString ( name ) {
err = Errorf (
"site name '%s' is invalid, expected simple characters such as [cal-rack0, myname...]" ,
name )
return
}
s . Name = name
}
return
2019-10-23 01:59:13 -04:00
}
// CheckValidKeys - checks if inputs KVS has the necessary keys,
// returns error if it find extra or superflous keys.
func CheckValidKeys ( subSys string , kv KVS , validKVS KVS ) error {
nkv := KVS { }
2019-11-20 18:10:24 -05:00
for _ , kv := range kv {
2019-12-05 07:47:42 -05:00
// Comment is a valid key, its also fully optional
// ignore it since it is a valid key for all
// sub-systems.
if kv . Key == Comment {
continue
}
2019-11-20 18:10:24 -05:00
if _ , ok := validKVS . Lookup ( kv . Key ) ; ! ok {
nkv = append ( nkv , kv )
2019-10-23 01:59:13 -04:00
}
}
if len ( nkv ) > 0 {
2019-12-04 18:32:37 -05:00
return Errorf (
"found invalid keys (%s) for '%s' sub-system, use 'mc admin config reset myminio %s' to fix invalid keys" , nkv . String ( ) , subSys , subSys )
2019-10-23 01:59:13 -04:00
}
return nil
}
// LookupWorm - check if worm is enabled
2019-11-20 18:10:24 -05:00
func LookupWorm ( ) ( bool , error ) {
2019-12-04 18:32:37 -05:00
return ParseBool ( env . Get ( EnvWorm , EnableOff ) )
2019-10-23 01:59:13 -04:00
}
2021-02-17 15:04:11 -05:00
// Carries all the renamed sub-systems from their
// previously known names
var renamedSubsys = map [ string ] string {
CrawlerSubSys : ScannerSubSys ,
// Add future sub-system renames
}
2020-05-23 20:38:39 -04:00
// Merge - merges a new config with all the
// missing values for default configs,
// returns a config.
func ( c Config ) Merge ( ) Config {
cp := New ( )
for subSys , tgtKV := range c {
for tgt := range tgtKV {
ckvs := c [ subSys ] [ tgt ]
for _ , kv := range cp [ subSys ] [ Default ] {
_ , ok := c [ subSys ] [ tgt ] . Lookup ( kv . Key )
if ! ok {
ckvs . Set ( kv . Key , kv . Value )
}
}
2020-09-08 11:57:04 -04:00
if _ , ok := cp [ subSys ] ; ! ok {
2021-02-17 15:04:11 -05:00
rnSubSys , ok := renamedSubsys [ subSys ]
if ! ok {
// A config subsystem was removed or server was downgraded.
continue
}
// Copy over settings from previous sub-system
// to newly renamed sub-system
for _ , kv := range cp [ rnSubSys ] [ Default ] {
_ , ok := c [ subSys ] [ tgt ] . Lookup ( kv . Key )
if ! ok {
ckvs . Set ( kv . Key , kv . Value )
}
}
subSys = rnSubSys
2020-09-08 11:57:04 -04:00
}
2020-05-23 20:38:39 -04:00
cp [ subSys ] [ tgt ] = ckvs
}
}
return cp
}
2019-10-23 01:59:13 -04:00
// New - initialize a new server config.
func New ( ) Config {
srvCfg := make ( Config )
for _ , k := range SubSystems . ToSlice ( ) {
srvCfg [ k ] = map [ string ] KVS { }
2019-12-03 13:50:20 -05:00
srvCfg [ k ] [ Default ] = DefaultKVS [ k ]
2019-10-23 01:59:13 -04:00
}
return srvCfg
}
2019-12-06 05:43:10 -05:00
// Target signifies an individual target
type Target struct {
SubSystem string
KVS KVS
}
// Targets sub-system targets
type Targets [ ] Target
2019-10-23 01:59:13 -04:00
// GetKVS - get kvs from specific subsystem.
2019-12-06 05:43:10 -05:00
func ( c Config ) GetKVS ( s string , defaultKVS map [ string ] KVS ) ( Targets , error ) {
2019-10-23 01:59:13 -04:00
if len ( s ) == 0 {
2019-12-14 20:27:57 -05:00
return nil , Errorf ( "input cannot be empty" )
2019-10-23 01:59:13 -04:00
}
inputs := strings . Fields ( s )
if len ( inputs ) > 1 {
2019-12-14 20:27:57 -05:00
return nil , Errorf ( "invalid number of arguments %s" , s )
2019-10-23 01:59:13 -04:00
}
subSystemValue := strings . SplitN ( inputs [ 0 ] , SubSystemSeparator , 2 )
if len ( subSystemValue ) == 0 {
2019-12-14 20:27:57 -05:00
return nil , Errorf ( "invalid number of arguments %s" , s )
2019-10-23 01:59:13 -04:00
}
found := SubSystems . Contains ( subSystemValue [ 0 ] )
if ! found {
2019-11-14 17:19:57 -05:00
// Check for sub-prefix only if the input value is only a
// single value, this rejects invalid inputs if any.
2019-10-23 01:59:13 -04:00
found = ! SubSystems . FuncMatch ( strings . HasPrefix , subSystemValue [ 0 ] ) . IsEmpty ( ) && len ( subSystemValue ) == 1
}
if ! found {
2019-12-14 20:27:57 -05:00
return nil , Errorf ( "unknown sub-system %s" , s )
2019-10-23 01:59:13 -04:00
}
2019-12-06 05:43:10 -05:00
targets := Targets { }
2019-11-20 18:10:24 -05:00
subSysPrefix := subSystemValue [ 0 ]
2019-10-23 01:59:13 -04:00
if len ( subSystemValue ) == 2 {
if len ( subSystemValue [ 1 ] ) == 0 {
2019-12-14 20:27:57 -05:00
return nil , Errorf ( "sub-system target '%s' cannot be empty" , s )
2019-10-23 01:59:13 -04:00
}
2019-12-06 05:43:10 -05:00
kvs , ok := c [ subSysPrefix ] [ subSystemValue [ 1 ] ]
2019-10-23 01:59:13 -04:00
if ! ok {
2019-12-14 20:27:57 -05:00
return nil , Errorf ( "sub-system target '%s' doesn't exist" , s )
2019-10-23 01:59:13 -04:00
}
2019-12-06 05:43:10 -05:00
for _ , kv := range defaultKVS [ subSysPrefix ] {
_ , ok = kvs . Lookup ( kv . Key )
if ! ok {
kvs . Set ( kv . Key , kv . Value )
}
}
targets = append ( targets , Target {
SubSystem : inputs [ 0 ] ,
KVS : kvs ,
} )
2019-11-20 18:10:24 -05:00
} else {
2021-11-25 16:06:25 -05:00
// Use help for sub-system to preserve the order. Add deprecated
// keys at the end (in some order).
kvsOrder := append ( [ ] HelpKV { } , HelpSubSysMap [ "" ] ... )
for _ , v := range HelpDeprecatedSubSysMap {
kvsOrder = append ( kvsOrder , v )
}
for _ , hkv := range kvsOrder {
2019-12-06 05:43:10 -05:00
if ! strings . HasPrefix ( hkv . Key , subSysPrefix ) {
2019-11-20 18:10:24 -05:00
continue
}
2019-12-06 16:53:51 -05:00
if c [ hkv . Key ] [ Default ] . Empty ( ) {
targets = append ( targets , Target {
SubSystem : hkv . Key ,
KVS : defaultKVS [ hkv . Key ] ,
} )
}
2019-12-06 05:43:10 -05:00
for k , kvs := range c [ hkv . Key ] {
2019-12-06 16:53:51 -05:00
for _ , dkv := range defaultKVS [ hkv . Key ] {
2019-12-06 05:43:10 -05:00
_ , ok := kvs . Lookup ( dkv . Key )
if ! ok {
kvs . Set ( dkv . Key , dkv . Value )
}
}
2019-11-20 18:10:24 -05:00
if k != Default {
2019-12-06 05:43:10 -05:00
targets = append ( targets , Target {
SubSystem : hkv . Key + SubSystemSeparator + k ,
KVS : kvs ,
} )
2019-11-20 18:10:24 -05:00
} else {
2019-12-06 05:43:10 -05:00
targets = append ( targets , Target {
SubSystem : hkv . Key ,
KVS : kvs ,
} )
2019-11-20 18:10:24 -05:00
}
2019-10-23 01:59:13 -04:00
}
}
}
2019-12-06 05:43:10 -05:00
return targets , nil
2019-10-23 01:59:13 -04:00
}
// DelKVS - delete a specific key.
func ( c Config ) DelKVS ( s string ) error {
if len ( s ) == 0 {
2019-12-14 20:27:57 -05:00
return Errorf ( "input arguments cannot be empty" )
2019-10-23 01:59:13 -04:00
}
inputs := strings . Fields ( s )
if len ( inputs ) > 1 {
2019-12-14 20:27:57 -05:00
return Errorf ( "invalid number of arguments %s" , s )
2019-10-23 01:59:13 -04:00
}
subSystemValue := strings . SplitN ( inputs [ 0 ] , SubSystemSeparator , 2 )
if len ( subSystemValue ) == 0 {
2019-12-14 20:27:57 -05:00
return Errorf ( "invalid number of arguments %s" , s )
2019-10-23 01:59:13 -04:00
}
if ! SubSystems . Contains ( subSystemValue [ 0 ] ) {
2019-12-04 18:32:37 -05:00
// Unknown sub-system found try to remove it anyways.
delete ( c , subSystemValue [ 0 ] )
return nil
2019-10-23 01:59:13 -04:00
}
2019-11-20 18:10:24 -05:00
tgt := Default
subSys := subSystemValue [ 0 ]
2019-10-23 01:59:13 -04:00
if len ( subSystemValue ) == 2 {
if len ( subSystemValue [ 1 ] ) == 0 {
2019-12-14 20:27:57 -05:00
return Errorf ( "sub-system target '%s' cannot be empty" , s )
2019-10-23 01:59:13 -04:00
}
2019-11-20 18:10:24 -05:00
tgt = subSystemValue [ 1 ]
}
_ , ok := c [ subSys ] [ tgt ]
if ! ok {
2019-12-14 20:27:57 -05:00
return Errorf ( "sub-system %s already deleted" , s )
2019-10-23 01:59:13 -04:00
}
2019-11-20 18:10:24 -05:00
delete ( c [ subSys ] , tgt )
2019-11-13 20:38:05 -05:00
return nil
2019-10-23 01:59:13 -04:00
}
// Clone - clones a config map entirely.
func ( c Config ) Clone ( ) Config {
cp := New ( )
for subSys , tgtKV := range c {
cp [ subSys ] = make ( map [ string ] KVS )
for tgt , kv := range tgtKV {
2019-11-20 18:10:24 -05:00
cp [ subSys ] [ tgt ] = append ( cp [ subSys ] [ tgt ] , kv ... )
2019-10-23 01:59:13 -04:00
}
}
return cp
}
2022-02-08 13:36:41 -05:00
// GetSubSys - extracts subssystem info from given config string
func GetSubSys ( s string ) ( subSys string , inputs [ ] string , tgt string , e error ) {
tgt = Default
2019-10-23 01:59:13 -04:00
if len ( s ) == 0 {
2022-02-08 13:36:41 -05:00
return subSys , inputs , tgt , Errorf ( "input arguments cannot be empty" )
2019-10-23 01:59:13 -04:00
}
2022-02-08 13:36:41 -05:00
inputs = strings . SplitN ( s , KvSpaceSeparator , 2 )
2022-02-22 13:59:28 -05:00
2019-10-23 01:59:13 -04:00
subSystemValue := strings . SplitN ( inputs [ 0 ] , SubSystemSeparator , 2 )
2022-02-22 13:59:28 -05:00
subSys = subSystemValue [ 0 ]
if ! SubSystems . Contains ( subSys ) {
return subSys , inputs , tgt , Errorf ( "unknown sub-system %s" , s )
2019-10-23 01:59:13 -04:00
}
2022-02-22 13:59:28 -05:00
if len ( inputs ) == 1 {
return subSys , inputs , tgt , nil
2019-10-23 01:59:13 -04:00
}
if SubSystemsSingleTargets . Contains ( subSystemValue [ 0 ] ) && len ( subSystemValue ) == 2 {
2022-02-08 13:36:41 -05:00
return subSys , inputs , tgt , Errorf ( "sub-system '%s' only supports single target" , subSystemValue [ 0 ] )
2019-10-23 01:59:13 -04:00
}
2020-04-10 00:45:17 -04:00
if len ( subSystemValue ) == 2 {
tgt = subSystemValue [ 1 ]
}
2022-02-08 13:36:41 -05:00
return subSys , inputs , tgt , e
}
// SetKVS - set specific key values per sub-system.
func ( c Config ) SetKVS ( s string , defaultKVS map [ string ] KVS ) ( dynamic bool , err error ) {
subSys , inputs , tgt , err := GetSubSys ( s )
if err != nil {
return false , err
}
dynamic = SubSystemsDynamic . Contains ( subSys )
2020-04-16 20:43:14 -04:00
fields := madmin . KvFields ( inputs [ 1 ] , defaultKVS [ subSys ] . Keys ( ) )
2020-04-10 00:45:17 -04:00
if len ( fields ) == 0 {
2020-12-04 12:32:35 -05:00
return false , Errorf ( "sub-system '%s' cannot have empty keys" , subSys )
2020-04-10 00:45:17 -04:00
}
2022-01-02 12:15:06 -05:00
kvs := KVS { }
2019-10-23 01:59:13 -04:00
var prevK string
2020-04-10 00:45:17 -04:00
for _ , v := range fields {
2019-10-23 01:59:13 -04:00
kv := strings . SplitN ( v , KvSeparator , 2 )
if len ( kv ) == 0 {
continue
}
if len ( kv ) == 1 && prevK != "" {
2019-12-05 07:47:42 -05:00
value := strings . Join ( [ ] string {
kvs . Get ( prevK ) ,
madmin . SanitizeValue ( kv [ 0 ] ) ,
} , KvSpaceSeparator )
kvs . Set ( prevK , value )
2019-10-23 01:59:13 -04:00
continue
}
2019-12-05 07:47:42 -05:00
if len ( kv ) == 2 {
prevK = kv [ 0 ]
kvs . Set ( prevK , madmin . SanitizeValue ( kv [ 1 ] ) )
continue
2019-11-12 06:16:25 -05:00
}
2020-12-04 12:32:35 -05:00
return false , Errorf ( "key '%s', cannot have empty value" , kv [ 0 ] )
2019-10-23 01:59:13 -04:00
}
2019-12-04 18:32:37 -05:00
_ , ok := kvs . Lookup ( Enable )
// Check if state is required
2019-12-12 20:02:14 -05:00
_ , enableRequired := defaultKVS [ subSys ] . Lookup ( Enable )
if ! ok && enableRequired {
2019-11-14 17:19:57 -05:00
// implicit state "on" if not specified.
2019-12-05 07:47:42 -05:00
kvs . Set ( Enable , EnableOn )
2019-11-20 18:10:24 -05:00
}
2019-12-04 18:32:37 -05:00
2021-11-10 13:01:32 -05:00
var currKVS KVS
ck , ok := c [ subSys ] [ tgt ]
2019-12-05 07:47:42 -05:00
if ! ok {
2021-11-10 13:01:32 -05:00
currKVS = defaultKVS [ subSys ] . Clone ( )
2020-01-10 19:57:18 -05:00
} else {
2021-11-10 13:01:32 -05:00
currKVS = ck . Clone ( )
2020-01-10 19:57:18 -05:00
for _ , kv := range defaultKVS [ subSys ] {
if _ , ok = currKVS . Lookup ( kv . Key ) ; ! ok {
currKVS . Set ( kv . Key , kv . Value )
}
}
2019-12-05 07:47:42 -05:00
}
2019-12-04 18:32:37 -05:00
for _ , kv := range kvs {
2019-12-05 07:47:42 -05:00
if kv . Key == Comment {
// Skip comment and add it later.
continue
}
2019-12-04 18:32:37 -05:00
currKVS . Set ( kv . Key , kv . Value )
}
2019-12-05 07:47:42 -05:00
v , ok := kvs . Lookup ( Comment )
if ok {
currKVS . Set ( Comment , v )
2019-10-23 01:59:13 -04:00
}
2019-12-04 18:32:37 -05:00
2019-12-12 17:55:07 -05:00
hkvs := HelpSubSysMap [ subSys ]
for _ , hkv := range hkvs {
2019-12-12 20:02:14 -05:00
var enabled bool
if enableRequired {
enabled = currKVS . Get ( Enable ) == EnableOn
} else {
// when enable arg is not required
// then it is implicit on for the sub-system.
enabled = true
}
2019-12-12 17:55:07 -05:00
v , _ := currKVS . Lookup ( hkv . Key )
2019-12-12 20:02:14 -05:00
if v == "" && ! hkv . Optional && enabled {
// Return error only if the
// key is enabled, for state=off
// let it be empty.
2020-12-04 12:32:35 -05:00
return false , Errorf (
2019-12-12 17:55:07 -05:00
"'%s' is not optional for '%s' sub-system, please check '%s' documentation" ,
hkv . Key , subSys , subSys )
}
}
2019-12-05 07:47:42 -05:00
c [ subSys ] [ tgt ] = currKVS
2020-12-04 12:32:35 -05:00
return dynamic , nil
2019-10-23 01:59:13 -04:00
}