Migrate config to KV data format (#8392)

- adding oauth support to MinIO browser (#8400) by @kanagaraj
- supports multi-line get/set/del for all config fields
- add support for comments, allow toggle
- add extensive validation of config before saving
- support MinIO browser to support proper claims, using STS tokens
- env support for all config parameters, legacy envs are also
  supported with all documentation now pointing to latest ENVs
- preserve accessKey/secretKey from FS mode setups
- add history support implements three APIs
  - ClearHistory
  - RestoreHistory
  - ListHistory
- add help command support for each config parameters
- all the bug fixes after migration to KV, and other bug
  fixes encountered during testing.

cmd/config/cache/config.go

@@ -28,9 +28,11 @@ import (
 
 // Config represents cache config settings
 type Config struct {
+	Enabled bool     `json:"-"`
 	Drives  []string `json:"drives"`
 	Expiry  int      `json:"expiry"`
 	MaxUse  int      `json:"maxuse"`
+	Quota   int      `json:"quota"`
 	Exclude []string `json:"exclude"`
 }
 
@@ -55,6 +57,10 @@ func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
 		return errors.New("config max use value should not be null or negative")
 	}
 
+	if _cfg.Quota < 0 {
+		return errors.New("config quota value should not be null or negative")
+	}
+
 	if _, err = parseCacheDrives(_cfg.Drives); err != nil {
 		return err
 	}

cmd/config/cache/config_test.go

@@ -69,7 +69,7 @@ func TestParseCacheDrives(t *testing.T) {
 		}{"/home/drive{1..3}", []string{}, false})
 	}
 	for i, testCase := range testCases {
-		drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter))
+		drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheDelimiter))
 		if err != nil && testCase.success {
 			t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
 		}
@@ -98,7 +98,7 @@ func TestParseCacheExclude(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		excludes, err := parseCacheExcludes(strings.Split(testCase.excludeStr, cacheEnvDelimiter))
+		excludes, err := parseCacheExcludes(strings.Split(testCase.excludeStr, cacheDelimiter))
 		if err != nil && testCase.success {
 			t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
 		}

cmd/config/cache/help.go

@@ -0,0 +1,31 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import "github.com/minio/minio/cmd/config"
+
+// Help template for caching feature.
+var (
+	Help = config.HelpKV{
+		Drives:         `List of mounted drives or directories delimited by ";"`,
+		Exclude:        `List of wildcard based cache exclusion patterns delimited by ";"`,
+		Expiry:         `Cache expiry duration in days. eg: "90"`,
+		Quota:          `Maximum permitted usage of the cache in percentage (0-100)`,
+		config.State:   "Indicates if caching is enabled or not",
+		config.Comment: "A comment to describe the caching setting",
+	}
+)

cmd/config/cache/legacy.go

@@ -0,0 +1,40 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+)
+
+// SetCacheConfig - One time migration code needed, for migrating from older config to new for Cache.
+func SetCacheConfig(s config.Config, cfg Config) {
+	s[config.CacheSubSys][config.Default] = DefaultKVS
+	s[config.CacheSubSys][config.Default][Drives] = strings.Join(cfg.Drives, cacheDelimiter)
+	s[config.CacheSubSys][config.Default][Exclude] = strings.Join(cfg.Exclude, cacheDelimiter)
+	s[config.CacheSubSys][config.Default][Expiry] = fmt.Sprintf("%d", cfg.Expiry)
+	s[config.CacheSubSys][config.Default][Quota] = fmt.Sprintf("%d", cfg.MaxUse)
+	s[config.CacheSubSys][config.Default][config.State] = func() string {
+		if len(cfg.Drives) > 0 {
+			return config.StateOn
+		}
+		return config.StateOff
+	}()
+	s[config.CacheSubSys][config.Default][config.Comment] = "Settings for Cache, after migrating config"
+}

cmd/config/cache/lookup.go

@@ -17,6 +17,7 @@
 package cache
 
 import (
+	"errors"
 	"strconv"
 	"strings"
 
@@ -26,53 +27,107 @@ import (
 
 // Cache ENVs
 const (
+	Drives  = "drives"
+	Exclude = "exclude"
+	Expiry  = "expiry"
+	MaxUse  = "maxuse"
+	Quota   = "quota"
+
+	EnvCacheState               = "MINIO_CACHE_STATE"
 	EnvCacheDrives              = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude             = "MINIO_CACHE_EXCLUDE"
 	EnvCacheExpiry              = "MINIO_CACHE_EXPIRY"
 	EnvCacheMaxUse              = "MINIO_CACHE_MAXUSE"
+	EnvCacheQuota               = "MINIO_CACHE_QUOTA"
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
+
+	DefaultExpiry = "90"
+	DefaultQuota  = "80"
+)
+
+// DefaultKVS - default KV settings for caching.
+var (
+	DefaultKVS = config.KVS{
+		config.State:   config.StateOff,
+		config.Comment: "This is a default cache configuration, only applicable in gateway setups",
+		Drives:         "",
+		Exclude:        "",
+		Expiry:         DefaultExpiry,
+		Quota:          DefaultQuota,
+	}
 )
 
 const (
-	cacheEnvDelimiter = ";"
+	cacheDelimiter = ";"
 )
 
 // LookupConfig - extracts cache configuration provided by environment
 // variables and merge them with provided CacheConfiguration.
-func LookupConfig(cfg Config) (Config, error) {
-	if drives := env.Get(EnvCacheDrives, strings.Join(cfg.Drives, ",")); drives != "" {
-		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
+func LookupConfig(kvs config.KVS) (Config, error) {
+	cfg := Config{}
+
+	if err := config.CheckValidKeys(config.CacheSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+
+	// Check if cache is explicitly disabled
+	stateBool, err := config.ParseBool(env.Get(EnvCacheState, kvs.Get(config.State)))
+	if err != nil {
+		return cfg, err
+	}
+	if !stateBool {
+		return cfg, nil
+	}
+
+	drives := env.Get(EnvCacheDrives, kvs.Get(Drives))
+	if len(drives) == 0 {
+		return cfg, nil
+	}
+
+	cfg.Drives, err = parseCacheDrives(strings.Split(drives, cacheDelimiter))
+	if err != nil {
+		return cfg, err
+	}
+
+	cfg.Enabled = true
+	if excludes := env.Get(EnvCacheExclude, kvs.Get(Exclude)); excludes != "" {
+		cfg.Exclude, err = parseCacheExcludes(strings.Split(excludes, cacheDelimiter))
 		if err != nil {
 			return cfg, err
 		}
-		cfg.Drives = driveList
 	}
 
-	if excludes := env.Get(EnvCacheExclude, strings.Join(cfg.Exclude, ",")); excludes != "" {
-		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
-		if err != nil {
-			return cfg, err
-		}
-		cfg.Exclude = excludeList
-	}
-
-	if expiryStr := env.Get(EnvCacheExpiry, strconv.Itoa(cfg.Expiry)); expiryStr != "" {
-		expiry, err := strconv.Atoi(expiryStr)
+	if expiryStr := env.Get(EnvCacheExpiry, kvs.Get(Expiry)); expiryStr != "" {
+		cfg.Expiry, err = strconv.Atoi(expiryStr)
 		if err != nil {
 			return cfg, config.ErrInvalidCacheExpiryValue(err)
 		}
-		cfg.Expiry = expiry
 	}
 
-	if maxUseStr := env.Get(EnvCacheMaxUse, strconv.Itoa(cfg.MaxUse)); maxUseStr != "" {
-		maxUse, err := strconv.Atoi(maxUseStr)
+	if maxUseStr := env.Get(EnvCacheMaxUse, kvs.Get(MaxUse)); maxUseStr != "" {
+		cfg.MaxUse, err = strconv.Atoi(maxUseStr)
 		if err != nil {
-			return cfg, config.ErrInvalidCacheMaxUse(err)
+			return cfg, config.ErrInvalidCacheQuota(err)
 		}
 		// maxUse should be a valid percentage.
-		if maxUse > 0 && maxUse <= 100 {
-			cfg.MaxUse = maxUse
+		if cfg.MaxUse < 0 || cfg.MaxUse > 100 {
+			err := errors.New("config max use value should not be null or negative")
+			return cfg, config.ErrInvalidCacheQuota(err)
 		}
+		cfg.Quota = cfg.MaxUse
+	}
+
+	if quotaStr := env.Get(EnvCacheQuota, kvs.Get(Quota)); quotaStr != "" {
+		cfg.Quota, err = strconv.Atoi(quotaStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheQuota(err)
+		}
+		// quota should be a valid percentage.
+		if cfg.Quota < 0 || cfg.Quota > 100 {
+			err := errors.New("config quota value should not be null or negative")
+			return cfg, config.ErrInvalidCacheQuota(err)
+		}
+		cfg.MaxUse = cfg.Quota
 	}
 
 	return cfg, nil