Migrate config to KV data format (#8392)

- adding oauth support to MinIO browser (#8400) by @kanagaraj
- supports multi-line get/set/del for all config fields
- add support for comments, allow toggle
- add extensive validation of config before saving
- support MinIO browser to support proper claims, using STS tokens
- env support for all config parameters, legacy envs are also
  supported with all documentation now pointing to latest ENVs
- preserve accessKey/secretKey from FS mode setups
- add history support implements three APIs
  - ClearHistory
  - RestoreHistory
  - ListHistory
- add help command support for each config parameters
- all the bug fixes after migration to KV, and other bug
  fixes encountered during testing.

cmd/config/compress/compress.go

@@ -18,7 +18,6 @@ package compress
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 
 	"github.com/minio/minio/cmd/config"
@@ -34,30 +33,62 @@ type Config struct {
 
 // Compression environment variables
 const (
-	EnvCompress           = "MINIO_COMPRESS"
+	Extensions = "extensions"
+	MimeTypes  = "mime_types"
+
+	EnvCompressState      = "MINIO_COMPRESS_STATE"
 	EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
-	EnvCompressMimeTypes  = "MINIO_COMPRESS_MIMETYPES"
+	EnvCompressMimeTypes  = "MINIO_COMPRESS_MIME_TYPES"
+
+	// Include-list for compression.
+	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
+	DefaultMimeTypes  = "text/*,application/json,application/xml"
+)
+
+// DefaultKVS - default KV config for compression settings
+var (
+	DefaultKVS = config.KVS{
+		config.State:   config.StateOff,
+		config.Comment: "This is a default compression configuration",
+		Extensions:     DefaultExtensions,
+		MimeTypes:      DefaultMimeTypes,
+	}
 )
 
 // Parses the given compression exclude list `extensions` or `content-types`.
 func parseCompressIncludes(includes []string) ([]string, error) {
 	for _, e := range includes {
 		if len(e) == 0 {
-			return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
+			return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be empty")
 		}
 	}
 	return includes, nil
 }
 
 // LookupConfig - lookup compression config.
-func LookupConfig(cfg Config) (Config, error) {
-	if compress := env.Get(EnvCompress, strconv.FormatBool(cfg.Enabled)); compress != "" {
-		cfg.Enabled = strings.EqualFold(compress, "true")
+func LookupConfig(kvs config.KVS) (Config, error) {
+	var err error
+	cfg := Config{}
+	if err = config.CheckValidKeys(config.CompressionSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
 	}
 
-	compressExtensions := env.Get(EnvCompressExtensions, strings.Join(cfg.Extensions, ","))
-	compressMimeTypes := env.Get(EnvCompressMimeTypes, strings.Join(cfg.MimeTypes, ","))
-	if compressExtensions != "" || compressMimeTypes != "" {
+	compress := env.Get(EnvCompress, "")
+	if compress == "" {
+		compress = env.Get(EnvCompressState, kvs.Get(config.State))
+	}
+	cfg.Enabled, err = config.ParseBool(compress)
+	if err != nil {
+		return cfg, err
+	}
+	if !cfg.Enabled {
+		return cfg, nil
+	}
+
+	compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
+	compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
+	compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))
+	if compressExtensions != "" || compressMimeTypes != "" || compressMimeTypesLegacy != "" {
 		if compressExtensions != "" {
 			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, config.ValueSeparator))
 			if err != nil {
@@ -66,11 +97,19 @@ func LookupConfig(cfg Config) (Config, error) {
 			cfg.Extensions = extensions
 		}
 		if compressMimeTypes != "" {
-			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, config.ValueSeparator))
+			mimeTypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, config.ValueSeparator))
 			if err != nil {
-				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, contenttypes)
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
 			}
-			cfg.MimeTypes = contenttypes
+			cfg.MimeTypes = mimeTypes
+		}
+		if compressMimeTypesLegacy != "" {
+			mimeTypes, err := parseCompressIncludes(strings.Split(compressMimeTypesLegacy,
+				config.ValueSeparator))
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
+			}
+			cfg.MimeTypes = mimeTypes
 		}
 	}
 

cmd/config/compress/help.go

@@ -0,0 +1,29 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package compress
+
+import "github.com/minio/minio/cmd/config"
+
+// Help template for compress feature.
+var (
+	Help = config.HelpKV{
+		Extensions:     `Comma separated file extensions to compress. eg: ".txt,.log,.csv"`,
+		MimeTypes:      `Comma separate wildcard mime-types to compress. eg: "text/*,application/json,application/xml"`,
+		config.State:   "Indicates if compression is enabled or not",
+		config.Comment: "A comment to describe the compression setting",
+	}
+)

cmd/config/compress/legacy.go

@@ -0,0 +1,44 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package compress
+
+import (
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+)
+
+// Legacy envs.
+const (
+	EnvCompress                = "MINIO_COMPRESS"
+	EnvCompressMimeTypesLegacy = "MINIO_COMPRESS_MIMETYPES"
+)
+
+// SetCompressionConfig - One time migration code needed, for migrating from older config to new for Compression.
+func SetCompressionConfig(s config.Config, cfg Config) {
+	s[config.CompressionSubSys][config.Default] = config.KVS{
+		config.State: func() string {
+			if cfg.Enabled {
+				return config.StateOn
+			}
+			return config.StateOff
+		}(),
+		config.Comment: "Settings for Compression, after migrating config",
+		Extensions:     strings.Join(cfg.Extensions, ","),
+		MimeTypes:      strings.Join(cfg.MimeTypes, ","),
+	}
+}