Migrate config to KV data format (#8392)

- adding oauth support to MinIO browser (#8400) by @kanagaraj
- supports multi-line get/set/del for all config fields
- add support for comments, allow toggle
- add extensive validation of config before saving
- support MinIO browser to support proper claims, using STS tokens
- env support for all config parameters, legacy envs are also
  supported with all documentation now pointing to latest ENVs
- preserve accessKey/secretKey from FS mode setups
- add history support implements three APIs
  - ClearHistory
  - RestoreHistory
  - ListHistory
- add help command support for each config parameters
- all the bug fixes after migration to KV, and other bug
  fixes encountered during testing.

pkg/quick/encoding.go

@@ -187,10 +187,6 @@ func loadFileConfig(filename string, v interface{}) error {
 		fileData = []byte(strings.Replace(string(fileData), "\r\n", "\n", -1))
 	}
 
-	if err = checkDupJSONKeys(string(fileData)); err != nil {
-		return err
-	}
-
 	// Unmarshal file's content
 	return toUnmarshaller(filepath.Ext(filename))(fileData, v)
 }

pkg/quick/errorutil.go

@@ -21,12 +21,10 @@ package quick
 import (
 	"bufio"
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 
 	"github.com/cheggaaa/pb"
-	"github.com/tidwall/gjson"
 )
 
 const errorFmt = "%5d: %s  <<<<"
@@ -82,59 +80,3 @@ func FormatJSONSyntaxError(data io.Reader, offset int64) (highlight string) {
 
 	return fmt.Sprintf(errorFmt, errLine, readLine.String()[idx:])
 }
-
-// doCheckDupJSONKeys recursively detects duplicate json keys
-func doCheckDupJSONKeys(key, value gjson.Result) error {
-	// Key occurrences map of the current scope to count
-	// if there is any duplicated json key.
-	keysOcc := make(map[string]int)
-
-	// Holds the found error
-	var checkErr error
-
-	// Iterate over keys in the current json scope
-	value.ForEach(func(k, v gjson.Result) bool {
-		// If current key is not null, check if its
-		// value contains some duplicated keys.
-		if k.Type != gjson.Null {
-			keysOcc[k.String()]++
-			checkErr = doCheckDupJSONKeys(k, v)
-		}
-		return checkErr == nil
-	})
-
-	// Check found err
-	if checkErr != nil {
-		return errors.New(key.String() + " => " + checkErr.Error())
-	}
-
-	// Check for duplicated keys
-	for k, v := range keysOcc {
-		if v > 1 {
-			return errors.New(key.String() + " => `" + k + "` entry is duplicated")
-		}
-	}
-
-	return nil
-}
-
-// Check recursively if a key is duplicated in the same json scope
-// e.g.:
-//  `{ "key" : { "key" ..` is accepted
-//  `{ "key" : { "subkey" : "val1", "subkey": "val2" ..` throws subkey duplicated error
-func checkDupJSONKeys(json string) error {
-	// Parse config with gjson library
-	config := gjson.Parse(json)
-
-	// Create a fake rootKey since root json doesn't seem to have representation
-	// in gjson library.
-	rootKey := gjson.Result{Type: gjson.String, Str: "config.json"}
-
-	// Check if loaded json contains any duplicated keys
-	return doCheckDupJSONKeys(rootKey, config)
-}
-
-// CheckDuplicateKeys - checks for duplicate entries in a JSON file
-func CheckDuplicateKeys(json string) error {
-	return checkDupJSONKeys(json)
-}

pkg/quick/quick_test.go

@@ -27,8 +27,6 @@ import (
 	"runtime"
 	"strings"
 	"testing"
-
-	"github.com/tidwall/gjson"
 )
 
 func TestReadVersion(t *testing.T) {
@@ -506,31 +504,3 @@ func TestDeepDiff(t *testing.T) {
 	//		fmt.Printf("DeepDiff[%d]: %s=%v\n", i, field.Name(), field.Value())
 	//	}
 }
-
-func TestCheckDupJSONKeys(t *testing.T) {
-	testCases := []struct {
-		json       string
-		shouldPass bool
-	}{
-		{`{}`, true},
-		{`{"version" : "13"}`, true},
-		{`{"version" : "13", "version": "14"}`, false},
-		{`{"version" : "13", "credential": {"accessKey": "12345"}}`, true},
-		{`{"version" : "13", "credential": {"accessKey": "12345", "accessKey":"12345"}}`, false},
-		{`{"version" : "13", "notify": {"amqp": {"1"}, "webhook":{"3"}}}`, true},
-		{`{"version" : "13", "notify": {"amqp": {"1"}, "amqp":{"3"}}}`, false},
-		{`{"version" : "13", "notify": {"amqp": {"1":{}, "2":{}}}}`, true},
-		{`{"version" : "13", "notify": {"amqp": {"1":{}, "1":{}}}}`, false},
-	}
-
-	for i, testCase := range testCases {
-		err := doCheckDupJSONKeys(gjson.Result{}, gjson.Parse(testCase.json))
-		if testCase.shouldPass && err != nil {
-			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, err)
-		}
-		if !testCase.shouldPass && err == nil {
-			t.Errorf("Test %d, should fail but it succeed.", i+1)
-		}
-	}
-
-}