Converge etcd functionality as part of quick.Config

pkg/quick/encoding.go

@@ -20,6 +20,7 @@ package quick
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -27,7 +28,9 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"time"
 
+	etcd "github.com/coreos/etcd/client"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -122,6 +125,55 @@ func saveFileConfig(filename string, v interface{}) error {
 
 }
 
+func saveFileConfigEtcd(filename string, clnt etcd.Client, v interface{}) error {
+	// Fetch filename's extension
+	ext := filepath.Ext(filename)
+	// Marshal data
+	dataBytes, err := toMarshaller(ext)(v)
+	if err != nil {
+		return err
+	}
+	if runtime.GOOS == "windows" {
+		dataBytes = []byte(strings.Replace(string(dataBytes), "\n", "\r\n", -1))
+	}
+
+	kapi := etcd.NewKeysAPI(clnt)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	_, err = kapi.Update(ctx, filename, string(dataBytes))
+	cancel()
+	return err
+}
+
+func loadFileConfigEtcd(filename string, clnt etcd.Client, v interface{}) error {
+	kapi := etcd.NewKeysAPI(clnt)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	resp, err := kapi.Get(ctx, filename, nil)
+	cancel()
+	if err != nil {
+		return err
+	}
+
+	var ev *etcd.Node
+	switch {
+	case resp.Node.Dir:
+		for _, ev = range resp.Node.Nodes {
+			if string(ev.Key) == filename {
+				break
+			}
+		}
+	default:
+		ev = resp.Node
+	}
+
+	fileData := ev.Value
+	if runtime.GOOS == "windows" {
+		fileData = strings.Replace(ev.Value, "\r\n", "\n", -1)
+	}
+
+	// Unmarshal file's content
+	return toUnmarshaller(filepath.Ext(filename))([]byte(fileData), v)
+}
+
 // loadFileConfig unmarshals the file's content with the right
 // decoder format according to the filename extension. If no
 // extension is provided, json will be selected by default.

pkg/quick/quick.go

@@ -26,6 +26,7 @@ import (
 	"reflect"
 	"sync"
 
+	etcd "github.com/coreos/etcd/client"
 	"github.com/fatih/structs"
 	"github.com/minio/minio/pkg/safe"
 )
@@ -41,21 +42,22 @@ type Config interface {
 	DeepDiff(Config) ([]structs.Field, error)
 }
 
-// localConfig - implements quick.Config interface
-type localConfig struct {
+// config - implements quick.Config interface
+type config struct {
 	data interface{}
+	clnt etcd.Client
 	lock *sync.RWMutex
 }
 
 // Version returns the current config file format version
-func (d localConfig) Version() string {
+func (d config) Version() string {
 	st := structs.New(d.data)
 	f := st.Field("Version")
 	return f.Value().(string)
 }
 
 // String converts JSON config to printable string
-func (d localConfig) String() string {
+func (d config) String() string {
 	configBytes, _ := json.MarshalIndent(d.data, "", "\t")
 	return string(configBytes)
 }
@@ -63,10 +65,14 @@ func (d localConfig) String() string {
 // Save writes config data to a file. Data format
 // is selected based on file extension or JSON if
 // not provided.
-func (d localConfig) Save(filename string) error {
+func (d config) Save(filename string) error {
 	d.lock.Lock()
 	defer d.lock.Unlock()
 
+	if d.clnt != nil {
+		return saveFileConfigEtcd(filename, d.clnt, d.data)
+	}
+
 	// Backup if given file exists
 	oldData, err := ioutil.ReadFile(filename)
 	if err != nil {
@@ -89,19 +95,22 @@ func (d localConfig) Save(filename string) error {
 // Load - loads config from file and merge with currently set values
 // File content format is guessed from the file name extension, if not
 // available, consider that we have JSON.
-func (d localConfig) Load(filename string) error {
+func (d config) Load(filename string) error {
 	d.lock.Lock()
 	defer d.lock.Unlock()
+	if d.clnt != nil {
+		return loadFileConfigEtcd(filename, d.clnt, d.data)
+	}
 	return loadFileConfig(filename, d.data)
 }
 
 // Data - grab internal data map for reading
-func (d localConfig) Data() interface{} {
+func (d config) Data() interface{} {
 	return d.data
 }
 
 // Diff  - list fields that are in A but not in B
-func (d localConfig) Diff(c Config) ([]structs.Field, error) {
+func (d config) Diff(c Config) ([]structs.Field, error) {
 	var fields []structs.Field
 
 	currFields := structs.Fields(d.Data())
@@ -123,7 +132,7 @@ func (d localConfig) Diff(c Config) ([]structs.Field, error) {
 }
 
 // DeepDiff  - list fields in A that are missing or not equal to fields in B
-func (d localConfig) DeepDiff(c Config) ([]structs.Field, error) {
+func (d config) DeepDiff(c Config) ([]structs.Field, error) {
 	var fields []structs.Field
 
 	currFields := structs.Fields(d.Data())
@@ -179,43 +188,50 @@ func writeFile(filename string, data []byte) error {
 	return safeFile.Close()
 }
 
-// NewLocalConfig - instantiate a new config r/w configs from local files.
-func NewLocalConfig(data interface{}) (Config, error) {
+// GetVersion - extracts the version information.
+func GetVersion(filename string, clnt etcd.Client) (version string, err error) {
+	var qc Config
+	qc, err = LoadConfig(filename, clnt, &struct {
+		Version string
+	}{})
+	if err != nil {
+		return "", err
+	}
+	return qc.Version(), nil
+}
+
+// LoadConfig - loads json config from filename for the a given struct data
+func LoadConfig(filename string, clnt etcd.Client, data interface{}) (qc Config, err error) {
+	qc, err = NewConfig(data, clnt)
+	if err != nil {
+		return nil, err
+	}
+	return qc, qc.Load(filename)
+}
+
+// SaveConfig - saves given configuration data into given file as JSON.
+func SaveConfig(data interface{}, filename string, clnt etcd.Client) (err error) {
+	if err = checkData(data); err != nil {
+		return err
+	}
+	var qc Config
+	qc, err = NewConfig(data, clnt)
+	if err != nil {
+		return err
+	}
+	return qc.Save(filename)
+}
+
+// NewConfig loads config from etcd client if provided, otherwise loads from a local filename.
+// fails when all else fails.
+func NewConfig(data interface{}, clnt etcd.Client) (cfg Config, err error) {
 	if err := checkData(data); err != nil {
 		return nil, err
 	}
 
-	d := new(localConfig)
+	d := new(config)
 	d.data = data
+	d.clnt = clnt
 	d.lock = new(sync.RWMutex)
 	return d, nil
 }
-
-// GetLocalVersion - extracts the version information.
-func GetLocalVersion(filename string) (version string, err error) {
-	var qc Config
-	if qc, err = LoadLocalConfig(filename, &struct {
-		Version string
-	}{}); err != nil {
-		return "", err
-	}
-	return qc.Version(), err
-}
-
-// LoadLocalConfig - loads json config from filename for the a given struct data
-func LoadLocalConfig(filename string, data interface{}) (qc Config, err error) {
-	if qc, err = NewLocalConfig(data); err == nil {
-		err = qc.Load(filename)
-	}
-	return qc, err
-}
-
-// SaveLocalConfig - saves given configuration data into given file as JSON.
-func SaveLocalConfig(filename string, data interface{}) (err error) {
-	var qc Config
-	if qc, err = NewLocalConfig(data); err == nil {
-		err = qc.Save(filename)
-	}
-
-	return err
-}

pkg/quick/quick_etcd.go

@@ -1,204 +0,0 @@
-/*
- * Quick - Quick key value store for config files and persistent state files
- *
- * Quick (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package quick
-
-import (
-	"encoding/json"
-	"path/filepath"
-	"reflect"
-	"runtime"
-	"strings"
-	"sync"
-	"time"
-
-	etcdc "github.com/coreos/etcd/client"
-	"github.com/fatih/structs"
-	"golang.org/x/net/context"
-)
-
-// etcdConfig - implements quick.Config interface
-type etcdConfig struct {
-	lock *sync.Mutex
-	data interface{}
-	clnt etcdc.Client
-}
-
-// Version returns the current config file format version
-func (d etcdConfig) Version() string {
-	st := structs.New(d.data)
-	f := st.Field("Version")
-	return f.Value().(string)
-}
-
-// String converts JSON config to printable string
-func (d etcdConfig) String() string {
-	configBytes, _ := json.MarshalIndent(d.data, "", "\t")
-	return string(configBytes)
-}
-
-// Save writes config data to an configured etcd endpoints. Data format
-// is selected based on file extension or JSON if not provided.
-func (d etcdConfig) Save(filename string) error {
-	d.lock.Lock()
-	defer d.lock.Unlock()
-
-	// Fetch filename's extension
-	ext := filepath.Ext(filename)
-	// Marshal data
-	dataBytes, err := toMarshaller(ext)(d.data)
-	if err != nil {
-		return err
-	}
-	if runtime.GOOS == "windows" {
-		dataBytes = []byte(strings.Replace(string(dataBytes), "\n", "\r\n", -1))
-	}
-
-	kapi := etcdc.NewKeysAPI(d.clnt)
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	_, err = kapi.Update(ctx, filename, string(dataBytes))
-	cancel()
-	return err
-}
-
-// Load - loads config from file and merge with currently set values
-// File content format is guessed from the file name extension, if not
-// available, consider that we have JSON.
-func (d etcdConfig) Load(filename string) error {
-	d.lock.Lock()
-	defer d.lock.Unlock()
-
-	kapi := etcdc.NewKeysAPI(d.clnt)
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	resp, err := kapi.Get(ctx, filename, nil)
-	cancel()
-	if err != nil {
-		return err
-	}
-
-	var ev *etcdc.Node
-	switch {
-	case resp.Node.Dir:
-		for _, ev = range resp.Node.Nodes {
-			if string(ev.Key) == filename {
-				break
-			}
-		}
-	default:
-		ev = resp.Node
-	}
-
-	fileData := ev.Value
-	if runtime.GOOS == "windows" {
-		fileData = strings.Replace(ev.Value, "\r\n", "\n", -1)
-	}
-
-	// Unmarshal file's content
-	return toUnmarshaller(filepath.Ext(filename))([]byte(fileData), d.data)
-}
-
-// Data - grab internal data map for reading
-func (d etcdConfig) Data() interface{} {
-	return d.data
-}
-
-//Diff  - list fields that are in A but not in B
-func (d etcdConfig) Diff(c Config) ([]structs.Field, error) {
-	var fields []structs.Field
-
-	currFields := structs.Fields(d.Data())
-	newFields := structs.Fields(c.Data())
-
-	var found bool
-	for _, currField := range currFields {
-		found = false
-		for _, newField := range newFields {
-			if reflect.DeepEqual(currField.Name(), newField.Name()) {
-				found = true
-			}
-		}
-		if !found {
-			fields = append(fields, *currField)
-		}
-	}
-	return fields, nil
-}
-
-// DeepDiff  - list fields in A that are missing or not equal to fields in B
-func (d etcdConfig) DeepDiff(c Config) ([]structs.Field, error) {
-	var fields []structs.Field
-
-	currFields := structs.Fields(d.Data())
-	newFields := structs.Fields(c.Data())
-
-	var found bool
-	for _, currField := range currFields {
-		found = false
-		for _, newField := range newFields {
-			if reflect.DeepEqual(currField.Value(), newField.Value()) {
-				found = true
-			}
-		}
-		if !found {
-			fields = append(fields, *currField)
-		}
-	}
-	return fields, nil
-}
-
-// NewEtcdConfig - instantiate a new etcd config
-func NewEtcdConfig(data interface{}, clnt etcdc.Client) (Config, error) {
-	if err := checkData(data); err != nil {
-		return nil, err
-	}
-
-	d := new(etcdConfig)
-	d.data = data
-	d.clnt = clnt
-	d.lock = &sync.Mutex{}
-	return d, nil
-}
-
-// GetEtcdVersion - extracts the version information.
-func GetEtcdVersion(filename string, clnt etcdc.Client) (version string, err error) {
-	var qc Config
-	if qc, err = LoadEtcdConfig(filename, &struct {
-		Version string
-	}{}, clnt); err != nil {
-		return "", err
-	}
-	return qc.Version(), err
-}
-
-// LoadEtcdConfig - loads json config from etcd backend for the given struct data
-func LoadEtcdConfig(filename string, data interface{}, clnt etcdc.Client) (qc Config, err error) {
-	if qc, err = NewEtcdConfig(data, clnt); err == nil {
-		err = qc.Load(filename)
-	}
-	return qc, err
-}
-
-// SaveEtcdConfig - saves given configuration data into etcd backend.
-func SaveEtcdConfig(filename string, data interface{}, clnt etcdc.Client) (err error) {
-	var qc Config
-	if qc, err = NewEtcdConfig(data, clnt); err == nil {
-		err = qc.Save(filename)
-	}
-
-	return err
-}

pkg/quick/quick_test.go

@@ -36,7 +36,7 @@ func TestReadVersion(t *testing.T) {
 		Version string
 	}
 	saveMe := myStruct{"1"}
-	config, err := NewLocalConfig(&saveMe)
+	config, err := NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -45,7 +45,7 @@ func TestReadVersion(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	version, err := GetLocalVersion("test.json")
+	version, err := GetVersion("test.json", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -59,7 +59,7 @@ func TestReadVersionErr(t *testing.T) {
 		Version int
 	}
 	saveMe := myStruct{1}
-	_, err := NewLocalConfig(&saveMe)
+	_, err := NewConfig(&saveMe, nil)
 	if err == nil {
 		t.Fatal("Unexpected should fail in initialization for bad input")
 	}
@@ -69,7 +69,7 @@ func TestReadVersionErr(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = GetLocalVersion("test.json")
+	_, err = GetVersion("test.json", nil)
 	if err == nil {
 		t.Fatal("Unexpected should fail to fetch version")
 	}
@@ -79,7 +79,7 @@ func TestReadVersionErr(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = GetLocalVersion("test.json")
+	_, err = GetVersion("test.json", nil)
 	if err == nil {
 		t.Fatal("Unexpected should fail to fetch version")
 	}
@@ -95,7 +95,7 @@ func TestSaveFailOnDir(t *testing.T) {
 		Version string
 	}
 	saveMe := myStruct{"1"}
-	config, err := NewLocalConfig(&saveMe)
+	config, err := NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -155,7 +155,7 @@ func TestLoadFile(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{}
-	_, err := LoadLocalConfig("test.json", &saveMe)
+	_, err := LoadConfig("test.json", nil, &saveMe)
 	if err == nil {
 		t.Fatal(err)
 	}
@@ -167,11 +167,11 @@ func TestLoadFile(t *testing.T) {
 	if err = file.Close(); err != nil {
 		t.Fatal(err)
 	}
-	_, err = LoadLocalConfig("test.json", &saveMe)
+	_, err = LoadConfig("test.json", nil, &saveMe)
 	if err == nil {
 		t.Fatal("Unexpected should fail to load empty JSON")
 	}
-	config, err := NewLocalConfig(&saveMe)
+	config, err := NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -186,7 +186,7 @@ func TestLoadFile(t *testing.T) {
 	}
 
 	saveMe = myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err = NewLocalConfig(&saveMe)
+	config, err = NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -195,7 +195,7 @@ func TestLoadFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	saveMe1 := myStruct{}
-	_, err = LoadLocalConfig("test.json", &saveMe1)
+	_, err = LoadConfig("test.json", nil, &saveMe1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -240,7 +240,7 @@ directories:
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
 
 	// Save format using
-	config, err := NewLocalConfig(&saveMe)
+	config, err := NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -262,7 +262,7 @@ directories:
 
 	// Check if the loaded data is the same as the saved one
 	loadMe := myStruct{}
-	config, err = NewLocalConfig(&loadMe)
+	config, err = NewConfig(&loadMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -306,7 +306,7 @@ func TestJSONFormat(t *testing.T) {
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
 
 	// Save format using
-	config, err := NewLocalConfig(&saveMe)
+	config, err := NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -328,7 +328,7 @@ func TestJSONFormat(t *testing.T) {
 
 	// Check if the loaded data is the same as the saved one
 	loadMe := myStruct{}
-	config, err = NewLocalConfig(&loadMe)
+	config, err = NewConfig(&loadMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -351,7 +351,7 @@ func TestSaveLoad(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err := NewLocalConfig(&saveMe)
+	config, err := NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -361,7 +361,7 @@ func TestSaveLoad(t *testing.T) {
 	}
 
 	loadMe := myStruct{Version: "1"}
-	newConfig, err := NewLocalConfig(&loadMe)
+	newConfig, err := NewConfig(&loadMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -393,7 +393,7 @@ func TestSaveBackup(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err := NewLocalConfig(&saveMe)
+	config, err := NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -403,7 +403,7 @@ func TestSaveBackup(t *testing.T) {
 	}
 
 	loadMe := myStruct{Version: "1"}
-	newConfig, err := NewLocalConfig(&loadMe)
+	newConfig, err := NewConfig(&loadMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -424,7 +424,7 @@ func TestSaveBackup(t *testing.T) {
 		t.Fatal("Expected to mismatch but succeeded instead")
 	}
 
-	config, err = NewLocalConfig(&mismatch)
+	config, err = NewConfig(&mismatch, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -442,20 +442,20 @@ func TestDiff(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err := NewLocalConfig(&saveMe)
+	config, err := NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	type myNewLocalConfigStruct struct {
+	type myNewConfigStruct struct {
 		Version string
 		// User     string
 		Password    string
 		Directories []string
 	}
 
-	mismatch := myNewLocalConfigStruct{"1", "nopassword", []string{"Work", "documents", "Music"}}
-	newConfig, err := NewLocalConfig(&mismatch)
+	mismatch := myNewConfigStruct{"1", "nopassword", []string{"Work", "documents", "Music"}}
+	newConfig, err := NewConfig(&mismatch, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -482,13 +482,13 @@ func TestDeepDiff(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err := NewLocalConfig(&saveMe)
+	config, err := NewConfig(&saveMe, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	mismatch := myStruct{"1", "Guest", "nopassword", []string{"Work", "documents", "Music"}}
-	newConfig, err := NewLocalConfig(&mismatch)
+	newConfig, err := NewConfig(&mismatch, nil)
 	if err != nil {
 		t.Fatal(err)
 	}