Bring etcd support for bucket DNS federation

- Supports centralized `config.json`
- Supports centralized `bucket` service records
  for client lookups
- implement a new proxy forwarder

pkg/quick/quick.go

@@ -41,21 +41,21 @@ type Config interface {
 	DeepDiff(Config) ([]structs.Field, error)
 }
 
-// config - implements quick.Config interface
-type config struct {
+// localConfig - implements quick.Config interface
+type localConfig struct {
 	data interface{}
 	lock *sync.RWMutex
 }
 
 // Version returns the current config file format version
-func (d config) Version() string {
+func (d localConfig) Version() string {
 	st := structs.New(d.data)
 	f := st.Field("Version")
 	return f.Value().(string)
 }
 
 // String converts JSON config to printable string
-func (d config) String() string {
+func (d localConfig) String() string {
 	configBytes, _ := json.MarshalIndent(d.data, "", "\t")
 	return string(configBytes)
 }
@@ -63,7 +63,7 @@ func (d config) String() string {
 // Save writes config data to a file. Data format
 // is selected based on file extension or JSON if
 // not provided.
-func (d config) Save(filename string) error {
+func (d localConfig) Save(filename string) error {
 	d.lock.Lock()
 	defer d.lock.Unlock()
 
@@ -89,19 +89,19 @@ func (d config) Save(filename string) error {
 // Load - loads config from file and merge with currently set values
 // File content format is guessed from the file name extension, if not
 // available, consider that we have JSON.
-func (d config) Load(filename string) error {
+func (d localConfig) Load(filename string) error {
 	d.lock.Lock()
 	defer d.lock.Unlock()
 	return loadFileConfig(filename, d.data)
 }
 
 // Data - grab internal data map for reading
-func (d config) Data() interface{} {
+func (d localConfig) Data() interface{} {
 	return d.data
 }
 
-//Diff  - list fields that are in A but not in B
-func (d config) Diff(c Config) ([]structs.Field, error) {
+// Diff  - list fields that are in A but not in B
+func (d localConfig) Diff(c Config) ([]structs.Field, error) {
 	var fields []structs.Field
 
 	currFields := structs.Fields(d.Data())
@@ -123,7 +123,7 @@ func (d config) Diff(c Config) ([]structs.Field, error) {
 }
 
 // DeepDiff  - list fields in A that are missing or not equal to fields in B
-func (d config) DeepDiff(c Config) ([]structs.Field, error) {
+func (d localConfig) DeepDiff(c Config) ([]structs.Field, error) {
 	var fields []structs.Field
 
 	currFields := structs.Fields(d.Data())
@@ -179,13 +179,13 @@ func writeFile(filename string, data []byte) error {
 	return safeFile.Close()
 }
 
-// New - instantiate a new config
-func New(data interface{}) (Config, error) {
+// NewLocalConfig - instantiate a new config r/w configs from local files.
+func NewLocalConfig(data interface{}) (Config, error) {
 	if err := checkData(data); err != nil {
 		return nil, err
 	}
 
-	d := new(config)
+	d := new(localConfig)
 	d.data = data
 	d.lock = new(sync.RWMutex)
 	return d, nil
@@ -204,7 +204,7 @@ func GetVersion(filename string) (version string, err error) {
 
 // Load - loads json config from filename for the a given struct data
 func Load(filename string, data interface{}) (qc Config, err error) {
-	if qc, err = New(data); err == nil {
+	if qc, err = NewLocalConfig(data); err == nil {
 		err = qc.Load(filename)
 	}
 	return qc, err
@@ -213,7 +213,7 @@ func Load(filename string, data interface{}) (qc Config, err error) {
 // Save - saves given configuration data into given file as JSON.
 func Save(filename string, data interface{}) (err error) {
 	var qc Config
-	if qc, err = New(data); err == nil {
+	if qc, err = NewLocalConfig(data); err == nil {
 		err = qc.Save(filename)
 	}
 

pkg/quick/quick_etcd.go

@@ -0,0 +1,175 @@
+/*
+ * Quick - Quick key value store for config files and persistent state files
+ *
+ * Quick (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package quick
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	etcdc "github.com/coreos/etcd/client"
+	"github.com/fatih/structs"
+	"golang.org/x/net/context"
+)
+
+// etcdConfig - implements quick.Config interface
+type etcdConfig struct {
+	lock *sync.Mutex
+	data interface{}
+	clnt etcdc.Client
+}
+
+// Version returns the current config file format version
+func (d etcdConfig) Version() string {
+	st := structs.New(d.data)
+	f := st.Field("Version")
+	return f.Value().(string)
+}
+
+// String converts JSON config to printable string
+func (d etcdConfig) String() string {
+	configBytes, _ := json.MarshalIndent(d.data, "", "\t")
+	return string(configBytes)
+}
+
+// Save writes config data to an configured etcd endpoints. Data format
+// is selected based on file extension or JSON if not provided.
+func (d etcdConfig) Save(filename string) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	// Fetch filename's extension
+	ext := filepath.Ext(filename)
+	// Marshal data
+	dataBytes, err := toMarshaller(ext)(d.data)
+	if err != nil {
+		return err
+	}
+	if runtime.GOOS == "windows" {
+		dataBytes = []byte(strings.Replace(string(dataBytes), "\n", "\r\n", -1))
+	}
+
+	kapi := etcdc.NewKeysAPI(d.clnt)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	_, err = kapi.Create(ctx, filename, string(dataBytes))
+	cancel()
+	return err
+}
+
+// Load - loads config from file and merge with currently set values
+// File content format is guessed from the file name extension, if not
+// available, consider that we have JSON.
+func (d etcdConfig) Load(filename string) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	kapi := etcdc.NewKeysAPI(d.clnt)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	resp, err := kapi.Get(ctx, filename, nil)
+	cancel()
+	if err != nil {
+		return err
+	}
+
+	var ev *etcdc.Node
+	switch {
+	case resp.Node.Dir:
+		for _, ev = range resp.Node.Nodes {
+			if string(ev.Key) == filename {
+				break
+			}
+		}
+	default:
+		ev = resp.Node
+	}
+
+	fileData := ev.Value
+	if runtime.GOOS == "windows" {
+		fileData = strings.Replace(ev.Value, "\r\n", "\n", -1)
+	}
+
+	// Unmarshal file's content
+	return toUnmarshaller(filepath.Ext(filename))([]byte(fileData), d.data)
+}
+
+// Data - grab internal data map for reading
+func (d etcdConfig) Data() interface{} {
+	return d.data
+}
+
+//Diff  - list fields that are in A but not in B
+func (d etcdConfig) Diff(c Config) ([]structs.Field, error) {
+	var fields []structs.Field
+
+	currFields := structs.Fields(d.Data())
+	newFields := structs.Fields(c.Data())
+
+	var found bool
+	for _, currField := range currFields {
+		found = false
+		for _, newField := range newFields {
+			if reflect.DeepEqual(currField.Name(), newField.Name()) {
+				found = true
+			}
+		}
+		if !found {
+			fields = append(fields, *currField)
+		}
+	}
+	return fields, nil
+}
+
+// DeepDiff  - list fields in A that are missing or not equal to fields in B
+func (d etcdConfig) DeepDiff(c Config) ([]structs.Field, error) {
+	var fields []structs.Field
+
+	currFields := structs.Fields(d.Data())
+	newFields := structs.Fields(c.Data())
+
+	var found bool
+	for _, currField := range currFields {
+		found = false
+		for _, newField := range newFields {
+			if reflect.DeepEqual(currField.Value(), newField.Value()) {
+				found = true
+			}
+		}
+		if !found {
+			fields = append(fields, *currField)
+		}
+	}
+	return fields, nil
+}
+
+// NewEtcdConfig - instantiate a new etcd config
+func NewEtcdConfig(data interface{}, clnt etcdc.Client) (Config, error) {
+	if err := checkData(data); err != nil {
+		return nil, err
+	}
+
+	d := new(etcdConfig)
+	d.data = data
+	d.clnt = clnt
+	d.lock = &sync.Mutex{}
+	return d, nil
+}

pkg/quick/quick_test.go

@@ -36,7 +36,7 @@ func TestReadVersion(t *testing.T) {
 		Version string
 	}
 	saveMe := myStruct{"1"}
-	config, err := New(&saveMe)
+	config, err := NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -59,7 +59,7 @@ func TestReadVersionErr(t *testing.T) {
 		Version int
 	}
 	saveMe := myStruct{1}
-	_, err := New(&saveMe)
+	_, err := NewLocalConfig(&saveMe)
 	if err == nil {
 		t.Fatal("Unexpected should fail in initialization for bad input")
 	}
@@ -95,7 +95,7 @@ func TestSaveFailOnDir(t *testing.T) {
 		Version string
 	}
 	saveMe := myStruct{"1"}
-	config, err := New(&saveMe)
+	config, err := NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -171,7 +171,7 @@ func TestLoadFile(t *testing.T) {
 	if err == nil {
 		t.Fatal("Unexpected should fail to load empty JSON")
 	}
-	config, err := New(&saveMe)
+	config, err := NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -186,7 +186,7 @@ func TestLoadFile(t *testing.T) {
 	}
 
 	saveMe = myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err = New(&saveMe)
+	config, err = NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -240,7 +240,7 @@ directories:
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
 
 	// Save format using
-	config, err := New(&saveMe)
+	config, err := NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -262,7 +262,7 @@ directories:
 
 	// Check if the loaded data is the same as the saved one
 	loadMe := myStruct{}
-	config, err = New(&loadMe)
+	config, err = NewLocalConfig(&loadMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -306,7 +306,7 @@ func TestJSONFormat(t *testing.T) {
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
 
 	// Save format using
-	config, err := New(&saveMe)
+	config, err := NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -328,7 +328,7 @@ func TestJSONFormat(t *testing.T) {
 
 	// Check if the loaded data is the same as the saved one
 	loadMe := myStruct{}
-	config, err = New(&loadMe)
+	config, err = NewLocalConfig(&loadMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -351,7 +351,7 @@ func TestSaveLoad(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err := New(&saveMe)
+	config, err := NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -361,7 +361,7 @@ func TestSaveLoad(t *testing.T) {
 	}
 
 	loadMe := myStruct{Version: "1"}
-	newConfig, err := New(&loadMe)
+	newConfig, err := NewLocalConfig(&loadMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -393,7 +393,7 @@ func TestSaveBackup(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err := New(&saveMe)
+	config, err := NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -403,7 +403,7 @@ func TestSaveBackup(t *testing.T) {
 	}
 
 	loadMe := myStruct{Version: "1"}
-	newConfig, err := New(&loadMe)
+	newConfig, err := NewLocalConfig(&loadMe)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -424,7 +424,7 @@ func TestSaveBackup(t *testing.T) {
 		t.Fatal("Expected to mismatch but succeeded instead")
 	}
 
-	config, err = New(&mismatch)
+	config, err = NewLocalConfig(&mismatch)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -442,20 +442,20 @@ func TestDiff(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err := New(&saveMe)
+	config, err := NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	type myNewStruct struct {
+	type myNewLocalConfigStruct struct {
 		Version string
 		// User     string
 		Password    string
 		Directories []string
 	}
 
-	mismatch := myNewStruct{"1", "nopassword", []string{"Work", "documents", "Music"}}
-	newConfig, err := New(&mismatch)
+	mismatch := myNewLocalConfigStruct{"1", "nopassword", []string{"Work", "documents", "Music"}}
+	newConfig, err := NewLocalConfig(&mismatch)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -482,13 +482,13 @@ func TestDeepDiff(t *testing.T) {
 		Directories []string
 	}
 	saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
-	config, err := New(&saveMe)
+	config, err := NewLocalConfig(&saveMe)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	mismatch := myStruct{"1", "Guest", "nopassword", []string{"Work", "documents", "Music"}}
-	newConfig, err := New(&mismatch)
+	newConfig, err := NewLocalConfig(&mismatch)
 	if err != nil {
 		t.Fatal(err)
 	}