xl: Moved to minio/minio - fixes #1112

This commit is contained in:
Harshavardhana
2016-02-10 16:40:09 -08:00
parent 33bd97d581
commit 62f6ffb6db
137 changed files with 9408 additions and 515 deletions

96
pkg/quick/errorutil.go Normal file
View File

@@ -0,0 +1,96 @@
/*
* Quick - Quick key value store for config files and persistent state files
*
* Minio Client (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package quick
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"github.com/olekukonko/ts"
)
const errorFmt = "%5d: %s <-- "
// FormatJSONSyntaxError generates a pretty printed json syntax error since
// golang doesn't provide an easy way to report the location of the error
func FormatJSONSyntaxError(data io.Reader, sErr *json.SyntaxError) error {
if sErr == nil {
return nil
}
var readLine bytes.Buffer
bio := bufio.NewReader(data)
errLine := int64(1)
readBytes := int64(0)
// termWidth is set to a default one to use when we are
// not able to calculate terminal width via OS syscalls
termWidth := 25
// errorShift is the length of the minimum needed place for
// error msg accessoires, like <--, etc.. We calculate it
// dynamically to avoid an eventual bug after modifying errorFmt
errorShift := len(fmt.Sprintf(errorFmt, 1, ""))
if termSize, err := ts.GetSize(); err == nil {
termWidth = termSize.Col()
}
for {
b, err := bio.ReadByte()
if err != nil {
if err != io.EOF {
return err
}
break
}
readBytes++
if readBytes > sErr.Offset {
break
}
switch b {
case '\n':
readLine.Reset()
errLine++
case '\t':
readLine.WriteByte(' ')
case '\r':
break
default:
readLine.WriteByte(b)
}
}
lineLen := readLine.Len()
idx := lineLen - termWidth + errorShift
if idx < 0 || idx > lineLen-1 {
idx = 0
}
errorStr := fmt.Sprintf("JSON syntax error at line %d, col %d : %s.\n",
errLine, readLine.Len(), sErr)
errorStr += fmt.Sprintf(errorFmt, errLine, readLine.String()[idx:])
return errors.New(errorStr)
}

346
pkg/quick/quick.go Normal file
View File

@@ -0,0 +1,346 @@
/*
* Quick - Quick key value store for config files and persistent state files
*
* Minio Client (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package quick
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"reflect"
"runtime"
"strings"
"sync"
"github.com/fatih/structs"
"github.com/minio/minio/pkg/atomic"
"github.com/minio/minio/pkg/probe"
)
// Config - generic config interface functions
type Config interface {
String() string
Version() string
Save(string) *probe.Error
Load(string) *probe.Error
Data() interface{}
Diff(Config) ([]structs.Field, *probe.Error)
DeepDiff(Config) ([]structs.Field, *probe.Error)
}
// config - implements quick.Config interface
type config struct {
data interface{}
lock *sync.RWMutex
}
// CheckData - checks the validity of config data. Data should be of
// type struct and contain a string type field called "Version".
func CheckData(data interface{}) *probe.Error {
if !structs.IsStruct(data) {
return probe.NewError(fmt.Errorf("Invalid argument type. Expecing \"struct\" type."))
}
st := structs.New(data)
f, ok := st.FieldOk("Version")
if !ok {
return probe.NewError(fmt.Errorf("Invalid type of struct argument. No [%s.Version] field found.", st.Name()))
}
if f.Kind() != reflect.String {
return probe.NewError(fmt.Errorf("Invalid type of struct argument. Expecting \"string\" type [%s.Version] field.", st.Name()))
}
return nil
}
// New - instantiate a new config
func New(data interface{}) (Config, *probe.Error) {
if err := CheckData(data); err != nil {
return nil, err.Trace()
}
d := new(config)
d.data = data
d.lock = new(sync.RWMutex)
return d, nil
}
// CheckVersion - loads json and compares the version number provided returns back true or false - any failure
// is returned as error.
func CheckVersion(filename string, version string) (bool, *probe.Error) {
_, e := os.Stat(filename)
if e != nil {
return false, probe.NewError(e)
}
fileData, e := ioutil.ReadFile(filename)
if e != nil {
return false, probe.NewError(e)
}
if runtime.GOOS == "windows" {
fileData = []byte(strings.Replace(string(fileData), "\r\n", "\n", -1))
}
data := struct {
Version string
}{
Version: "",
}
e = json.Unmarshal(fileData, &data)
if e != nil {
switch e := e.(type) {
case *json.SyntaxError:
return false, probe.NewError(FormatJSONSyntaxError(bytes.NewReader(fileData), e))
default:
return false, probe.NewError(e)
}
}
config, err := New(data)
if err != nil {
return false, err.Trace()
}
if config.Version() != version {
return false, nil
}
return true, nil
}
// Load - loads json config from filename for the a given struct data
func Load(filename string, data interface{}) (Config, *probe.Error) {
_, e := os.Stat(filename)
if e != nil {
return nil, probe.NewError(e)
}
fileData, e := ioutil.ReadFile(filename)
if e != nil {
return nil, probe.NewError(e)
}
if runtime.GOOS == "windows" {
fileData = []byte(strings.Replace(string(fileData), "\r\n", "\n", -1))
}
e = json.Unmarshal(fileData, &data)
if e != nil {
switch e := e.(type) {
case *json.SyntaxError:
return nil, probe.NewError(FormatJSONSyntaxError(bytes.NewReader(fileData), e))
default:
return nil, probe.NewError(e)
}
}
config, err := New(data)
if err != nil {
return nil, err.Trace()
}
return config, nil
}
// Version returns the current config file format version
func (d config) Version() string {
st := structs.New(d.data)
f, ok := st.FieldOk("Version")
if !ok {
return ""
}
val := f.Value()
ver, ok := val.(string)
if ok {
return ver
}
return ""
}
// writeFile writes data to a file named by filename.
// If the file does not exist, writeFile creates it;
// otherwise writeFile truncates it before writing.
func writeFile(filename string, data []byte) *probe.Error {
atomicFile, e := atomic.FileCreate(filename)
if e != nil {
return probe.NewError(e)
}
_, e = atomicFile.Write(data)
if e != nil {
return probe.NewError(e)
}
e = atomicFile.Close()
if e != nil {
return probe.NewError(e)
}
return nil
}
// String converts JSON config to printable string
func (d config) String() string {
configBytes, _ := json.MarshalIndent(d.data, "", "\t")
return string(configBytes)
}
// Save writes config data in JSON format to a file.
func (d config) Save(filename string) *probe.Error {
d.lock.Lock()
defer d.lock.Unlock()
// Check for existing file, if yes create a backup.
st, e := os.Stat(filename)
// If file exists and stat failed return here.
if e != nil && !os.IsNotExist(e) {
return probe.NewError(e)
}
// File exists and proceed to take backup.
if e == nil {
// File exists and is not a regular file return error.
if !st.Mode().IsRegular() {
return probe.NewError(fmt.Errorf("%s is not a regular file", filename))
}
// Read old data.
var oldData []byte
oldData, e = ioutil.ReadFile(filename)
if e != nil {
return probe.NewError(e)
}
// Save read data to the backup file.
if err := writeFile(filename+".old", oldData); err != nil {
return err.Trace(filename + ".old")
}
}
// Proceed to create or overwrite file.
jsonData, e := json.MarshalIndent(d.data, "", "\t")
if e != nil {
return probe.NewError(e)
}
if runtime.GOOS == "windows" {
jsonData = []byte(strings.Replace(string(jsonData), "\n", "\r\n", -1))
}
// Save data.
err := writeFile(filename, jsonData)
return err.Trace(filename)
}
// Load - loads JSON config from file and merge with currently set values
func (d *config) Load(filename string) *probe.Error {
d.lock.Lock()
defer d.lock.Unlock()
_, e := os.Stat(filename)
if e != nil {
return probe.NewError(e)
}
fileData, e := ioutil.ReadFile(filename)
if e != nil {
return probe.NewError(e)
}
if runtime.GOOS == "windows" {
fileData = []byte(strings.Replace(string(fileData), "\r\n", "\n", -1))
}
st := structs.New(d.data)
f, ok := st.FieldOk("Version")
if !ok {
return probe.NewError(fmt.Errorf("Argument struct [%s] does not contain field \"Version\".", st.Name()))
}
e = json.Unmarshal(fileData, d.data)
if e != nil {
switch e := e.(type) {
case *json.SyntaxError:
return probe.NewError(FormatJSONSyntaxError(bytes.NewReader(fileData), e))
default:
return probe.NewError(e)
}
}
if err := CheckData(d.data); err != nil {
return err.Trace(filename)
}
if (*d).Version() != f.Value() {
return probe.NewError(fmt.Errorf("Version mismatch"))
}
return nil
}
// Data - grab internal data map for reading
func (d config) Data() interface{} {
return d.data
}
//Diff - list fields that are in A but not in B
func (d config) Diff(c Config) ([]structs.Field, *probe.Error) {
var fields []structs.Field
err := CheckData(c.Data())
if err != nil {
return []structs.Field{}, err.Trace()
}
currFields := structs.Fields(d.Data())
newFields := structs.Fields(c.Data())
found := false
for _, currField := range currFields {
found = false
for _, newField := range newFields {
if reflect.DeepEqual(currField.Name(), newField.Name()) {
found = true
}
}
if !found {
fields = append(fields, *currField)
}
}
return fields, nil
}
//DeepDiff - list fields in A that are missing or not equal to fields in B
func (d config) DeepDiff(c Config) ([]structs.Field, *probe.Error) {
var fields []structs.Field
err := CheckData(c.Data())
if err != nil {
return []structs.Field{}, err.Trace()
}
currFields := structs.Fields(d.Data())
newFields := structs.Fields(c.Data())
found := false
for _, currField := range currFields {
found = false
for _, newField := range newFields {
if reflect.DeepEqual(currField.Value(), newField.Value()) {
found = true
}
}
if !found {
fields = append(fields, *currField)
}
}
return fields, nil
}

222
pkg/quick/quick_test.go Normal file
View File

@@ -0,0 +1,222 @@
/*
* Quick - Quick key value store for config files and persistent state files
*
* Minio Client (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package quick_test
import (
"os"
"testing"
"github.com/minio/minio/pkg/quick"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestSaveFailOnDir(c *C) {
defer os.RemoveAll("test.json")
e := os.MkdirAll("test.json", 0644)
c.Assert(e, IsNil)
type myStruct struct {
Version string
}
saveMe := myStruct{"1"}
config, err := quick.New(&saveMe)
c.Assert(err, IsNil)
c.Assert(config, Not(IsNil))
err = config.Save("test.json")
c.Assert(err, Not(IsNil))
}
func (s *MySuite) TestCheckData(c *C) {
err := quick.CheckData(nil)
c.Assert(err, Not(IsNil))
type myStructBad struct {
User string
Password string
Folders []string
}
saveMeBad := myStructBad{"guest", "nopassword", []string{"Work", "Documents", "Music"}}
err = quick.CheckData(&saveMeBad)
c.Assert(err, Not(IsNil))
type myStructGood struct {
Version string
User string
Password string
Folders []string
}
saveMeGood := myStructGood{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
err = quick.CheckData(&saveMeGood)
c.Assert(err, IsNil)
}
func (s *MySuite) TestVersion(c *C) {
defer os.RemoveAll("test.json")
type myStruct struct {
Version string
User string
Password string
Folders []string
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err := quick.New(&saveMe)
c.Assert(err, IsNil)
c.Assert(config, Not(IsNil))
err = config.Save("test.json")
c.Assert(err, IsNil)
valid, err := quick.CheckVersion("test.json", "1")
c.Assert(err, IsNil)
c.Assert(valid, Equals, true)
valid, err = quick.CheckVersion("test.json", "2")
c.Assert(err, IsNil)
c.Assert(valid, Equals, false)
}
func (s *MySuite) TestSaveLoad(c *C) {
defer os.RemoveAll("test.json")
type myStruct struct {
Version string
User string
Password string
Folders []string
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err := quick.New(&saveMe)
c.Assert(err, IsNil)
c.Assert(config, Not(IsNil))
err = config.Save("test.json")
c.Assert(err, IsNil)
loadMe := myStruct{Version: "1"}
newConfig, err := quick.New(&loadMe)
c.Assert(err, IsNil)
c.Assert(newConfig, Not(IsNil))
err = newConfig.Load("test.json")
c.Assert(err, IsNil)
c.Assert(config.Data(), DeepEquals, newConfig.Data())
c.Assert(config.Data(), DeepEquals, &loadMe)
mismatch := myStruct{"1.1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
c.Assert(newConfig.Data(), Not(DeepEquals), &mismatch)
}
func (s *MySuite) TestSaveBackup(c *C) {
defer os.RemoveAll("test.json")
defer os.RemoveAll("test.json.old")
type myStruct struct {
Version string
User string
Password string
Folders []string
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err := quick.New(&saveMe)
c.Assert(err, IsNil)
c.Assert(config, Not(IsNil))
err = config.Save("test.json")
c.Assert(err, IsNil)
loadMe := myStruct{Version: "1"}
newConfig, err := quick.New(&loadMe)
c.Assert(err, IsNil)
c.Assert(newConfig, Not(IsNil))
err = newConfig.Load("test.json")
c.Assert(err, IsNil)
c.Assert(config.Data(), DeepEquals, newConfig.Data())
c.Assert(config.Data(), DeepEquals, &loadMe)
mismatch := myStruct{"1.1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
c.Assert(newConfig.Data(), Not(DeepEquals), &mismatch)
config, err = quick.New(&mismatch)
c.Assert(err, IsNil)
c.Assert(config, Not(IsNil))
err = config.Save("test.json")
c.Assert(err, IsNil)
}
func (s *MySuite) TestDiff(c *C) {
type myStruct struct {
Version string
User string
Password string
Folders []string
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err := quick.New(&saveMe)
c.Assert(err, IsNil)
c.Assert(config, Not(IsNil))
type myNewStruct struct {
Version string
// User string
Password string
Folders []string
}
mismatch := myNewStruct{"1", "nopassword", []string{"Work", "documents", "Music"}}
newConfig, err := quick.New(&mismatch)
c.Assert(err, IsNil)
c.Assert(newConfig, Not(IsNil))
fields, ok := config.Diff(newConfig)
c.Assert(ok, IsNil)
c.Assert(len(fields), Equals, 1)
// Uncomment for debugging
// for i, field := range fields {
// fmt.Printf("Diff[%d]: %s=%v\n", i, field.Name(), field.Value())
// }
}
func (s *MySuite) TestDeepDiff(c *C) {
type myStruct struct {
Version string
User string
Password string
Folders []string
}
saveMe := myStruct{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
config, err := quick.New(&saveMe)
c.Assert(err, IsNil)
c.Assert(config, Not(IsNil))
mismatch := myStruct{"1", "Guest", "nopassword", []string{"Work", "documents", "Music"}}
newConfig, err := quick.New(&mismatch)
c.Assert(err, IsNil)
c.Assert(newConfig, Not(IsNil))
fields, err := config.DeepDiff(newConfig)
c.Assert(err, IsNil)
c.Assert(len(fields), Equals, 2)
// Uncomment for debugging
// for i, field := range fields {
// fmt.Printf("DeepDiff[%d]: %s=%v\n", i, field.Name(), field.Value())
// }
}