mirror of https://github.com/minio/minio.git
Merge pull request #397 from fkautz/pr_out_updating_iodine_to_match_new_api
This commit is contained in:
commit
9883af2337
|
@ -20,7 +20,7 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"ImportPath": "github.com/minio-io/iodine",
|
"ImportPath": "github.com/minio-io/iodine",
|
||||||
"Rev": "8c8985d5a45e388dccb5ca280c897672abb18278"
|
"Rev": "b44f7faaff4e722dcdb60b9788d33850606dc07b"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"ImportPath": "gopkg.in/check.v1",
|
"ImportPath": "gopkg.in/check.v1",
|
||||||
|
|
|
@ -30,7 +30,7 @@ import (
|
||||||
|
|
||||||
// WrappedError is the iodine error which contains a pointer to the original error
|
// WrappedError is the iodine error which contains a pointer to the original error
|
||||||
// and stack traces.
|
// and stack traces.
|
||||||
type WrappedError struct {
|
type Error struct {
|
||||||
EmbeddedError error `json:"-"`
|
EmbeddedError error `json:"-"`
|
||||||
ErrorMessage string
|
ErrorMessage string
|
||||||
|
|
||||||
|
@ -90,20 +90,20 @@ func GetGlobalStateKey(k string) string {
|
||||||
|
|
||||||
// Error - instantiate an error, turning it into an iodine error.
|
// Error - instantiate an error, turning it into an iodine error.
|
||||||
// Adds an initial stack trace.
|
// Adds an initial stack trace.
|
||||||
func Error(err error, data map[string]string) error {
|
func New(err error, data map[string]string) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
entry := createStackEntry()
|
entry := createStackEntry()
|
||||||
var newErr WrappedError
|
var newErr Error
|
||||||
|
|
||||||
// check if error is wrapped
|
// check if error is wrapped
|
||||||
switch typedError := err.(type) {
|
switch typedError := err.(type) {
|
||||||
case WrappedError:
|
case Error:
|
||||||
{
|
{
|
||||||
newErr = typedError
|
newErr = typedError
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
{
|
{
|
||||||
newErr = WrappedError{
|
newErr = Error{
|
||||||
EmbeddedError: err,
|
EmbeddedError: err,
|
||||||
ErrorMessage: err.Error(),
|
ErrorMessage: err.Error(),
|
||||||
Stack: []StackEntry{},
|
Stack: []StackEntry{},
|
||||||
|
@ -170,12 +170,12 @@ func getSystemData() map[string]string {
|
||||||
//}
|
//}
|
||||||
|
|
||||||
// EmitJSON writes JSON output for the error
|
// EmitJSON writes JSON output for the error
|
||||||
func (err WrappedError) EmitJSON() ([]byte, error) {
|
func (err Error) EmitJSON() ([]byte, error) {
|
||||||
return json.Marshal(err)
|
return json.Marshal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// EmitHumanReadable returns a human readable error message
|
// EmitHumanReadable returns a human readable error message
|
||||||
func (err WrappedError) EmitHumanReadable() string {
|
func (err Error) EmitHumanReadable() string {
|
||||||
var errorBuffer bytes.Buffer
|
var errorBuffer bytes.Buffer
|
||||||
fmt.Fprintln(&errorBuffer, err.ErrorMessage)
|
fmt.Fprintln(&errorBuffer, err.ErrorMessage)
|
||||||
for i, entry := range err.Stack {
|
for i, entry := range err.Stack {
|
||||||
|
@ -185,7 +185,7 @@ func (err WrappedError) EmitHumanReadable() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Emits the original error message
|
// Emits the original error message
|
||||||
func (err WrappedError) Error() string {
|
func (err Error) Error() string {
|
||||||
return err.EmitHumanReadable()
|
return err.EmitHumanReadable()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -24,12 +24,12 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestIodine(t *testing.T) {
|
func TestIodine(t *testing.T) {
|
||||||
iodineError := Error(errors.New("Hello"), nil)
|
iodineError := New(errors.New("Hello"), nil)
|
||||||
iodineError = Error(iodineError, nil)
|
iodineError = New(iodineError, nil)
|
||||||
iodineError = Error(iodineError, nil)
|
iodineError = New(iodineError, nil)
|
||||||
iodineError = Error(iodineError, nil)
|
iodineError = New(iodineError, nil)
|
||||||
switch typedError := iodineError.(type) {
|
switch typedError := iodineError.(type) {
|
||||||
case WrappedError:
|
case Error:
|
||||||
{
|
{
|
||||||
if len(typedError.Stack) != 4 {
|
if len(typedError.Stack) != 4 {
|
||||||
t.Fail()
|
t.Fail()
|
||||||
|
@ -63,9 +63,9 @@ func TestState(t *testing.T) {
|
||||||
t.Fail()
|
t.Fail()
|
||||||
}
|
}
|
||||||
SetGlobalState("foo", "bar")
|
SetGlobalState("foo", "bar")
|
||||||
err := Error(errors.New("a simple error"), nil)
|
err := New(errors.New("a simple error"), nil)
|
||||||
switch typedError := err.(type) {
|
switch typedError := err.(type) {
|
||||||
case WrappedError:
|
case Error:
|
||||||
{
|
{
|
||||||
if res, ok := typedError.Stack[0].Data["foo"]; ok {
|
if res, ok := typedError.Stack[0].Data["foo"]; ok {
|
||||||
if res != "bar" {
|
if res != "bar" {
|
||||||
|
@ -74,7 +74,7 @@ func TestState(t *testing.T) {
|
||||||
} else {
|
} else {
|
||||||
t.Fail()
|
t.Fail()
|
||||||
}
|
}
|
||||||
typedError = Error(typedError, map[string]string{"foo2": "bar2"}).(WrappedError)
|
typedError = New(typedError, map[string]string{"foo2": "bar2"}).(Error)
|
||||||
if res, ok := typedError.Stack[0].Data["foo"]; ok {
|
if res, ok := typedError.Stack[0].Data["foo"]; ok {
|
||||||
if res != "bar" {
|
if res != "bar" {
|
||||||
t.Error("annotate should not modify previous data entries")
|
t.Error("annotate should not modify previous data entries")
|
||||||
|
|
2
main.go
2
main.go
|
@ -133,7 +133,7 @@ func main() {
|
||||||
app.Action = runCmd
|
app.Action = runCmd
|
||||||
err := app.Run(os.Args)
|
err := app.Run(os.Args)
|
||||||
switch typedErr := err.(type) {
|
switch typedErr := err.(type) {
|
||||||
case *iodine.WrappedError:
|
case *iodine.Error:
|
||||||
{
|
{
|
||||||
log.Errorln(typedErr.EmitHumanReadable())
|
log.Errorln(typedErr.EmitHumanReadable())
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,7 +48,7 @@ func Start(path string) (chan<- string, <-chan error, drivers.Driver) {
|
||||||
// TODO donut driver should be passed in as Start param and driven by config
|
// TODO donut driver should be passed in as Start param and driven by config
|
||||||
var err error
|
var err error
|
||||||
s.donut, err = donut.NewDonut(path)
|
s.donut, err = donut.NewDonut(path)
|
||||||
err = iodine.Error(err, map[string]string{"path": path})
|
err = iodine.New(err, map[string]string{"path": path})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println(err)
|
log.Println(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -110,7 +110,7 @@ func getHTTPChannels(configs []Config) (ctrlChans []chan<- string, statusChans [
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
{
|
{
|
||||||
err := iodine.Error(errors.New("Invalid API type"), nil)
|
err := iodine.New(errors.New("Invalid API type"), nil)
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -142,7 +142,7 @@ func getDriverChannels(driverType DriverType) (ctrlChans []chan<- string, status
|
||||||
{
|
{
|
||||||
u, err := user.Current()
|
u, err := user.Current()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorln(iodine.Error(err, nil))
|
log.Errorln(iodine.New(err, nil))
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
root := path.Join(u.HomeDir, "minio-storage", "file")
|
root := path.Join(u.HomeDir, "minio-storage", "file")
|
||||||
|
@ -154,7 +154,7 @@ func getDriverChannels(driverType DriverType) (ctrlChans []chan<- string, status
|
||||||
{
|
{
|
||||||
u, err := user.Current()
|
u, err := user.Current()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorln(iodine.Error(err, nil))
|
log.Errorln(iodine.New(err, nil))
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
root := path.Join(u.HomeDir, "minio-driver", "donut")
|
root := path.Join(u.HomeDir, "minio-driver", "donut")
|
||||||
|
@ -164,7 +164,7 @@ func getDriverChannels(driverType DriverType) (ctrlChans []chan<- string, status
|
||||||
}
|
}
|
||||||
default: // should never happen
|
default: // should never happen
|
||||||
{
|
{
|
||||||
log.Fatal(iodine.Error(errors.New("No driver found"), nil))
|
log.Fatal(iodine.New(errors.New("No driver found"), nil))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
|
|
|
@ -26,7 +26,7 @@ func (b donutBucket) AddNode(nodeID, bucketID string) error {
|
||||||
errParams := map[string]string{"node": nodeID, "bucketID": bucketID}
|
errParams := map[string]string{"node": nodeID, "bucketID": bucketID}
|
||||||
tokens := strings.Split(bucketID, ":")
|
tokens := strings.Split(bucketID, ":")
|
||||||
if len(tokens) != 3 {
|
if len(tokens) != 3 {
|
||||||
return iodine.Error(errors.New("Bucket ID malformeD: "+bucketID), errParams)
|
return iodine.New(errors.New("Bucket ID malformeD: "+bucketID), errParams)
|
||||||
|
|
||||||
}
|
}
|
||||||
// bucketName := tokens[0]
|
// bucketName := tokens[0]
|
||||||
|
@ -34,7 +34,7 @@ func (b donutBucket) AddNode(nodeID, bucketID string) error {
|
||||||
// aggregate := "0"
|
// aggregate := "0"
|
||||||
part, err := strconv.Atoi(tokens[2])
|
part, err := strconv.Atoi(tokens[2])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return iodine.Error(errors.New("Part malformed: "+tokens[2]), errParams)
|
return iodine.New(errors.New("Part malformed: "+tokens[2]), errParams)
|
||||||
}
|
}
|
||||||
b.nodes[part] = nodeID
|
b.nodes[part] = nodeID
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -26,7 +26,7 @@ func NewDonut(root string) (Donut, error) {
|
||||||
for nodeID, node := range nodes {
|
for nodeID, node := range nodes {
|
||||||
bucketIDs, err := node.GetBuckets()
|
bucketIDs, err := node.GetBuckets()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, iodine.Error(err, map[string]string{"root": root})
|
return nil, iodine.New(err, map[string]string{"root": root})
|
||||||
}
|
}
|
||||||
for _, bucketID := range bucketIDs {
|
for _, bucketID := range bucketIDs {
|
||||||
tokens := strings.Split(bucketID, ":")
|
tokens := strings.Split(bucketID, ":")
|
||||||
|
@ -38,7 +38,7 @@ func NewDonut(root string) (Donut, error) {
|
||||||
driver.buckets[tokens[0]] = bucket
|
driver.buckets[tokens[0]] = bucket
|
||||||
}
|
}
|
||||||
if err = driver.buckets[tokens[0]].AddNode(nodeID, bucketID); err != nil {
|
if err = driver.buckets[tokens[0]].AddNode(nodeID, bucketID); err != nil {
|
||||||
return nil, iodine.Error(err, map[string]string{"root": root})
|
return nil, iodine.New(err, map[string]string{"root": root})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -50,7 +50,7 @@ func (d donut) CreateBucket(bucketName string) error {
|
||||||
if _, ok := d.buckets[bucketName]; ok == false {
|
if _, ok := d.buckets[bucketName]; ok == false {
|
||||||
bucketName = strings.TrimSpace(bucketName)
|
bucketName = strings.TrimSpace(bucketName)
|
||||||
if bucketName == "" {
|
if bucketName == "" {
|
||||||
return iodine.Error(errors.New("Cannot create bucket with no name"), map[string]string{"bucket": bucketName})
|
return iodine.New(errors.New("Cannot create bucket with no name"), map[string]string{"bucket": bucketName})
|
||||||
}
|
}
|
||||||
// assign nodes
|
// assign nodes
|
||||||
// TODO assign other nodes
|
// TODO assign other nodes
|
||||||
|
@ -60,7 +60,7 @@ func (d donut) CreateBucket(bucketName string) error {
|
||||||
if node, ok := d.nodes["localhost"]; ok {
|
if node, ok := d.nodes["localhost"]; ok {
|
||||||
err := node.CreateBucket(bucketName + ":0:" + strconv.Itoa(i))
|
err := node.CreateBucket(bucketName + ":0:" + strconv.Itoa(i))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return iodine.Error(err, map[string]string{"node": nodes[i], "bucket": bucketName})
|
return iodine.New(err, map[string]string{"node": nodes[i], "bucket": bucketName})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -70,7 +70,7 @@ func (d donut) CreateBucket(bucketName string) error {
|
||||||
d.buckets[bucketName] = bucket
|
d.buckets[bucketName] = bucket
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return iodine.Error(errors.New("Bucket exists"), map[string]string{"bucket": bucketName})
|
return iodine.New(errors.New("Bucket exists"), map[string]string{"bucket": bucketName})
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListBuckets - list all buckets
|
// ListBuckets - list all buckets
|
||||||
|
@ -89,7 +89,7 @@ func (d donut) GetObjectWriter(bucketName, objectName string) (ObjectWriter, err
|
||||||
writers := make([]Writer, 16)
|
writers := make([]Writer, 16)
|
||||||
nodes, err := bucket.GetNodes()
|
nodes, err := bucket.GetNodes()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, iodine.Error(err, map[string]string{"bucket": bucketName, "object": objectName})
|
return nil, iodine.New(err, map[string]string{"bucket": bucketName, "object": objectName})
|
||||||
}
|
}
|
||||||
for i, nodeID := range nodes {
|
for i, nodeID := range nodes {
|
||||||
if node, ok := d.nodes[nodeID]; ok == true {
|
if node, ok := d.nodes[nodeID]; ok == true {
|
||||||
|
@ -98,17 +98,17 @@ func (d donut) GetObjectWriter(bucketName, objectName string) (ObjectWriter, err
|
||||||
if err != nil {
|
if err != nil {
|
||||||
for _, writerToClose := range writers {
|
for _, writerToClose := range writers {
|
||||||
if writerToClose != nil {
|
if writerToClose != nil {
|
||||||
writerToClose.CloseWithError(iodine.Error(err, nil))
|
writerToClose.CloseWithError(iodine.New(err, nil))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil, iodine.Error(err, map[string]string{"bucketid": bucketID})
|
return nil, iodine.New(err, map[string]string{"bucketid": bucketID})
|
||||||
}
|
}
|
||||||
writers[i] = writer
|
writers[i] = writer
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return newErasureWriter(writers), nil
|
return newErasureWriter(writers), nil
|
||||||
}
|
}
|
||||||
return nil, iodine.Error(errors.New("Bucket not found"), map[string]string{"bucket": bucketName})
|
return nil, iodine.New(errors.New("Bucket not found"), map[string]string{"bucket": bucketName})
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetObjectReader - get a new reader interface for a new object
|
// GetObjectReader - get a new reader interface for a new object
|
||||||
|
@ -119,7 +119,7 @@ func (d donut) GetObjectReader(bucketName, objectName string) (io.ReadCloser, er
|
||||||
readers := make([]io.ReadCloser, 16)
|
readers := make([]io.ReadCloser, 16)
|
||||||
nodes, err := bucket.GetNodes()
|
nodes, err := bucket.GetNodes()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
var metadata map[string]string
|
var metadata map[string]string
|
||||||
for i, nodeID := range nodes {
|
for i, nodeID := range nodes {
|
||||||
|
@ -128,14 +128,14 @@ func (d donut) GetObjectReader(bucketName, objectName string) (io.ReadCloser, er
|
||||||
reader, err := node.GetReader(bucketID, objectName)
|
reader, err := node.GetReader(bucketID, objectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errParams["node"] = nodeID
|
errParams["node"] = nodeID
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
readers[i] = reader
|
readers[i] = reader
|
||||||
if metadata == nil {
|
if metadata == nil {
|
||||||
metadata, err = node.GetDonutMetadata(bucketID, objectName)
|
metadata, err = node.GetDonutMetadata(bucketID, objectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errParams["node"] = nodeID
|
errParams["node"] = nodeID
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -143,7 +143,7 @@ func (d donut) GetObjectReader(bucketName, objectName string) (io.ReadCloser, er
|
||||||
go erasureReader(readers, metadata, w)
|
go erasureReader(readers, metadata, w)
|
||||||
return r, nil
|
return r, nil
|
||||||
}
|
}
|
||||||
return nil, iodine.Error(errors.New("Bucket not found"), errParams)
|
return nil, iodine.New(errors.New("Bucket not found"), errParams)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetObjectMetadata returns metadata for a given object in a bucket
|
// GetObjectMetadata returns metadata for a given object in a bucket
|
||||||
|
@ -152,19 +152,19 @@ func (d donut) GetObjectMetadata(bucketName, object string) (map[string]string,
|
||||||
if bucket, ok := d.buckets[bucketName]; ok {
|
if bucket, ok := d.buckets[bucketName]; ok {
|
||||||
nodes, err := bucket.GetNodes()
|
nodes, err := bucket.GetNodes()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
if node, ok := d.nodes[nodes[0]]; ok {
|
if node, ok := d.nodes[nodes[0]]; ok {
|
||||||
bucketID := bucketName + ":0:0"
|
bucketID := bucketName + ":0:0"
|
||||||
metadata, err := node.GetMetadata(bucketID, object)
|
metadata, err := node.GetMetadata(bucketID, object)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errParams["bucketID"] = bucketID
|
errParams["bucketID"] = bucketID
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
donutMetadata, err := node.GetDonutMetadata(bucketID, object)
|
donutMetadata, err := node.GetDonutMetadata(bucketID, object)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errParams["bucketID"] = bucketID
|
errParams["bucketID"] = bucketID
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
metadata["sys.created"] = donutMetadata["created"]
|
metadata["sys.created"] = donutMetadata["created"]
|
||||||
metadata["sys.md5"] = donutMetadata["md5"]
|
metadata["sys.md5"] = donutMetadata["md5"]
|
||||||
|
@ -172,7 +172,7 @@ func (d donut) GetObjectMetadata(bucketName, object string) (map[string]string,
|
||||||
return metadata, nil
|
return metadata, nil
|
||||||
}
|
}
|
||||||
errParams["node"] = nodes[0]
|
errParams["node"] = nodes[0]
|
||||||
return nil, iodine.Error(errors.New("Cannot connect to node: "+nodes[0]), errParams)
|
return nil, iodine.New(errors.New("Cannot connect to node: "+nodes[0]), errParams)
|
||||||
}
|
}
|
||||||
return nil, errors.New("Bucket not found")
|
return nil, errors.New("Bucket not found")
|
||||||
}
|
}
|
||||||
|
@ -183,14 +183,14 @@ func (d donut) ListObjects(bucketName string) ([]string, error) {
|
||||||
if bucket, ok := d.buckets[bucketName]; ok {
|
if bucket, ok := d.buckets[bucketName]; ok {
|
||||||
nodes, err := bucket.GetNodes()
|
nodes, err := bucket.GetNodes()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
if node, ok := d.nodes[nodes[0]]; ok {
|
if node, ok := d.nodes[nodes[0]]; ok {
|
||||||
bucketID := bucketName + ":0:0"
|
bucketID := bucketName + ":0:0"
|
||||||
objects, err := node.ListObjects(bucketID)
|
objects, err := node.ListObjects(bucketID)
|
||||||
errParams["bucketID"] = bucketID
|
errParams["bucketID"] = bucketID
|
||||||
return objects, iodine.Error(err, errParams)
|
return objects, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil, iodine.Error(errors.New("Bucket not found"), errParams)
|
return nil, iodine.New(errors.New("Bucket not found"), errParams)
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,7 +25,7 @@ func getErasureTechnique(technique string) (erasure.Technique, error) {
|
||||||
case technique == "Vandermonde":
|
case technique == "Vandermonde":
|
||||||
return erasure.Cauchy, nil
|
return erasure.Cauchy, nil
|
||||||
default:
|
default:
|
||||||
return erasure.None, iodine.Error(errors.New("Invalid erasure technique: "+technique), nil)
|
return erasure.None, iodine.New(errors.New("Invalid erasure technique: "+technique), nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -33,45 +33,45 @@ func getErasureTechnique(technique string) (erasure.Technique, error) {
|
||||||
func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
|
func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
|
||||||
totalChunks, err := strconv.Atoi(donutMetadata["chunkCount"])
|
totalChunks, err := strconv.Atoi(donutMetadata["chunkCount"])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.CloseWithError(iodine.Error(err, donutMetadata))
|
writer.CloseWithError(iodine.New(err, donutMetadata))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
totalLeft, err := strconv.ParseInt(donutMetadata["size"], 10, 64)
|
totalLeft, err := strconv.ParseInt(donutMetadata["size"], 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.CloseWithError(iodine.Error(err, donutMetadata))
|
writer.CloseWithError(iodine.New(err, donutMetadata))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
blockSize, err := strconv.Atoi(donutMetadata["blockSize"])
|
blockSize, err := strconv.Atoi(donutMetadata["blockSize"])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.CloseWithError(iodine.Error(err, donutMetadata))
|
writer.CloseWithError(iodine.New(err, donutMetadata))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
parsedk, err := strconv.ParseUint(donutMetadata["erasureK"], 10, 8)
|
parsedk, err := strconv.ParseUint(donutMetadata["erasureK"], 10, 8)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.CloseWithError(iodine.Error(err, donutMetadata))
|
writer.CloseWithError(iodine.New(err, donutMetadata))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
k := uint8(parsedk)
|
k := uint8(parsedk)
|
||||||
parsedm, err := strconv.ParseUint(donutMetadata["erasureM"], 10, 8)
|
parsedm, err := strconv.ParseUint(donutMetadata["erasureM"], 10, 8)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.CloseWithError(iodine.Error(err, donutMetadata))
|
writer.CloseWithError(iodine.New(err, donutMetadata))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
m := uint8(parsedm)
|
m := uint8(parsedm)
|
||||||
expectedMd5sum, err := hex.DecodeString(donutMetadata["md5"])
|
expectedMd5sum, err := hex.DecodeString(donutMetadata["md5"])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.CloseWithError(iodine.Error(err, donutMetadata))
|
writer.CloseWithError(iodine.New(err, donutMetadata))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
technique, err := getErasureTechnique(donutMetadata["erasureTechnique"])
|
technique, err := getErasureTechnique(donutMetadata["erasureTechnique"])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.CloseWithError(iodine.Error(err, donutMetadata))
|
writer.CloseWithError(iodine.New(err, donutMetadata))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
hasher := md5.New()
|
hasher := md5.New()
|
||||||
params, err := erasure.ParseEncoderParams(k, m, technique)
|
params, err := erasure.ParseEncoderParams(k, m, technique)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.CloseWithError(iodine.Error(err, donutMetadata))
|
writer.CloseWithError(iodine.New(err, donutMetadata))
|
||||||
}
|
}
|
||||||
encoder := erasure.NewEncoder(params)
|
encoder := erasure.NewEncoder(params)
|
||||||
for i := 0; i < totalChunks; i++ {
|
for i := 0; i < totalChunks; i++ {
|
||||||
|
@ -83,12 +83,12 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
|
||||||
for k, v := range donutMetadata {
|
for k, v := range donutMetadata {
|
||||||
errParams[k] = v
|
errParams[k] = v
|
||||||
}
|
}
|
||||||
writer.CloseWithError(iodine.Error(err, errParams))
|
writer.CloseWithError(iodine.New(err, errParams))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
actualMd5sum := hasher.Sum(nil)
|
actualMd5sum := hasher.Sum(nil)
|
||||||
if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
|
if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
|
||||||
writer.CloseWithError(iodine.Error(errors.New("decoded md5sum did not match. expected: "+string(expectedMd5sum)+" actual: "+string(actualMd5sum)), donutMetadata))
|
writer.CloseWithError(iodine.New(errors.New("decoded md5sum did not match. expected: "+string(expectedMd5sum)+" actual: "+string(actualMd5sum)), donutMetadata))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
writer.Close()
|
writer.Close()
|
||||||
|
@ -114,7 +114,7 @@ func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *erasur
|
||||||
errParams["part"] = strconv.FormatInt(written, 10)
|
errParams["part"] = strconv.FormatInt(written, 10)
|
||||||
errParams["block.written"] = strconv.FormatInt(written, 10)
|
errParams["block.written"] = strconv.FormatInt(written, 10)
|
||||||
errParams["block.length"] = strconv.Itoa(curChunkSize)
|
errParams["block.length"] = strconv.Itoa(curChunkSize)
|
||||||
return totalLeft, iodine.Error(err, errParams)
|
return totalLeft, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
encodedBytes[i] = bytesBuffer.Bytes()
|
encodedBytes[i] = bytesBuffer.Bytes()
|
||||||
}
|
}
|
||||||
|
@ -122,19 +122,19 @@ func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *erasur
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errParams := map[string]string{}
|
errParams := map[string]string{}
|
||||||
errParams["block.length"] = strconv.Itoa(curChunkSize)
|
errParams["block.length"] = strconv.Itoa(curChunkSize)
|
||||||
return totalLeft, iodine.Error(err, errParams)
|
return totalLeft, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
_, err = hasher.Write(decodedData) // not expecting errors from hash, will also catch further down on .Sum mismatch in parent
|
_, err = hasher.Write(decodedData) // not expecting errors from hash, will also catch further down on .Sum mismatch in parent
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errParams := map[string]string{}
|
errParams := map[string]string{}
|
||||||
errParams["block.length"] = strconv.Itoa(curChunkSize)
|
errParams["block.length"] = strconv.Itoa(curChunkSize)
|
||||||
return totalLeft, iodine.Error(err, errParams)
|
return totalLeft, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
_, err = io.Copy(writer, bytes.NewBuffer(decodedData))
|
_, err = io.Copy(writer, bytes.NewBuffer(decodedData))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errParams := map[string]string{}
|
errParams := map[string]string{}
|
||||||
errParams["block.length"] = strconv.Itoa(curChunkSize)
|
errParams["block.length"] = strconv.Itoa(curChunkSize)
|
||||||
return totalLeft, iodine.Error(err, errParams)
|
return totalLeft, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
totalLeft = totalLeft - int64(blockSize)
|
totalLeft = totalLeft - int64(blockSize)
|
||||||
return totalLeft, nil
|
return totalLeft, nil
|
||||||
|
|
|
@ -19,13 +19,13 @@ type localDirectoryNode struct {
|
||||||
|
|
||||||
func (d localDirectoryNode) CreateBucket(bucket string) error {
|
func (d localDirectoryNode) CreateBucket(bucket string) error {
|
||||||
objectPath := path.Join(d.root, bucket)
|
objectPath := path.Join(d.root, bucket)
|
||||||
return iodine.Error(os.MkdirAll(objectPath, 0700), map[string]string{"bucket": bucket})
|
return iodine.New(os.MkdirAll(objectPath, 0700), map[string]string{"bucket": bucket})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d localDirectoryNode) GetBuckets() ([]string, error) {
|
func (d localDirectoryNode) GetBuckets() ([]string, error) {
|
||||||
files, err := ioutil.ReadDir(d.root)
|
files, err := ioutil.ReadDir(d.root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, iodine.Error(err, nil)
|
return nil, iodine.New(err, nil)
|
||||||
}
|
}
|
||||||
var results []string
|
var results []string
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
|
@ -41,24 +41,24 @@ func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) {
|
||||||
objectPath := path.Join(d.root, bucket, object)
|
objectPath := path.Join(d.root, bucket, object)
|
||||||
err := os.MkdirAll(objectPath, 0700)
|
err := os.MkdirAll(objectPath, 0700)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
writer, err := newDonutObjectWriter(objectPath)
|
writer, err := newDonutObjectWriter(objectPath)
|
||||||
return writer, iodine.Error(err, errParams)
|
return writer, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) {
|
func (d localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) {
|
||||||
reader, err := os.Open(path.Join(d.root, bucket, object, "data"))
|
reader, err := os.Open(path.Join(d.root, bucket, object, "data"))
|
||||||
return reader, iodine.Error(err, map[string]string{"bucket": bucket, "object": object})
|
return reader, iodine.New(err, map[string]string{"bucket": bucket, "object": object})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d localDirectoryNode) GetMetadata(bucket, object string) (map[string]string, error) {
|
func (d localDirectoryNode) GetMetadata(bucket, object string) (map[string]string, error) {
|
||||||
m, err := d.getMetadata(bucket, object, "metadata.json")
|
m, err := d.getMetadata(bucket, object, "metadata.json")
|
||||||
return m, iodine.Error(err, map[string]string{"bucket": bucket, "object": object})
|
return m, iodine.New(err, map[string]string{"bucket": bucket, "object": object})
|
||||||
}
|
}
|
||||||
func (d localDirectoryNode) GetDonutMetadata(bucket, object string) (map[string]string, error) {
|
func (d localDirectoryNode) GetDonutMetadata(bucket, object string) (map[string]string, error) {
|
||||||
m, err := d.getMetadata(bucket, object, "donutMetadata.json")
|
m, err := d.getMetadata(bucket, object, "donutMetadata.json")
|
||||||
return m, iodine.Error(err, map[string]string{"bucket": bucket, "object": object})
|
return m, iodine.New(err, map[string]string{"bucket": bucket, "object": object})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d localDirectoryNode) getMetadata(bucket, object, fileName string) (map[string]string, error) {
|
func (d localDirectoryNode) getMetadata(bucket, object, fileName string) (map[string]string, error) {
|
||||||
|
@ -66,12 +66,12 @@ func (d localDirectoryNode) getMetadata(bucket, object, fileName string) (map[st
|
||||||
file, err := os.Open(path.Join(d.root, bucket, object, fileName))
|
file, err := os.Open(path.Join(d.root, bucket, object, fileName))
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
metadata := make(map[string]string)
|
metadata := make(map[string]string)
|
||||||
decoder := json.NewDecoder(file)
|
decoder := json.NewDecoder(file)
|
||||||
if err := decoder.Decode(&metadata); err != nil {
|
if err := decoder.Decode(&metadata); err != nil {
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
return metadata, nil
|
return metadata, nil
|
||||||
|
|
||||||
|
@ -83,7 +83,7 @@ func (d localDirectoryNode) ListObjects(bucketName string) ([]string, error) {
|
||||||
var objects []string
|
var objects []string
|
||||||
if err := filepath.Walk(prefix, func(path string, info os.FileInfo, err error) error {
|
if err := filepath.Walk(prefix, func(path string, info os.FileInfo, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return iodine.Error(err, errParams)
|
return iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
if !info.IsDir() && strings.HasSuffix(path, "data") {
|
if !info.IsDir() && strings.HasSuffix(path, "data") {
|
||||||
object := strings.TrimPrefix(path, prefix+"/")
|
object := strings.TrimPrefix(path, prefix+"/")
|
||||||
|
@ -92,7 +92,7 @@ func (d localDirectoryNode) ListObjects(bucketName string) ([]string, error) {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return nil, iodine.Error(err, errParams)
|
return nil, iodine.New(err, errParams)
|
||||||
}
|
}
|
||||||
sort.Strings(objects)
|
sort.Strings(objects)
|
||||||
return objects, nil
|
return objects, nil
|
||||||
|
|
|
@ -11,7 +11,7 @@ import (
|
||||||
func newDonutObjectWriter(objectDir string) (Writer, error) {
|
func newDonutObjectWriter(objectDir string) (Writer, error) {
|
||||||
dataFile, err := os.OpenFile(path.Join(objectDir, "data"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
dataFile, err := os.OpenFile(path.Join(objectDir, "data"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, iodine.Error(err, map[string]string{"objectDir": objectDir})
|
return nil, iodine.New(err, map[string]string{"objectDir": objectDir})
|
||||||
}
|
}
|
||||||
return donutObjectWriter{
|
return donutObjectWriter{
|
||||||
root: objectDir,
|
root: objectDir,
|
||||||
|
@ -31,26 +31,26 @@ type donutObjectWriter struct {
|
||||||
|
|
||||||
func (d donutObjectWriter) Write(data []byte) (int, error) {
|
func (d donutObjectWriter) Write(data []byte) (int, error) {
|
||||||
written, err := d.file.Write(data)
|
written, err := d.file.Write(data)
|
||||||
return written, iodine.Error(err, nil)
|
return written, iodine.New(err, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d donutObjectWriter) Close() error {
|
func (d donutObjectWriter) Close() error {
|
||||||
if d.err != nil {
|
if d.err != nil {
|
||||||
return iodine.Error(d.err, nil)
|
return iodine.New(d.err, nil)
|
||||||
}
|
}
|
||||||
metadata, _ := json.Marshal(d.metadata)
|
metadata, _ := json.Marshal(d.metadata)
|
||||||
ioutil.WriteFile(path.Join(d.root, "metadata.json"), metadata, 0600)
|
ioutil.WriteFile(path.Join(d.root, "metadata.json"), metadata, 0600)
|
||||||
donutMetadata, _ := json.Marshal(d.donutMetadata)
|
donutMetadata, _ := json.Marshal(d.donutMetadata)
|
||||||
ioutil.WriteFile(path.Join(d.root, "donutMetadata.json"), donutMetadata, 0600)
|
ioutil.WriteFile(path.Join(d.root, "donutMetadata.json"), donutMetadata, 0600)
|
||||||
|
|
||||||
return iodine.Error(d.file.Close(), nil)
|
return iodine.New(d.file.Close(), nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d donutObjectWriter) CloseWithError(err error) error {
|
func (d donutObjectWriter) CloseWithError(err error) error {
|
||||||
if d.err != nil {
|
if d.err != nil {
|
||||||
d.err = err
|
d.err = err
|
||||||
}
|
}
|
||||||
return iodine.Error(d.Close(), nil)
|
return iodine.New(d.Close(), nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d donutObjectWriter) SetMetadata(metadata map[string]string) error {
|
func (d donutObjectWriter) SetMetadata(metadata map[string]string) error {
|
||||||
|
|
Loading…
Reference in New Issue