Merge pull request #357 from fkautz/pr_out_populate_buckets_at_start

pkg/drivers/donut/donut.go

@@ -44,7 +44,11 @@ func Start(path string) (chan<- string, <-chan error, drivers.Driver) {
 	s := new(donutDriver)
 
 	// TODO donut driver should be passed in as Start param and driven by config
-	s.donut = donut.NewDonut(path)
+	var err error
+	s.donut, err = donut.NewDonut(path)
+	if err != nil {
+		errorChannel <- err
+	}
 
 	go start(ctrlChannel, errorChannel, s)
 	return ctrlChannel, errorChannel, s

pkg/storage/donut/bucket.go

@@ -1,5 +1,11 @@
 package donut
 
+import (
+	"errors"
+	"strconv"
+	"strings"
+)
+
 type donutBucket struct {
 	nodes   []string
 	objects map[string][]byte
@@ -13,3 +19,19 @@ func (b donutBucket) GetNodes() ([]string, error) {
 	}
 	return nodes, nil
 }
+
+func (b donutBucket) AddNode(nodeID, bucketID string) error {
+	tokens := strings.Split(bucketID, ":")
+	if len(tokens) != 3 {
+		return errors.New("Bucket ID malformed: " + bucketID)
+	}
+	// bucketName := tokens[0]
+	// aggregate := tokens[1]
+	// aggregate := "0"
+	part, err := strconv.Atoi(tokens[2])
+	if err != nil {
+		return errors.New("Part malformed: " + tokens[2])
+	}
+	b.nodes[part] = nodeID
+	return nil
+}

pkg/storage/donut/donut.go

@@ -13,15 +13,36 @@ type donut struct {
 	nodes   map[string]Node
 }
 
-// NewDonut - instantiate new donut
+// NewDonut - instantiate new donut driver
-func NewDonut(root string) Donut {
+func NewDonut(root string) (Donut, error) {
 	nodes := make(map[string]Node)
-	nodes["localhost"] = localDirectoryNode{root: root}
+	nodes["localhost"] = &localDirectoryNode{root: root}
-	d := donut{
+	driver := &donut{
 		buckets: make(map[string]Bucket),
 		nodes:   nodes,
 	}
-	return d
+	for nodeID, node := range nodes {
+		bucketIDs, err := node.GetBuckets()
+		if err != nil {
+			return nil, err
+		}
+		for _, bucketID := range bucketIDs {
+			tokens := strings.Split(bucketID, ":")
+			if _, ok := driver.buckets[tokens[0]]; ok {
+				// found bucket, skip creating
+			} else {
+				bucket := donutBucket{
+					nodes: make([]string, 16),
+				}
+				// TODO catch errors
+				driver.buckets[tokens[0]] = bucket
+			}
+			if err = driver.buckets[tokens[0]].AddNode(nodeID, bucketID); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return driver, nil
 }
 
 // CreateBucket - create a new bucket

pkg/storage/donut/donut_test.go

@@ -21,7 +21,8 @@ func (s *MySuite) TestEmptyBucket(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 
 	// check buckets are empty
 	buckets, err := donut.ListBuckets()
@@ -33,7 +34,8 @@ func (s *MySuite) TestBucketWithoutNameFails(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 	// fail to create new bucket without a name
 	err = donut.CreateBucket("")
 	c.Assert(err, Not(IsNil))
@@ -46,7 +48,8 @@ func (s *MySuite) TestCreateBucketAndList(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 	// create bucket
 	err = donut.CreateBucket("foo")
 	c.Assert(err, IsNil)
@@ -61,7 +64,8 @@ func (s *MySuite) TestCreateBucketWithSameNameFails(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 	err = donut.CreateBucket("foo")
 	c.Assert(err, IsNil)
 
@@ -73,7 +77,8 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 	// add a second bucket
 	err = donut.CreateBucket("foo")
 	c.Assert(err, IsNil)
@@ -97,7 +102,8 @@ func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 
 	writer, err := donut.GetObjectWriter("foo", "obj")
 	c.Assert(err, Not(IsNil))
@@ -108,7 +114,8 @@ func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 
 	writer, err := donut.GetObjectWriter("foo", "")
 	c.Assert(err, Not(IsNil))
@@ -123,7 +130,8 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 
 	err = donut.CreateBucket("foo")
 	c.Assert(err, IsNil)
@@ -175,7 +183,8 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 
 	c.Assert(donut.CreateBucket("foo"), IsNil)
 	writer, err := donut.GetObjectWriter("foo", "obj1")
@@ -215,7 +224,8 @@ func (s *MySuite) TestSysPrefixShouldFail(c *C) {
 	root, err := ioutil.TempDir(os.TempDir(), "donut-")
 	c.Assert(err, IsNil)
 	defer os.RemoveAll(root)
-	donut := NewDonut(root)
+	donut, err := NewDonut(root)
+	c.Assert(err, IsNil)
 
 	c.Assert(donut.CreateBucket("foo"), IsNil)
 	writer, err := donut.GetObjectWriter("foo", "obj1")

pkg/storage/donut/interfaces.go

@@ -19,6 +19,7 @@ type Donut interface {
 // Bucket interface
 type Bucket interface {
 	GetNodes() ([]string, error)
+	AddNode(nodeID, bucketID string) error
 }
 
 // Node interface

pkg/storage/donut/node_local.go

@@ -1,7 +1,6 @@
 package donut
 
 import (
-	"errors"
 	"io"
 	"os"
 	"path"
@@ -9,6 +8,7 @@ import (
 	"strings"
 
 	"encoding/json"
+	"io/ioutil"
 	"path/filepath"
 )
 
@@ -22,7 +22,17 @@ func (d localDirectoryNode) CreateBucket(bucket string) error {
 }
 
 func (d localDirectoryNode) GetBuckets() ([]string, error) {
-	return nil, errors.New("Not Implemented")
+	files, err := ioutil.ReadDir(d.root)
+	if err != nil {
+		return nil, err
+	}
+	var results []string
+	for _, file := range files {
+		if file.IsDir() {
+			results = append(results, file.Name())
+		}
+	}
+	return results, nil
 }
 
 func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) {