MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #332 from harshavardhana/pr_out_fix_build_issues_for_new_donutdriver_

This commit is contained in:
Harshavardhana 2015-03-22 02:53:39 -07:00
parent 15b205c439 95affcb119
commit 537870ec4f
5 changed files with 86 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/storage/donut/bucketdriver.go
View File

@ -5,9 +5,9 @@ type bucketDriver struct {
objects map[string][]byte objects map[string][]byte
} }
func (self bucketDriver) GetNodes() ([]string, error) { func (b bucketDriver) GetNodes() ([]string, error) {
var nodes []string var nodes []string
for _, node := range self.nodes { for _, node := range b.nodes {
nodes = append(nodes, node) nodes = append(nodes, node)
} }
return nodes, nil return nodes, nil

6
pkg/storage/donut/donut.go
View File

@ -20,9 +20,10 @@ type Bucket interface {
GetNodes() ([]string, error) GetNodes() ([]string, error)
} }
// Node interface
type Node interface { type Node interface {
GetBuckets() ([]string, error) GetBuckets() ([]string, error)
GetWriter(bucket, object string) (DonutWriter, error) GetWriter(bucket, object string) (Writer, error)
GetReader(bucket, object string) (io.ReadCloser, error) GetReader(bucket, object string) (io.ReadCloser, error)
GetMetadata(bucket, object string) (map[string]string, error) GetMetadata(bucket, object string) (map[string]string, error)
GetDonutMetadata(bucket, object string) (map[string]string, error) GetDonutMetadata(bucket, object string) (map[string]string, error)
@ -37,7 +38,8 @@ type ObjectWriter interface {
GetMetadata() (map[string]string, error) GetMetadata() (map[string]string, error)
} }
type DonutWriter interface { // Writer interface
type Writer interface {
ObjectWriter ObjectWriter
GetDonutMetadata() (map[string]string, error) GetDonutMetadata() (map[string]string, error)

137
pkg/storage/donut/donutdriver.go
View File

@ -2,11 +2,7 @@ package donut
import ( import (
"bytes" "bytes"
"encoding/gob"
"encoding/json"
"errors" "errors"
"github.com/minio-io/minio/pkg/encoding/erasure"
"github.com/minio-io/minio/pkg/utils/split"
"io" "io"
"io/ioutil" "io/ioutil"
"log" "log"
@ -16,6 +12,12 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"encoding/gob"
"encoding/json"
"github.com/minio-io/minio/pkg/encoding/erasure"
"github.com/minio-io/minio/pkg/utils/split"
) )
type donutDriver struct { type donutDriver struct {
@ -23,6 +25,7 @@ type donutDriver struct {
nodes map[string]Node nodes map[string]Node
} }
// NewDonutDriver - instantiate new donut driver
func NewDonutDriver(root string) Donut { func NewDonutDriver(root string) Donut {
nodes := make(map[string]Node) nodes := make(map[string]Node)
nodes["localhost"] = localDirectoryNode{root: root} nodes["localhost"] = localDirectoryNode{root: root}
@ -55,8 +58,8 @@ func (driver donutDriver) CreateBucket(bucketName string) error {
} }
func (driver donutDriver) ListBuckets() ([]string, error) { func (driver donutDriver) ListBuckets() ([]string, error) {
buckets := make([]string, 0) var buckets []string
for bucket, _ := range driver.buckets { for bucket := range driver.buckets {
buckets = append(buckets, bucket) buckets = append(buckets, bucket)
} }
sort.Strings(buckets) sort.Strings(buckets)
@ -65,13 +68,13 @@ func (driver donutDriver) ListBuckets() ([]string, error) {
func (driver donutDriver) GetObjectWriter(bucketName, objectName string) (ObjectWriter, error) { func (driver donutDriver) GetObjectWriter(bucketName, objectName string) (ObjectWriter, error) {
if bucket, ok := driver.buckets[bucketName]; ok == true { if bucket, ok := driver.buckets[bucketName]; ok == true {
writers := make([]DonutWriter, 16) writers := make([]Writer, 16)
nodes, err := bucket.GetNodes() nodes, err := bucket.GetNodes()
if err != nil { if err != nil {
return nil, err return nil, err
} }
for i, nodeId := range nodes { for i, nodeID := range nodes {
if node, ok := driver.nodes[nodeId]; ok == true { if node, ok := driver.nodes[nodeID]; ok == true {
writer, _ := node.GetWriter(bucketName+":0:"+strconv.Itoa(i), objectName) writer, _ := node.GetWriter(bucketName+":0:"+strconv.Itoa(i), objectName)
writers[i] = writer writers[i] = writer
} }
@ -90,16 +93,16 @@ func (driver donutDriver) GetObject(bucketName, objectName string) (io.ReadClose
return nil, err return nil, err
} }
var metadata map[string]string var metadata map[string]string
for i, nodeId := range nodes { for i, nodeID := range nodes {
if node, ok := driver.nodes[nodeId]; ok == true { if node, ok := driver.nodes[nodeID]; ok == true {
bucketId := bucketName + ":0:" + strconv.Itoa(i) bucketID := bucketName + ":0:" + strconv.Itoa(i)
reader, err := node.GetReader(bucketId, objectName) reader, err := node.GetReader(bucketID, objectName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
readers[i] = reader readers[i] = reader
if metadata == nil { if metadata == nil {
metadata, err = node.GetDonutMetadata(bucketId, objectName) metadata, err = node.GetDonutMetadata(bucketID, objectName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -151,14 +154,14 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
// erasure writer // erasure writer
type erasureWriter struct { type erasureWriter struct {
writers []DonutWriter writers []Writer
metadata map[string]string metadata map[string]string
donutMetadata map[string]string // not exposed donutMetadata map[string]string // not exposed
erasureWriter *io.PipeWriter erasureWriter *io.PipeWriter
isClosed <-chan bool isClosed <-chan bool
} }
func newErasureWriter(writers []DonutWriter) ObjectWriter { func newErasureWriter(writers []Writer) ObjectWriter {
r, w := io.Pipe() r, w := io.Pipe()
isClosed := make(chan bool) isClosed := make(chan bool)
writer := erasureWriter{ writer := erasureWriter{
@ -208,19 +211,19 @@ func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- b
isClosed <- true isClosed <- true
} }
func (self erasureWriter) Write(data []byte) (int, error) { func (d erasureWriter) Write(data []byte) (int, error) {
io.Copy(self.erasureWriter, bytes.NewBuffer(data)) io.Copy(d.erasureWriter, bytes.NewBuffer(data))
return len(data), nil return len(data), nil
} }
func (self erasureWriter) Close() error { func (d erasureWriter) Close() error {
self.erasureWriter.Close() d.erasureWriter.Close()
<-self.isClosed <-d.isClosed
return nil return nil
} }
func (self erasureWriter) CloseWithError(err error) error { func (d erasureWriter) CloseWithError(err error) error {
for _, writer := range self.writers { for _, writer := range d.writers {
if writer != nil { if writer != nil {
writer.CloseWithError(err) writer.CloseWithError(err)
} }
@ -228,19 +231,19 @@ func (self erasureWriter) CloseWithError(err error) error {
return nil return nil
} }
func (self erasureWriter) SetMetadata(metadata map[string]string) error { func (d erasureWriter) SetMetadata(metadata map[string]string) error {
for k, _ := range self.metadata { for k := range d.metadata {
delete(self.metadata, k) delete(d.metadata, k)
} }
for k, v := range metadata { for k, v := range metadata {
self.metadata[k] = v d.metadata[k] = v
} }
return nil return nil
} }
func (self erasureWriter) GetMetadata() (map[string]string, error) { func (d erasureWriter) GetMetadata() (map[string]string, error) {
metadata := make(map[string]string) metadata := make(map[string]string)
for k, v := range self.metadata { for k, v := range d.metadata {
metadata[k] = v metadata[k] = v
} }
return metadata, nil return metadata, nil
@ -250,12 +253,12 @@ type localDirectoryNode struct {
root string root string
} }
func (self localDirectoryNode) GetBuckets() ([]string, error) { func (d localDirectoryNode) GetBuckets() ([]string, error) {
return nil, errors.New("Not Implemented") return nil, errors.New("Not Implemented")
} }
func (self localDirectoryNode) GetWriter(bucket, object string) (DonutWriter, error) { func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) {
objectPath := path.Join(self.root, bucket, object) objectPath := path.Join(d.root, bucket, object)
err := os.MkdirAll(objectPath, 0700) err := os.MkdirAll(objectPath, 0700)
if err != nil { if err != nil {
return nil, err return nil, err
@ -263,19 +266,19 @@ func (self localDirectoryNode) GetWriter(bucket, object string) (DonutWriter, er
return newDonutFileWriter(objectPath) return newDonutFileWriter(objectPath)
} }
func (self localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) { func (d localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) {
return os.Open(path.Join(self.root, bucket, object, "data")) return os.Open(path.Join(d.root, bucket, object, "data"))
} }
func (self localDirectoryNode) GetMetadata(bucket, object string) (map[string]string, error) { func (d localDirectoryNode) GetMetadata(bucket, object string) (map[string]string, error) {
return self.getMetadata(bucket, object, "metadata.json") return d.getMetadata(bucket, object, "metadata.json")
} }
func (self localDirectoryNode) GetDonutMetadata(bucket, object string) (map[string]string, error) { func (d localDirectoryNode) GetDonutMetadata(bucket, object string) (map[string]string, error) {
return self.getMetadata(bucket, object, "donutMetadata.json") return d.getMetadata(bucket, object, "donutMetadata.json")
} }
func (self localDirectoryNode) getMetadata(bucket, object, fileName string) (map[string]string, error) { func (d localDirectoryNode) getMetadata(bucket, object, fileName string) (map[string]string, error) {
file, err := os.Open(path.Join(self.root, bucket, object, fileName)) file, err := os.Open(path.Join(d.root, bucket, object, fileName))
defer file.Close() defer file.Close()
if err != nil { if err != nil {
return nil, err return nil, err
@ -289,7 +292,7 @@ func (self localDirectoryNode) getMetadata(bucket, object, fileName string) (map
} }
func newDonutFileWriter(objectDir string) (DonutWriter, error) { func newDonutFileWriter(objectDir string) (Writer, error) {
dataFile, err := os.OpenFile(path.Join(objectDir, "data"), os.O_WRONLY|os.O_CREATE, 0600) dataFile, err := os.OpenFile(path.Join(objectDir, "data"), os.O_WRONLY|os.O_CREATE, 0600)
if err != nil { if err != nil {
return nil, err return nil, err
@ -310,64 +313,64 @@ type donutFileWriter struct {
err error err error
} }
func (self donutFileWriter) Write(data []byte) (int, error) { func (d donutFileWriter) Write(data []byte) (int, error) {
return self.file.Write(data) return d.file.Write(data)
} }
func (self donutFileWriter) Close() error { func (d donutFileWriter) Close() error {
if self.err != nil { if d.err != nil {
return self.err return d.err
} }
self.file.Close() d.file.Close()
metadata, _ := json.Marshal(self.metadata) metadata, _ := json.Marshal(d.metadata)
ioutil.WriteFile(path.Join(self.root, "metadata.json"), metadata, 0600) ioutil.WriteFile(path.Join(d.root, "metadata.json"), metadata, 0600)
donutMetadata, _ := json.Marshal(self.donutMetadata) donutMetadata, _ := json.Marshal(d.donutMetadata)
ioutil.WriteFile(path.Join(self.root, "donutMetadata.json"), donutMetadata, 0600) ioutil.WriteFile(path.Join(d.root, "donutMetadata.json"), donutMetadata, 0600)
return nil return nil
} }
func (self donutFileWriter) CloseWithError(err error) error { func (d donutFileWriter) CloseWithError(err error) error {
if self.err != nil { if d.err != nil {
self.err = err d.err = err
} }
self.file.Close() d.file.Close()
return nil return nil
} }
func (self donutFileWriter) SetMetadata(metadata map[string]string) error { func (d donutFileWriter) SetMetadata(metadata map[string]string) error {
for k := range self.metadata { for k := range d.metadata {
delete(self.metadata, k) delete(d.metadata, k)
} }
for k, v := range metadata { for k, v := range metadata {
self.metadata[k] = v d.metadata[k] = v
} }
return nil return nil
} }
func (self donutFileWriter) GetMetadata() (map[string]string, error) { func (d donutFileWriter) GetMetadata() (map[string]string, error) {
metadata := make(map[string]string) metadata := make(map[string]string)
for k, v := range self.metadata { for k, v := range d.metadata {
metadata[k] = v metadata[k] = v
} }
return metadata, nil return metadata, nil
} }
func (self donutFileWriter) SetDonutMetadata(metadata map[string]string) error { func (d donutFileWriter) SetDonutMetadata(metadata map[string]string) error {
for k := range self.donutMetadata { for k := range d.donutMetadata {
delete(self.donutMetadata, k) delete(d.donutMetadata, k)
} }
for k, v := range metadata { for k, v := range metadata {
self.donutMetadata[k] = v d.donutMetadata[k] = v
} }
return nil return nil
} }
func (self donutFileWriter) GetDonutMetadata() (map[string]string, error) { func (d donutFileWriter) GetDonutMetadata() (map[string]string, error) {
donutMetadata := make(map[string]string) donutMetadata := make(map[string]string)
for k, v := range self.donutMetadata { for k, v := range d.donutMetadata {
donutMetadata[k] = v donutMetadata[k] = v
} }
return donutMetadata, nil return donutMetadata, nil

2
pkg/storage/donut/donutdriver_test.go
View File

@ -25,7 +25,7 @@ func (s *MySuite) TestEmptyBucket(c *C) {
// check buckets are empty // check buckets are empty
buckets, err := donut.ListBuckets() buckets, err := donut.ListBuckets()
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(buckets, DeepEquals, make([]string, 0)) c.Assert(buckets, IsNil)
} }
func (s *MySuite) TestBucketWithoutNameFails(c *C) { func (s *MySuite) TestBucketWithoutNameFails(c *C) {

18
pkg/storage/donut/objectwriter.go
View File

@ -8,31 +8,31 @@ type objectWriter struct {
metadata map[string]string metadata map[string]string
} }
func (self objectWriter) Write(data []byte) (length int, err error) { func (obj objectWriter) Write(data []byte) (length int, err error) {
return 11, nil return 11, nil
} }
func (self objectWriter) Close() error { func (obj objectWriter) Close() error {
return nil return nil
} }
func (self objectWriter) CloseWithError(err error) error { func (obj objectWriter) CloseWithError(err error) error {
return errors.New("Not Implemented") return errors.New("Not Implemented")
} }
func (self objectWriter) SetMetadata(metadata map[string]string) error { func (obj objectWriter) SetMetadata(metadata map[string]string) error {
for k := range self.metadata { for k := range obj.metadata {
delete(self.metadata, k) delete(obj.metadata, k)
} }
for k, v := range metadata { for k, v := range metadata {
self.metadata[k] = v obj.metadata[k] = v
} }
return nil return nil
} }
func (self objectWriter) GetMetadata() (map[string]string, error) { func (obj objectWriter) GetMetadata() (map[string]string, error) {
ret := make(map[string]string) ret := make(map[string]string)
for k, v := range self.metadata { for k, v := range obj.metadata {
ret[k] = v ret[k] = v
} }
return ret, nil return ret, nil