Merge remote-tracking branch 'origin/master' into HEAD

Conflicts:
	pkg/drivers/donut/donut.go
	pkg/storage/donut/bucket.go
	pkg/storage/donut/donut.go
	pkg/storage/donut/donut_test.go
	pkg/storage/donut/donutdriver.go

pkg/drivers/donut/donut.go

@@ -0,0 +1,264 @@
+/*
+ * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package donut
+
+import (
+	"errors"
+	"io"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio-io/minio/pkg/drivers"
+	"github.com/minio-io/minio/pkg/storage/donut"
+)
+
+// donutDriver - creates a new single disk drivers driver using donut
+type donutDriver struct {
+	donut donut.Donut
+}
+
+const (
+	blockSize = 10 * 1024 * 1024
+)
+
+// Start a single disk subsystem
+func Start(path string) (chan<- string, <-chan error, drivers.Driver) {
+	ctrlChannel := make(chan string)
+	errorChannel := make(chan error)
+	s := new(donutDriver)
+
+	// TODO donut driver should be passed in as Start param and driven by config
+	var err error
+	s.donut, err = donut.NewDonut(path)
+	if err != nil {
+		errorChannel <- err
+	}
+
+	go start(ctrlChannel, errorChannel, s)
+	return ctrlChannel, errorChannel, s
+}
+
+func start(ctrlChannel <-chan string, errorChannel chan<- error, s *donutDriver) {
+	close(errorChannel)
+}
+
+// ListBuckets returns a list of buckets
+func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error) {
+	buckets, err := d.donut.ListBuckets()
+	if err != nil {
+		return nil, err
+	}
+	for _, bucket := range buckets {
+		if err != nil {
+			return nil, err
+		}
+		result := drivers.BucketMetadata{
+			Name: bucket,
+			// TODO Add real created date
+			Created: time.Now(),
+		}
+		results = append(results, result)
+	}
+	return results, nil
+}
+
+// CreateBucket creates a new bucket
+func (d donutDriver) CreateBucket(bucket string) error {
+	return d.donut.CreateBucket(bucket)
+}
+
+// GetBucketMetadata retrieves an bucket's metadata
+func (d donutDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
+	return drivers.BucketMetadata{}, errors.New("Not Implemented")
+}
+
+// CreateBucketPolicy sets a bucket's access policy
+func (d donutDriver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error {
+	return errors.New("Not Implemented")
+}
+
+// GetBucketPolicy returns a bucket's access policy
+func (d donutDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
+	return drivers.BucketPolicy{}, errors.New("Not Implemented")
+}
+
+// GetObject retrieves an object and writes it to a writer
+func (d donutDriver) GetObject(target io.Writer, bucket, key string) (int64, error) {
+	reader, err := d.donut.GetObjectReader(bucket, key)
+	if err != nil {
+		return 0, drivers.ObjectNotFound{
+			Bucket: bucket,
+			Object: key,
+		}
+	}
+	return io.Copy(target, reader)
+}
+
+// GetPartialObject retrieves an object range and writes it to a writer
+func (d donutDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
+	return 0, errors.New("Not Implemented")
+}
+
+// GetObjectMetadata retrieves an object's metadata
+func (d donutDriver) GetObjectMetadata(bucket, key string, prefix string) (drivers.ObjectMetadata, error) {
+	metadata, err := d.donut.GetObjectMetadata(bucket, key)
+	created, err := time.Parse(time.RFC3339Nano, metadata["sys.created"])
+	if err != nil {
+		return drivers.ObjectMetadata{}, err
+	}
+	size, err := strconv.ParseInt(metadata["sys.size"], 10, 64)
+	if err != nil {
+		return drivers.ObjectMetadata{}, err
+	}
+	objectMetadata := drivers.ObjectMetadata{
+		Bucket: bucket,
+		Key:    key,
+
+		ContentType: metadata["contentType"],
+		Created:     created,
+		Md5:         metadata["sys.md5"],
+		Size:        size,
+	}
+	return objectMetadata, nil
+}
+
+// ListObjects - returns list of objects
+func (d donutDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
+	// TODO Fix IsPrefixSet && IsDelimiterSet and use them
+	objects, err := d.donut.ListObjects(bucket)
+	if err != nil {
+		return nil, drivers.BucketResourcesMetadata{}, err
+	}
+	sort.Strings(objects)
+	if resources.Prefix != "" {
+		objects = filterPrefix(objects, resources.Prefix)
+		objects = removePrefix(objects, resources.Prefix)
+	}
+	if resources.Maxkeys <= 0 || resources.Maxkeys > 1000 {
+		resources.Maxkeys = 1000
+	}
+
+	var actualObjects []string
+	var commonPrefixes []string
+	if strings.TrimSpace(resources.Delimiter) != "" {
+		actualObjects = filterDelimited(objects, resources.Delimiter)
+		commonPrefixes = filterNotDelimited(objects, resources.Delimiter)
+		commonPrefixes = extractDir(commonPrefixes, resources.Delimiter)
+		commonPrefixes = uniqueObjects(commonPrefixes)
+		resources.CommonPrefixes = commonPrefixes
+	} else {
+		actualObjects = objects
+	}
+
+	var results []drivers.ObjectMetadata
+	for _, object := range actualObjects {
+		if len(results) >= resources.Maxkeys {
+			resources.IsTruncated = true
+			break
+		}
+		metadata, err := d.GetObjectMetadata(bucket, resources.Prefix+object, "")
+		if err != nil {
+			return nil, drivers.BucketResourcesMetadata{}, err
+		}
+		results = append(results, metadata)
+	}
+	return results, resources, nil
+}
+
+func filterPrefix(objects []string, prefix string) []string {
+	var results []string
+	for _, object := range objects {
+		if strings.HasPrefix(object, prefix) {
+			results = append(results, object)
+		}
+	}
+	return results
+}
+
+func removePrefix(objects []string, prefix string) []string {
+	var results []string
+	for _, object := range objects {
+		results = append(results, strings.TrimPrefix(object, prefix))
+	}
+	return results
+}
+
+func filterDelimited(objects []string, delim string) []string {
+	var results []string
+	for _, object := range objects {
+		if !strings.Contains(object, delim) {
+			results = append(results, object)
+		}
+	}
+	return results
+}
+func filterNotDelimited(objects []string, delim string) []string {
+	var results []string
+	for _, object := range objects {
+		if strings.Contains(object, delim) {
+			results = append(results, object)
+		}
+	}
+	return results
+}
+
+func extractDir(objects []string, delim string) []string {
+	var results []string
+	for _, object := range objects {
+		parts := strings.Split(object, delim)
+		results = append(results, parts[0]+"/")
+	}
+	return results
+}
+
+func uniqueObjects(objects []string) []string {
+	objectMap := make(map[string]string)
+	for _, v := range objects {
+		objectMap[v] = v
+	}
+	var results []string
+	for k := range objectMap {
+		results = append(results, k)
+	}
+	sort.Strings(results)
+	return results
+}
+
+// CreateObject creates a new object
+func (d donutDriver) CreateObject(bucketKey, objectKey, contentType, expectedMd5sum string, reader io.Reader) error {
+	writer, err := d.donut.GetObjectWriter(bucketKey, objectKey)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(writer, reader); err != nil {
+		return err
+	}
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	contentType = strings.TrimSpace(contentType)
+	metadata := make(map[string]string)
+	metadata["bucket"] = bucketKey
+	metadata["object"] = objectKey
+	metadata["contentType"] = contentType
+	if err = writer.SetMetadata(metadata); err != nil {
+		return err
+	}
+	return writer.Close()
+}

pkg/drivers/donut/donut_test.go

@@ -0,0 +1,54 @@
+/*
+ * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package donut
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/minio-io/minio/pkg/drivers"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestAPISuite(c *C) {
+	//	c.Skip("Not Implemented")
+	var storageList []string
+	create := func() drivers.Driver {
+		path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
+		c.Check(err, IsNil)
+		storageList = append(storageList, path)
+		_, _, store := Start(path) // TODO Make InMemory driver
+		return store
+	}
+	drivers.APITestSuite(c, create)
+	removeRoots(c, storageList)
+}
+
+func removeRoots(c *C, roots []string) {
+	for _, root := range roots {
+		err := os.RemoveAll(root)
+		c.Check(err, IsNil)
+	}
+}