MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #70 from fkautz/pr_out_adding_fs_storage

This commit is contained in:
Frederick F. Kautz IV 2015-01-27 11:11:25 -08:00
parent 3461e7e300 163a6c35db
commit 027d1b0f70
3 changed files with 260 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
pkg/storage/fs/fs.go Normal file
View File

@ -0,0 +1,201 @@
package fs
import (
"errors"
"io"
"io/ioutil"
"os"
"path"
"strings"
"sync"
mstorage "github.com/minio-io/minio/pkg/storage"
)
type storage struct {
root string
writeLock sync.Mutex
}
type MkdirFailedError struct{}
func (self MkdirFailedError) Error() string {
return "Mkdir Failed"
}
func Start(root string) (chan<- string, <-chan error, *storage) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
go start(ctrlChannel, errorChannel)
return ctrlChannel, errorChannel, &storage{root: root}
}
func start(ctrlChannel <-chan string, errorChannel chan<- error) {
close(errorChannel)
}
// Bucket Operaotions
func (storage *storage) ListBuckets(prefix string) ([]mstorage.BucketMetadata, error) {
return []mstorage.BucketMetadata{}, errors.New("Not Implemented")
}
func (storage *storage) StoreBucket(bucket string) error {
storage.writeLock.Lock()
defer storage.writeLock.Unlock()
// verify bucket path legal
if mstorage.IsValidBucket(bucket) == false {
return mstorage.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(storage.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err == nil {
return mstorage.BucketExists{
Bucket: bucket,
}
}
// make bucket
err := os.Mkdir(bucketDir, 0700)
if err != nil {
return mstorage.EmbedError(bucket, "", err)
}
return nil
}
// Object Operations
func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) {
// validate bucket
if mstorage.IsValidBucket(bucket) == false {
return 0, mstorage.BucketNameInvalid{Bucket: bucket}
}
// validate object
if mstorage.IsValidObject(object) == false {
return 0, mstorage.ObjectNameInvalid{Bucket: bucket, Object: object}
}
objectPath := path.Join(storage.root, bucket, object)
file, err := os.Open(objectPath)
if err != nil {
return 0, mstorage.EmbedError(bucket, object, err)
}
count, err := io.Copy(w, file)
if err != nil {
return count, mstorage.EmbedError(bucket, object, err)
}
return count, nil
}
func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorage.ObjectMetadata, error) {
if mstorage.IsValidBucket(bucket) == false {
return mstorage.ObjectMetadata{}, mstorage.BucketNameInvalid{Bucket: bucket}
}
if mstorage.IsValidObject(bucket) == false {
return mstorage.ObjectMetadata{}, mstorage.ObjectNameInvalid{Bucket: bucket, Object: bucket}
}
objectPath := path.Join(storage.root, bucket, object)
stat, err := os.Stat(objectPath)
if os.IsNotExist(err) {
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
}
metadata := mstorage.ObjectMetadata{
Bucket: bucket,
Key: object,
Created: stat.ModTime(),
Size: stat.Size(),
ETag: bucket + "#" + object,
}
return metadata, nil
}
func (storage *storage) ListObjects(bucket, prefix string, count int) ([]mstorage.ObjectMetadata, bool, error) {
if mstorage.IsValidBucket(bucket) == false {
return []mstorage.ObjectMetadata{}, false, mstorage.BucketNameInvalid{Bucket: bucket}
}
if mstorage.IsValidObject(prefix) == false {
return []mstorage.ObjectMetadata{}, false, mstorage.ObjectNameInvalid{Bucket: bucket, Object: prefix}
}
rootPrefix := path.Join(storage.root, bucket)
files, err := ioutil.ReadDir(rootPrefix)
if err != nil {
return []mstorage.ObjectMetadata{}, false, mstorage.EmbedError("bucket", "", err)
}
var metadataList []mstorage.ObjectMetadata
for _, file := range files {
if len(metadataList) >= count {
return metadataList, true, nil
}
if strings.HasPrefix(file.Name(), prefix) {
metadata := mstorage.ObjectMetadata{
Bucket: bucket,
Key: file.Name(),
Created: file.ModTime(),
Size: file.Size(),
ETag: bucket + "#" + file.Name(),
}
metadataList = append(metadataList, metadata)
}
}
return metadataList, false, nil
}
func (storage *storage) StoreObject(bucket string, key string, data io.Reader) error {
// TODO Commits should stage then move instead of writing directly
storage.writeLock.Lock()
defer storage.writeLock.Unlock()
// check bucket name valid
if mstorage.IsValidBucket(bucket) == false {
return mstorage.BucketNameInvalid{Bucket: bucket}
}
// check bucket exists
if _, err := os.Stat(path.Join(storage.root, bucket)); os.IsNotExist(err) {
return mstorage.BucketNotFound{Bucket: bucket}
}
// verify object path legal
if mstorage.IsValidObject(key) == false {
return mstorage.ObjectNameInvalid{Bucket: bucket, Object: key}
}
// get object path
objectPath := path.Join(storage.root, bucket, key)
// check if object exists
if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
return mstorage.ObjectExists{
Bucket: bucket,
Key: key,
}
}
// write object
file, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
defer file.Close()
if err != nil {
return mstorage.EmbedError(bucket, key, err)
}
_, err = io.Copy(file, data)
if err != nil {
return mstorage.EmbedError(bucket, key, err)
}
return nil
}

42
pkg/storage/fs/fs_test.go Normal file
View File

@ -0,0 +1,42 @@
package fs
import (
"io/ioutil"
"log"
"os"
"testing"
mstorage "github.com/minio-io/minio/pkg/storage"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
var storageList []string
create := func() mstorage.Storage {
path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
log.Println(path)
c.Check(err, IsNil)
storageList = append(storageList, path)
_, _, store := Start(path)
return store
}
log.Println("FOO")
mstorage.APITestSuite(c, create)
log.Println("BAR")
removeRoots(c, storageList)
}
func removeRoots(c *C, roots []string) {
log.Println("REMOVING ROOTS: ", roots)
for _, root := range roots {
err := os.RemoveAll(root)
c.Check(err, IsNil)
}
}

29
pkg/storage/storage_api_suite.go
View File

@ -11,10 +11,10 @@ import (
func APITestSuite(c *C, create func() Storage) {
testCreateBucket(c, create)
testMultipleObjectCreation(c, create)
//testPaging(c, create)
//testObjectOverwriteFails(c, create)
//testNonExistantBucketOperations(c, create)
//testBucketRecreateFails(c, create)
testPaging(c, create)
testObjectOverwriteFails(c, create)
testNonExistantBucketOperations(c, create)
testBucketRecreateFails(c, create)
}
func testCreateBucket(c *C, create func() Storage) {
@ -58,24 +58,24 @@ func testMultipleObjectCreation(c *C, create func() Storage) {
func testPaging(c *C, create func() Storage) {
storage := create()
storage.StoreBucket("bucket")
storage.ListObjects("bucket", "", 1000)
objects, isTruncated, err := storage.ListObjects("bucket", "", 1000)
storage.ListObjects("bucket", "", 5)
objects, isTruncated, err := storage.ListObjects("bucket", "", 5)
c.Check(len(objects), Equals, 0)
c.Check(isTruncated, Equals, false)
c.Check(err, IsNil)
for i := 1; i <= 1000; i++ {
for i := 1; i <= 5; i++ {
key := "obj" + strconv.Itoa(i)
storage.StoreObject("bucket", key, bytes.NewBufferString(key))
objects, isTruncated, err = storage.ListObjects("bucket", "", 1000)
objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
c.Check(len(objects), Equals, i)
c.Check(isTruncated, Equals, false)
c.Check(err, IsNil)
}
for i := 1001; i <= 2000; i++ {
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
storage.StoreObject("bucket", key, bytes.NewBufferString(key))
objects, isTruncated, err = storage.ListObjects("bucket", "", 1000)
c.Check(len(objects), Equals, 1000)
objects, isTruncated, err = storage.ListObjects("bucket", "", 5)
c.Check(len(objects), Equals, 5)
c.Check(isTruncated, Equals, true)
c.Check(err, IsNil)
}
@ -86,8 +86,13 @@ func testObjectOverwriteFails(c *C, create func() Storage) {
storage.StoreBucket("bucket")
err := storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
c.Check(err, IsNil)
err = storage.StoreObject("bucket", "object", bytes.NewBufferString("one"))
err = storage.StoreObject("bucket", "object", bytes.NewBufferString("three"))
c.Check(err, Not(IsNil))
var bytesBuffer bytes.Buffer
length, err := storage.CopyObjectToWriter(&bytesBuffer, "bucket", "object")
c.Check(length, Equals, int64(len("one")))
c.Check(err, IsNil)
c.Check(string(bytesBuffer.Bytes()), Equals, "one")
}
func testNonExistantBucketOperations(c *C, create func() Storage) {