MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #115 from harshavardhana/pr_out_add_list_object_support_for_erasure_and_append_storage_drivers

Browse Source
This commit is contained in:
Harshavardhana 2014-12-12 02:56:13 -08:00
commit b570b486de
8 changed files with 65 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
cmd/erasure-demo/erasure.go
View File

@ -1,7 +1,8 @@
package main
import (
"errors"
"bytes"
"encoding/json"
"io"
"os"
"path"
@ -11,8 +12,23 @@ import (
)
func erasureGetList(config inputConfig) (io.Reader, error) {
// do nothing
return nil, errors.New("Not Implemented")
var objectStorage storage.ObjectStorage
rootDir := path.Join(config.rootDir, config.storageDriver)
objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
if err != nil {
return nil, err
}
objectDescList, err := objectStorage.List()
if err != nil {
return nil, err
}
var objectDescListBytes []byte
if objectDescListBytes, err = json.Marshal(objectDescList); err != nil {
return nil, err
}
objectDescListBuffer := bytes.NewBuffer(objectDescListBytes)
return objectDescListBuffer, nil
}
func erasureGet(config inputConfig, objectPath string) (io.Reader, error) {

2
cmd/erasure-demo/fs.go
View File

@ -15,7 +15,7 @@ func fsGetList(config inputConfig) (io.Reader, error) {
var objectStorage storage.ObjectStorage
rootDir := path.Join(config.rootDir, config.storageDriver)
objectStorage, _ = fsstorage.NewStorage(rootDir)
objectList, err := objectStorage.List("/")
objectList, err := objectStorage.List()
if err != nil {
return nil, err
}

15
pkgs/storage/appendstorage/append_storage.go
View File

@ -117,6 +117,17 @@ func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
return nil
}
func (aStorage *appendStorage) List(listPath string) ([]storage.ObjectDescription, error) {
return nil, errors.New("Not Implemented")
func (aStorage *appendStorage) List() ([]storage.ObjectDescription, error) {
var objectDescList []storage.ObjectDescription
for objectName, _ := range aStorage.objects {
var objectDescription storage.ObjectDescription
objectDescription.Name = objectName
objectDescription.Md5sum = ""
objectDescription.Protectionlevel = ""
objectDescList = append(objectDescList, objectDescription)
}
if len(objectDescList) == 0 {
return nil, errors.New("No objects found")
}
return objectDescList, nil
}

21
pkgs/storage/encodedstorage/encoded_storage.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"crypto/md5"
"encoding/gob"
"encoding/hex"
"errors"
"io"
"io/ioutil"
@ -93,8 +94,20 @@ func (eStorage *encodedStorage) Get(objectPath string) (io.Reader, error) {
return reader, nil
}
func (eStorage *encodedStorage) List(listPath string) ([]storage.ObjectDescription, error) {
return nil, errors.New("Not Implemented")
func (eStorage *encodedStorage) List() ([]storage.ObjectDescription, error) {
var objectDescList []storage.ObjectDescription
for objectName, objectEntry := range eStorage.objects {
var objectDescription storage.ObjectDescription
protectionLevel := strconv.Itoa(objectEntry.Encoderparams.K) + "," + strconv.Itoa(objectEntry.Encoderparams.M)
objectDescription.Name = objectName
objectDescription.Md5sum = hex.EncodeToString(objectEntry.Md5sum)
objectDescription.Protectionlevel = protectionLevel
objectDescList = append(objectDescList, objectDescription)
}
if len(objectDescList) == 0 {
return nil, errors.New("No objects found")
}
return objectDescList, nil
}
func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
@ -124,8 +137,6 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
// encode
for chunk := range chunks {
if chunk.Err == nil {
// md5sum on chunk
hash.Write(chunk.Data)
// encode each
blocks, length := encoder.Encode(chunk.Data)
// store each
@ -135,6 +146,8 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
return err
}
}
// md5sum only after chunk is committed to disk
hash.Write(chunk.Data)
blockEntry := StorageBlockEntry{
Index: i,
Length: length,

28
pkgs/storage/encodedstorage/encoded_storage_test.go
View File

@ -38,9 +38,9 @@ func (s *EncodedStorageSuite) TestFileStoragePutAtRootPath(c *C) {
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1")
// objectList, err := objectStorage.List("/")
// c.Assert(err, IsNil)
// c.Assert(objectList[0].Path, Equals, "path1")
objectList, err := objectStorage.List()
c.Assert(err, IsNil)
c.Assert(objectList[0].Name, Equals, "path1")
}
func (s *EncodedStorageSuite) TestFileStoragePutDirPath(c *C) {
@ -70,28 +70,6 @@ func (s *EncodedStorageSuite) TestFileStoragePutDirPath(c *C) {
objectBuffer3 := bytes.NewBuffer([]byte("object3"))
err = objectStorage.Put("object3", objectBuffer3)
c.Assert(err, IsNil)
// TODO support list
// objectList, err := objectStorage.List("/")
// c.Assert(err, IsNil)
// c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "object3", IsDir: false, Hash: ""})
// c.Assert(objectList[1], Equals, storage.ObjectDescription{Path: "path1", IsDir: true, Hash: ""})
// c.Assert(objectList[2], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
// c.Assert(len(objectList), Equals, 3)
//
// objectList, err = objectStorage.List("/path1")
// c.Assert(err, IsNil)
// c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
// c.Assert(len(objectList), Equals, 1)
//
// objectList, err = objectStorage.List("/path1/path2")
// c.Assert(err, IsNil)
// c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path3", IsDir: false, Hash: ""})
// c.Assert(len(objectList), Equals, 1)
//
// objectList, err = objectStorage.List("/path1/path2/path3")
// c.Assert(err, Not(IsNil))
// c.Assert(objectList, IsNil)
}
func (s *EncodedStorageSuite) TestObjectWithChunking(c *C) {

10
pkgs/storage/fsstorage/fs_storage.go
View File

@ -21,8 +21,8 @@ func NewStorage(rootDir string) (storage.ObjectStorage, error) {
return &newStorage, nil
}
func (fsStorage *fileSystemStorage) List(listPath string) ([]storage.ObjectDescription, error) {
fileInfos, err := ioutil.ReadDir(path.Join(fsStorage.RootDir, listPath))
func (fsStorage *fileSystemStorage) List() ([]storage.ObjectDescription, error) {
fileInfos, err := ioutil.ReadDir(fsStorage.RootDir)
if err != nil {
return nil, err
}
@ -31,9 +31,9 @@ func (fsStorage *fileSystemStorage) List(listPath string) ([]storage.ObjectDescr
for _, fi := range fileInfos {
description := storage.ObjectDescription{
Path: fi.Name(),
IsDir: fi.IsDir(),
Hash: "", // TODO
Name: fi.Name(),
Md5sum: "",
Protectionlevel: "",
}
descriptions = append(descriptions, description)
}

53
pkgs/storage/fsstorage/fs_storage_test.go
View File

@ -37,56 +37,7 @@ func (s *fileSystemStorageSuite) TestfileStoragePutAtRootPath(c *C) {
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1")
objectList, err := objectStorage.List("/")
objectList, err := objectStorage.List()
c.Assert(err, IsNil)
c.Assert(objectList[0].Path, Equals, "path1")
}
func (s *fileSystemStorageSuite) TestfileStoragePutDirPath(c *C) {
rootDir, err := makeTempTestDir()
c.Assert(err, IsNil)
defer os.RemoveAll(rootDir)
var objectStorage storage.ObjectStorage
objectStorage, _ = NewStorage(rootDir)
objectBuffer1 := bytes.NewBuffer([]byte("object1"))
objectStorage.Put("path1/path2/path3", objectBuffer1)
// assert object1 was created in correct path
objectResult1, err := objectStorage.Get("path1/path2/path3")
c.Assert(err, IsNil)
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1")
// add second object
objectBuffer2 := bytes.NewBuffer([]byte("object2"))
err = objectStorage.Put("path2/path2/path2", objectBuffer2)
c.Assert(err, IsNil)
// add third object
objectBuffer3 := bytes.NewBuffer([]byte("object3"))
err = objectStorage.Put("object3", objectBuffer3)
c.Assert(err, IsNil)
objectList, err := objectStorage.List("/")
c.Assert(err, IsNil)
c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "object3", IsDir: false, Hash: ""})
c.Assert(objectList[1], Equals, storage.ObjectDescription{Path: "path1", IsDir: true, Hash: ""})
c.Assert(objectList[2], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
c.Assert(len(objectList), Equals, 3)
objectList, err = objectStorage.List("/path1")
c.Assert(err, IsNil)
c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path2", IsDir: true, Hash: ""})
c.Assert(len(objectList), Equals, 1)
objectList, err = objectStorage.List("/path1/path2")
c.Assert(err, IsNil)
c.Assert(objectList[0], Equals, storage.ObjectDescription{Path: "path3", IsDir: false, Hash: ""})
c.Assert(len(objectList), Equals, 1)
objectList, err = objectStorage.List("/path1/path2/path3")
c.Assert(err, Not(IsNil))
c.Assert(objectList, IsNil)
c.Assert(objectList[0].Name, Equals, "path1")
}

9
pkgs/storage/storage.go
View File

@ -3,13 +3,14 @@ package storage
import "io"
type ObjectStorage interface {
List(path string) ([]ObjectDescription, error)
List() ([]ObjectDescription, error)
Get(path string) (io.Reader, error)
Put(path string, object io.Reader) error
}
type ObjectDescription struct {
Path string
IsDir bool
Hash string
Name string
Md5sum string
Protectionlevel string
// Hash string - TODO
}