MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

ObjectStorage refactor to take io.Reader instead of []byte

Browse Source
This commit is contained in:
Frederick F. Kautz IV 2014-12-10 20:40:53 -08:00
parent c7b4e14f64
commit 19da9760b3
7 changed files with 65 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/erasure-demo/fs.go
View File

@ -35,8 +35,7 @@ func fsGet(config inputConfig, objectPath string) (io.Reader, error) {
if err != nil {
return nil, err
}
objectBuffer := bytes.NewBuffer(object)
return objectBuffer, nil
return object, nil
}
func fsPut(config inputConfig, objectPath string, reader io.Reader) error {
@ -46,11 +45,8 @@ func fsPut(config inputConfig, objectPath string, reader io.Reader) error {
return err
}
var objectStorage storage.ObjectStorage
buffer := new(bytes.Buffer)
buffer.ReadFrom(reader)
object := buffer.Bytes()
objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir}
if err = objectStorage.Put(objectPath, object); err != nil {
if err = objectStorage.Put(objectPath, reader); err != nil {
return err
}
return nil

3
pkgs/gateway/gateway.go
View File

@ -1,6 +1,7 @@
package gateway
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
@ -212,7 +213,7 @@ func SimpleFileStorageDriver(bucket string, input chan ObjectRequest, config Gat
}
case "PUT":
objectPath := path.Join(bucket, request.path)
fileStorage.Put(objectPath, request.object)
fileStorage.Put(objectPath, bytes.NewBuffer(request.object))
request.callback <- nil
default:
request.callback <- errors.New("Unexpected message")

15
pkgs/storage/appendstorage/append_storage.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/gob"
"errors"
"io"
"io/ioutil"
"os"
"path"
@ -58,7 +59,7 @@ func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
}, nil
}
func (storage *appendStorage) Get(objectPath string) ([]byte, error) {
func (storage *appendStorage) Get(objectPath string) (io.Reader, error) {
header, ok := storage.objects[objectPath]
if ok == false {
return nil, nil
@ -72,10 +73,10 @@ func (storage *appendStorage) Get(objectPath string) ([]byte, error) {
if err != nil {
return nil, err
}
return object, nil
return bytes.NewBuffer(object), nil
}
func (aStorage *appendStorage) Put(objectPath string, object []byte) error {
func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
header := Header{
Path: objectPath,
Offset: 0,
@ -86,11 +87,15 @@ func (aStorage *appendStorage) Put(objectPath string, object []byte) error {
if err != nil {
return err
}
if _, err := aStorage.file.Write(object); err != nil {
objectBytes, err := ioutil.ReadAll(object)
if err != nil {
return err
}
if _, err := aStorage.file.Write(objectBytes); err != nil {
return err
}
header.Offset = offset
header.Length = len(object)
header.Length = len(objectBytes)
aStorage.objects[objectPath] = header
var mapBuffer bytes.Buffer
encoder := gob.NewEncoder(&mapBuffer)

42
pkgs/storage/appendstorage/append_storage_test.go
View File

@ -1,6 +1,7 @@
package appendstorage
import (
"bytes"
"io/ioutil"
"os"
"testing"
@ -28,24 +29,27 @@ func (s *AppendStorageSuite) TestAppendStoragePutAtRootPath(c *C) {
objectStorage, err = NewStorage(rootDir, 0)
c.Assert(err, IsNil)
err = objectStorage.Put("path1", []byte("object1"))
err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1")))
c.Assert(err, IsNil)
// assert object1 was created in correct path
object1, err := objectStorage.Get("path1")
objectResult1, err := objectStorage.Get("path1")
c.Assert(err, IsNil)
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1")
err = objectStorage.Put("path2", []byte("object2"))
err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2")))
c.Assert(err, IsNil)
// assert object1 was created in correct path
object2, err := objectStorage.Get("path2")
objectResult2, err := objectStorage.Get("path2")
c.Assert(err, IsNil)
object2, _ := ioutil.ReadAll(objectResult2)
c.Assert(string(object2), Equals, "object2")
object1, err = objectStorage.Get("path1")
objectResult1, err = objectStorage.Get("path1")
c.Assert(err, IsNil)
object1, _ = ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1")
}
@ -59,19 +63,21 @@ func (s *AppendStorageSuite) TestAppendStoragePutDirPath(c *C) {
c.Assert(err, IsNil)
// add object 1
objectStorage.Put("path1/path2/path3", []byte("object"))
objectStorage.Put("path1/path2/path3", bytes.NewBuffer([]byte("object")))
// assert object1 was created in correct path
object1, err := objectStorage.Get("path1/path2/path3")
objectResult1, err := objectStorage.Get("path1/path2/path3")
c.Assert(err, IsNil)
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object")
// add object 2
objectStorage.Put("path1/path1/path1", []byte("object2"))
objectStorage.Put("path1/path1/path1", bytes.NewBuffer([]byte("object2")))
// assert object1 was created in correct path
object2, err := objectStorage.Get("path1/path1/path1")
objectResult2, err := objectStorage.Get("path1/path1/path1")
c.Assert(err, IsNil)
object2, _ := ioutil.ReadAll(objectResult2)
c.Assert(string(object2), Equals, "object2")
}
@ -83,11 +89,11 @@ func (s *AppendStorageSuite) TestSerialization(c *C) {
objectStorage, err := NewStorage(rootDir, 0)
c.Assert(err, IsNil)
err = objectStorage.Put("path1", []byte("object1"))
err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1")))
c.Assert(err, IsNil)
err = objectStorage.Put("path2", []byte("object2"))
err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2")))
c.Assert(err, IsNil)
err = objectStorage.Put("path3/obj3", []byte("object3"))
err = objectStorage.Put("path3/obj3", bytes.NewBuffer([]byte("object3")))
c.Assert(err, IsNil)
es := objectStorage.(*appendStorage)
@ -96,18 +102,18 @@ func (s *AppendStorageSuite) TestSerialization(c *C) {
objectStorage2, err := NewStorage(rootDir, 0)
c.Assert(err, IsNil)
object1, err := objectStorage2.Get("path1")
objectResult1, err := objectStorage2.Get("path1")
c.Assert(err, IsNil)
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1")
object2, err := objectStorage2.Get("path2")
objectResult2, err := objectStorage2.Get("path2")
c.Assert(err, IsNil)
object2, _ := ioutil.ReadAll(objectResult2)
c.Assert(string(object2), Equals, "object2")
object3, err := objectStorage2.Get("path3/obj3")
objectResult3, err := objectStorage2.Get("path3/obj3")
c.Assert(err, IsNil)
object3, _ := ioutil.ReadAll(objectResult3)
c.Assert(string(object3), Equals, "object3")
}
func (s *AppendStorageSuite) TestSlice(c *C) {
}

13
pkgs/storage/fsstorage/fs_storage.go
View File

@ -1,6 +1,7 @@
package fsstorage
import (
"io"
"io/ioutil"
"os"
"path"
@ -32,14 +33,18 @@ func (fsStorage FileSystemStorage) List(listPath string) ([]storage.ObjectDescri
return descriptions, nil
}
func (storage FileSystemStorage) Get(objectPath string) ([]byte, error) {
return ioutil.ReadFile(path.Join(storage.RootDir, objectPath))
func (storage FileSystemStorage) Get(objectPath string) (io.Reader, error) {
return os.Open(path.Join(storage.RootDir, objectPath))
}
func (storage FileSystemStorage) Put(objectPath string, object []byte) error {
func (storage FileSystemStorage) Put(objectPath string, object io.Reader) error {
err := os.MkdirAll(filepath.Dir(path.Join(storage.RootDir, objectPath)), 0700)
if err != nil {
return err
}
return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), object, 0600)
objectBytes, err := ioutil.ReadAll(object)
if err != nil {
return err
}
return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), objectBytes, 0600)
}

21
pkgs/storage/fsstorage/fs_storage_test.go
View File

@ -1,6 +1,7 @@
package fsstorage
import (
"bytes"
"io/ioutil"
"os"
"testing"
@ -29,11 +30,13 @@ func (s *FileSystemStorageSuite) TestFileStoragePutAtRootPath(c *C) {
RootDir: rootDir,
}
objectStorage.Put("path1", []byte("object1"))
objectBuffer := bytes.NewBuffer([]byte("object1"))
objectStorage.Put("path1", objectBuffer)
// assert object1 was created in correct path
object1, err := objectStorage.Get("path1")
objectResult1, err := objectStorage.Get("path1")
c.Assert(err, IsNil)
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1")
objectList, err := objectStorage.List("/")
@ -51,19 +54,23 @@ func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) {
RootDir: rootDir,
}
objectStorage.Put("path1/path2/path3", []byte("object"))
objectBuffer1 := bytes.NewBuffer([]byte("object1"))
objectStorage.Put("path1/path2/path3", objectBuffer1)
// assert object1 was created in correct path
object1, err := objectStorage.Get("path1/path2/path3")
objectResult1, err := objectStorage.Get("path1/path2/path3")
c.Assert(err, IsNil)
c.Assert(string(object1), Equals, "object")
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1")
// add second object
err = objectStorage.Put("path2/path2/path2", []byte("object2"))
objectBuffer2 := bytes.NewBuffer([]byte("object2"))
err = objectStorage.Put("path2/path2/path2", objectBuffer2)
c.Assert(err, IsNil)
// add third object
err = objectStorage.Put("object3", []byte("object3"))
objectBuffer3 := bytes.NewBuffer([]byte("object3"))
err = objectStorage.Put("object3", objectBuffer3)
c.Assert(err, IsNil)
objectList, err := objectStorage.List("/")

6
pkgs/storage/storage.go
View File

@ -1,9 +1,11 @@
package storage
import "io"
type ObjectStorage interface {
List(path string) ([]ObjectDescription, error)
Get(path string) ([]byte, error)
Put(path string, object []byte) error
Get(path string) (io.Reader, error)
Put(path string, object io.Reader) error
}
type ObjectDescription struct {