MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

ObjectStorage refactor to take io.Reader instead of []byte

Browse Source
This commit is contained in:
Frederick F. Kautz IV 2014-12-10 20:40:53 -08:00
parent c7b4e14f64
commit 19da9760b3
7 changed files with 65 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/erasure-demo/fs.go
View File

@ -35,8 +35,7 @@ func fsGet(config inputConfig, objectPath string) (io.Reader, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
objectBuffer := bytes.NewBuffer(object) return object, nil
return objectBuffer, nil
} }
func fsPut(config inputConfig, objectPath string, reader io.Reader) error { func fsPut(config inputConfig, objectPath string, reader io.Reader) error {
@ -46,11 +45,8 @@ func fsPut(config inputConfig, objectPath string, reader io.Reader) error {
return err return err
} }
var objectStorage storage.ObjectStorage var objectStorage storage.ObjectStorage
buffer := new(bytes.Buffer)
buffer.ReadFrom(reader)
object := buffer.Bytes()
objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir} objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir}
if err = objectStorage.Put(objectPath, object); err != nil { if err = objectStorage.Put(objectPath, reader); err != nil {
return err return err
} }
return nil return nil

3
pkgs/gateway/gateway.go
View File

@ -1,6 +1,7 @@
package gateway package gateway
import ( import (
"bytes"
"errors" "errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
@ -212,7 +213,7 @@ func SimpleFileStorageDriver(bucket string, input chan ObjectRequest, config Gat
} }
case "PUT": case "PUT":
objectPath := path.Join(bucket, request.path) objectPath := path.Join(bucket, request.path)
fileStorage.Put(objectPath, request.object) fileStorage.Put(objectPath, bytes.NewBuffer(request.object))
request.callback <- nil request.callback <- nil
default: default:
request.callback <- errors.New("Unexpected message") request.callback <- errors.New("Unexpected message")

15
pkgs/storage/appendstorage/append_storage.go
View File

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"encoding/gob" "encoding/gob"
"errors" "errors"
"io"
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path"
@ -58,7 +59,7 @@ func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
}, nil }, nil
} }
func (storage *appendStorage) Get(objectPath string) ([]byte, error) { func (storage *appendStorage) Get(objectPath string) (io.Reader, error) {
header, ok := storage.objects[objectPath] header, ok := storage.objects[objectPath]
if ok == false { if ok == false {
return nil, nil return nil, nil
@ -72,10 +73,10 @@ func (storage *appendStorage) Get(objectPath string) ([]byte, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return object, nil return bytes.NewBuffer(object), nil
} }
func (aStorage *appendStorage) Put(objectPath string, object []byte) error { func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
header := Header{ header := Header{
Path: objectPath, Path: objectPath,
Offset: 0, Offset: 0,
@ -86,11 +87,15 @@ func (aStorage *appendStorage) Put(objectPath string, object []byte) error {
if err != nil { if err != nil {
return err return err
} }
if _, err := aStorage.file.Write(object); err != nil { objectBytes, err := ioutil.ReadAll(object)
if err != nil {
return err
}
if _, err := aStorage.file.Write(objectBytes); err != nil {
return err return err
} }
header.Offset = offset header.Offset = offset
header.Length = len(object) header.Length = len(objectBytes)
aStorage.objects[objectPath] = header aStorage.objects[objectPath] = header
var mapBuffer bytes.Buffer var mapBuffer bytes.Buffer
encoder := gob.NewEncoder(&mapBuffer) encoder := gob.NewEncoder(&mapBuffer)

42
pkgs/storage/appendstorage/append_storage_test.go
View File

@ -1,6 +1,7 @@
package appendstorage package appendstorage
import ( import (
"bytes"
"io/ioutil" "io/ioutil"
"os" "os"
"testing" "testing"
@ -28,24 +29,27 @@ func (s *AppendStorageSuite) TestAppendStoragePutAtRootPath(c *C) {
objectStorage, err = NewStorage(rootDir, 0) objectStorage, err = NewStorage(rootDir, 0)
c.Assert(err, IsNil) c.Assert(err, IsNil)
err = objectStorage.Put("path1", []byte("object1")) err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1")))
c.Assert(err, IsNil) c.Assert(err, IsNil)
// assert object1 was created in correct path // assert object1 was created in correct path
object1, err := objectStorage.Get("path1") objectResult1, err := objectStorage.Get("path1")
c.Assert(err, IsNil) c.Assert(err, IsNil)
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1") c.Assert(string(object1), Equals, "object1")
err = objectStorage.Put("path2", []byte("object2")) err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2")))
c.Assert(err, IsNil) c.Assert(err, IsNil)
// assert object1 was created in correct path // assert object1 was created in correct path
object2, err := objectStorage.Get("path2") objectResult2, err := objectStorage.Get("path2")
c.Assert(err, IsNil) c.Assert(err, IsNil)
object2, _ := ioutil.ReadAll(objectResult2)
c.Assert(string(object2), Equals, "object2") c.Assert(string(object2), Equals, "object2")
object1, err = objectStorage.Get("path1") objectResult1, err = objectStorage.Get("path1")
c.Assert(err, IsNil) c.Assert(err, IsNil)
object1, _ = ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1") c.Assert(string(object1), Equals, "object1")
} }
@ -59,19 +63,21 @@ func (s *AppendStorageSuite) TestAppendStoragePutDirPath(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
// add object 1 // add object 1
objectStorage.Put("path1/path2/path3", []byte("object")) objectStorage.Put("path1/path2/path3", bytes.NewBuffer([]byte("object")))
// assert object1 was created in correct path // assert object1 was created in correct path
object1, err := objectStorage.Get("path1/path2/path3") objectResult1, err := objectStorage.Get("path1/path2/path3")
c.Assert(err, IsNil) c.Assert(err, IsNil)
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object") c.Assert(string(object1), Equals, "object")
// add object 2 // add object 2
objectStorage.Put("path1/path1/path1", []byte("object2")) objectStorage.Put("path1/path1/path1", bytes.NewBuffer([]byte("object2")))
// assert object1 was created in correct path // assert object1 was created in correct path
object2, err := objectStorage.Get("path1/path1/path1") objectResult2, err := objectStorage.Get("path1/path1/path1")
c.Assert(err, IsNil) c.Assert(err, IsNil)
object2, _ := ioutil.ReadAll(objectResult2)
c.Assert(string(object2), Equals, "object2") c.Assert(string(object2), Equals, "object2")
} }
@ -83,11 +89,11 @@ func (s *AppendStorageSuite) TestSerialization(c *C) {
objectStorage, err := NewStorage(rootDir, 0) objectStorage, err := NewStorage(rootDir, 0)
c.Assert(err, IsNil) c.Assert(err, IsNil)
err = objectStorage.Put("path1", []byte("object1")) err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1")))
c.Assert(err, IsNil) c.Assert(err, IsNil)
err = objectStorage.Put("path2", []byte("object2")) err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2")))
c.Assert(err, IsNil) c.Assert(err, IsNil)
err = objectStorage.Put("path3/obj3", []byte("object3")) err = objectStorage.Put("path3/obj3", bytes.NewBuffer([]byte("object3")))
c.Assert(err, IsNil) c.Assert(err, IsNil)
es := objectStorage.(*appendStorage) es := objectStorage.(*appendStorage)
@ -96,18 +102,18 @@ func (s *AppendStorageSuite) TestSerialization(c *C) {
objectStorage2, err := NewStorage(rootDir, 0) objectStorage2, err := NewStorage(rootDir, 0)
c.Assert(err, IsNil) c.Assert(err, IsNil)
object1, err := objectStorage2.Get("path1") objectResult1, err := objectStorage2.Get("path1")
c.Assert(err, IsNil) c.Assert(err, IsNil)
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1") c.Assert(string(object1), Equals, "object1")
object2, err := objectStorage2.Get("path2") objectResult2, err := objectStorage2.Get("path2")
c.Assert(err, IsNil) c.Assert(err, IsNil)
object2, _ := ioutil.ReadAll(objectResult2)
c.Assert(string(object2), Equals, "object2") c.Assert(string(object2), Equals, "object2")
object3, err := objectStorage2.Get("path3/obj3") objectResult3, err := objectStorage2.Get("path3/obj3")
c.Assert(err, IsNil) c.Assert(err, IsNil)
object3, _ := ioutil.ReadAll(objectResult3)
c.Assert(string(object3), Equals, "object3") c.Assert(string(object3), Equals, "object3")
} }
func (s *AppendStorageSuite) TestSlice(c *C) {
}

13
pkgs/storage/fsstorage/fs_storage.go
View File

@ -1,6 +1,7 @@
package fsstorage package fsstorage
import ( import (
"io"
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path"
@ -32,14 +33,18 @@ func (fsStorage FileSystemStorage) List(listPath string) ([]storage.ObjectDescri
return descriptions, nil return descriptions, nil
} }
func (storage FileSystemStorage) Get(objectPath string) ([]byte, error) { func (storage FileSystemStorage) Get(objectPath string) (io.Reader, error) {
return ioutil.ReadFile(path.Join(storage.RootDir, objectPath)) return os.Open(path.Join(storage.RootDir, objectPath))
} }
func (storage FileSystemStorage) Put(objectPath string, object []byte) error { func (storage FileSystemStorage) Put(objectPath string, object io.Reader) error {
err := os.MkdirAll(filepath.Dir(path.Join(storage.RootDir, objectPath)), 0700) err := os.MkdirAll(filepath.Dir(path.Join(storage.RootDir, objectPath)), 0700)
if err != nil { if err != nil {
return err return err
} }
return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), object, 0600) objectBytes, err := ioutil.ReadAll(object)
if err != nil {
return err
}
return ioutil.WriteFile(path.Join(storage.RootDir, objectPath), objectBytes, 0600)
} }

21
pkgs/storage/fsstorage/fs_storage_test.go
View File

@ -1,6 +1,7 @@
package fsstorage package fsstorage
import ( import (
"bytes"
"io/ioutil" "io/ioutil"
"os" "os"
"testing" "testing"
@ -29,11 +30,13 @@ func (s *FileSystemStorageSuite) TestFileStoragePutAtRootPath(c *C) {
RootDir: rootDir, RootDir: rootDir,
} }
objectStorage.Put("path1", []byte("object1")) objectBuffer := bytes.NewBuffer([]byte("object1"))
objectStorage.Put("path1", objectBuffer)
// assert object1 was created in correct path // assert object1 was created in correct path
object1, err := objectStorage.Get("path1") objectResult1, err := objectStorage.Get("path1")
c.Assert(err, IsNil) c.Assert(err, IsNil)
object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1") c.Assert(string(object1), Equals, "object1")
objectList, err := objectStorage.List("/") objectList, err := objectStorage.List("/")
@ -51,19 +54,23 @@ func (s *FileSystemStorageSuite) TestFileStoragePutDirPath(c *C) {
RootDir: rootDir, RootDir: rootDir,
} }
objectStorage.Put("path1/path2/path3", []byte("object")) objectBuffer1 := bytes.NewBuffer([]byte("object1"))
objectStorage.Put("path1/path2/path3", objectBuffer1)
// assert object1 was created in correct path // assert object1 was created in correct path
object1, err := objectStorage.Get("path1/path2/path3") objectResult1, err := objectStorage.Get("path1/path2/path3")
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(string(object1), Equals, "object") object1, _ := ioutil.ReadAll(objectResult1)
c.Assert(string(object1), Equals, "object1")
// add second object // add second object
err = objectStorage.Put("path2/path2/path2", []byte("object2")) objectBuffer2 := bytes.NewBuffer([]byte("object2"))
err = objectStorage.Put("path2/path2/path2", objectBuffer2)
c.Assert(err, IsNil) c.Assert(err, IsNil)
// add third object // add third object
err = objectStorage.Put("object3", []byte("object3")) objectBuffer3 := bytes.NewBuffer([]byte("object3"))
err = objectStorage.Put("object3", objectBuffer3)
c.Assert(err, IsNil) c.Assert(err, IsNil)
objectList, err := objectStorage.List("/") objectList, err := objectStorage.List("/")

6
pkgs/storage/storage.go
View File

@ -1,9 +1,11 @@
package storage package storage
import "io"
type ObjectStorage interface { type ObjectStorage interface {
List(path string) ([]ObjectDescription, error) List(path string) ([]ObjectDescription, error)
Get(path string) ([]byte, error) Get(path string) (io.Reader, error)
Put(path string, object []byte) error Put(path string, object io.Reader) error
} }
type ObjectDescription struct { type ObjectDescription struct {