Merge pull request #97 from fkautz/pr_out_adding_append_storage_driver

This commit is contained in:
Frederick F. Kautz IV 2014-12-08 23:37:54 -08:00
commit 93e57118f8
2 changed files with 212 additions and 0 deletions

View File

@ -0,0 +1,99 @@
package appendstorage
import (
"bytes"
"encoding/gob"
"io/ioutil"
"os"
"path"
"strconv"
"github.com/minio-io/minio/pkgs/storage"
)
type appendStorage struct {
RootDir string
file *os.File
objects map[string]Header
objectsFile string
}
type Header struct {
Path string
Offset int64
Length int
Crc []byte
}
func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
rootPath := path.Join(rootDir, strconv.Itoa(slice))
// TODO verify and fix partial writes
file, err := os.OpenFile(rootPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return &appendStorage{}, err
}
objectsFile := path.Join(rootDir, strconv.Itoa(slice)+".map")
objects := make(map[string]Header)
if _, err := os.Stat(objectsFile); err == nil {
mapFile, err := os.Open(objectsFile)
defer mapFile.Close()
if err != nil {
return &appendStorage{}, nil
}
dec := gob.NewDecoder(mapFile)
err = dec.Decode(&objects)
if err != nil {
return &appendStorage{}, nil
}
}
if err != nil {
return &appendStorage{}, err
}
return &appendStorage{
RootDir: rootDir,
file: file,
objects: objects,
objectsFile: objectsFile,
}, nil
}
func (storage *appendStorage) Get(objectPath string) ([]byte, error) {
header, ok := storage.objects[objectPath]
if ok == false {
return nil, nil
}
offset := header.Offset
length := header.Length
object := make([]byte, length)
_, err := storage.file.ReadAt(object, offset)
if err != nil {
return nil, err
}
return object, nil
}
func (storage *appendStorage) Put(objectPath string, object []byte) error {
header := Header{
Path: objectPath,
Offset: 0,
Length: 0,
Crc: nil,
}
offset, err := storage.file.Seek(0, os.SEEK_END)
if err != nil {
return err
}
if _, err := storage.file.Write(object); err != nil {
return err
}
header.Offset = offset
header.Length = len(object)
storage.objects[objectPath] = header
var mapBuffer bytes.Buffer
encoder := gob.NewEncoder(&mapBuffer)
encoder.Encode(storage.objects)
ioutil.WriteFile(storage.objectsFile, mapBuffer.Bytes(), 0600)
return nil
}

View File

@ -0,0 +1,113 @@
package appendstorage
import (
"io/ioutil"
"os"
"testing"
"github.com/minio-io/minio/pkgs/storage"
. "gopkg.in/check.v1"
)
type AppendStorageSuite struct{}
var _ = Suite(&AppendStorageSuite{})
func Test(t *testing.T) { TestingT(t) }
func makeTempTestDir() (string, error) {
return ioutil.TempDir("/tmp", "minio-test-")
}
func (s *AppendStorageSuite) TestAppendStoragePutAtRootPath(c *C) {
rootDir, err := makeTempTestDir()
c.Assert(err, IsNil)
defer os.RemoveAll(rootDir)
var objectStorage storage.ObjectStorage
objectStorage, err = NewStorage(rootDir, 0)
c.Assert(err, IsNil)
err = objectStorage.Put("path1", []byte("object1"))
c.Assert(err, IsNil)
// assert object1 was created in correct path
object1, err := objectStorage.Get("path1")
c.Assert(err, IsNil)
c.Assert(string(object1), Equals, "object1")
err = objectStorage.Put("path2", []byte("object2"))
c.Assert(err, IsNil)
// assert object1 was created in correct path
object2, err := objectStorage.Get("path2")
c.Assert(err, IsNil)
c.Assert(string(object2), Equals, "object2")
object1, err = objectStorage.Get("path1")
c.Assert(err, IsNil)
c.Assert(string(object1), Equals, "object1")
}
func (s *AppendStorageSuite) TestAppendStoragePutDirPath(c *C) {
rootDir, err := makeTempTestDir()
c.Assert(err, IsNil)
defer os.RemoveAll(rootDir)
var objectStorage storage.ObjectStorage
objectStorage, err = NewStorage(rootDir, 0)
c.Assert(err, IsNil)
// add object 1
objectStorage.Put("path1/path2/path3", []byte("object"))
// assert object1 was created in correct path
object1, err := objectStorage.Get("path1/path2/path3")
c.Assert(err, IsNil)
c.Assert(string(object1), Equals, "object")
// add object 2
objectStorage.Put("path1/path1/path1", []byte("object2"))
// assert object1 was created in correct path
object2, err := objectStorage.Get("path1/path1/path1")
c.Assert(err, IsNil)
c.Assert(string(object2), Equals, "object2")
}
func (s *AppendStorageSuite) TestSerialization(c *C) {
rootDir, err := makeTempTestDir()
c.Assert(err, IsNil)
defer os.RemoveAll(rootDir)
objectStorage, err := NewStorage(rootDir, 0)
c.Assert(err, IsNil)
err = objectStorage.Put("path1", []byte("object1"))
c.Assert(err, IsNil)
err = objectStorage.Put("path2", []byte("object2"))
c.Assert(err, IsNil)
err = objectStorage.Put("path3/obj3", []byte("object3"))
c.Assert(err, IsNil)
es := objectStorage.(*appendStorage)
es.file.Close()
objectStorage2, err := NewStorage(rootDir, 0)
c.Assert(err, IsNil)
object1, err := objectStorage2.Get("path1")
c.Assert(err, IsNil)
c.Assert(string(object1), Equals, "object1")
object2, err := objectStorage2.Get("path2")
c.Assert(err, IsNil)
c.Assert(string(object2), Equals, "object2")
object3, err := objectStorage2.Get("path3/obj3")
c.Assert(err, IsNil)
c.Assert(string(object3), Equals, "object3")
}
func (s *AppendStorageSuite) TestSlice(c *C) {
}