mirror of
https://github.com/minio/minio.git
synced 2024-12-23 21:55:53 -05:00
Adding append storage driver
This commit is contained in:
parent
76f21c1cfc
commit
1d7b3866cb
99
pkgs/storage/appendstorage/append_storage.go
Normal file
99
pkgs/storage/appendstorage/append_storage.go
Normal file
@ -0,0 +1,99 @@
|
||||
package appendstorage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/minio-io/minio/pkgs/storage"
|
||||
)
|
||||
|
||||
type appendStorage struct {
|
||||
RootDir string
|
||||
file *os.File
|
||||
objects map[string]Header
|
||||
objectsFile string
|
||||
}
|
||||
|
||||
type Header struct {
|
||||
Path string
|
||||
Offset int64
|
||||
Length int
|
||||
Crc []byte
|
||||
}
|
||||
|
||||
func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
|
||||
rootPath := path.Join(rootDir, strconv.Itoa(slice))
|
||||
// TODO verify and fix partial writes
|
||||
file, err := os.OpenFile(rootPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return &appendStorage{}, err
|
||||
}
|
||||
objectsFile := path.Join(rootDir, strconv.Itoa(slice)+".map")
|
||||
objects := make(map[string]Header)
|
||||
if _, err := os.Stat(objectsFile); err == nil {
|
||||
mapFile, err := os.Open(objectsFile)
|
||||
defer mapFile.Close()
|
||||
if err != nil {
|
||||
return &appendStorage{}, nil
|
||||
}
|
||||
dec := gob.NewDecoder(mapFile)
|
||||
err = dec.Decode(&objects)
|
||||
if err != nil {
|
||||
return &appendStorage{}, nil
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return &appendStorage{}, err
|
||||
}
|
||||
return &appendStorage{
|
||||
RootDir: rootDir,
|
||||
file: file,
|
||||
objects: objects,
|
||||
objectsFile: objectsFile,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (storage *appendStorage) Get(objectPath string) ([]byte, error) {
|
||||
header, ok := storage.objects[objectPath]
|
||||
if ok == false {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
offset := header.Offset
|
||||
length := header.Length
|
||||
|
||||
object := make([]byte, length)
|
||||
_, err := storage.file.ReadAt(object, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return object, nil
|
||||
}
|
||||
|
||||
func (storage *appendStorage) Put(objectPath string, object []byte) error {
|
||||
header := Header{
|
||||
Path: objectPath,
|
||||
Offset: 0,
|
||||
Length: 0,
|
||||
Crc: nil,
|
||||
}
|
||||
offset, err := storage.file.Seek(0, os.SEEK_END)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := storage.file.Write(object); err != nil {
|
||||
return err
|
||||
}
|
||||
header.Offset = offset
|
||||
header.Length = len(object)
|
||||
storage.objects[objectPath] = header
|
||||
var mapBuffer bytes.Buffer
|
||||
encoder := gob.NewEncoder(&mapBuffer)
|
||||
encoder.Encode(storage.objects)
|
||||
ioutil.WriteFile(storage.objectsFile, mapBuffer.Bytes(), 0600)
|
||||
return nil
|
||||
}
|
113
pkgs/storage/appendstorage/append_storage_test.go
Normal file
113
pkgs/storage/appendstorage/append_storage_test.go
Normal file
@ -0,0 +1,113 @@
|
||||
package appendstorage
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/minio-io/minio/pkgs/storage"
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
type AppendStorageSuite struct{}
|
||||
|
||||
var _ = Suite(&AppendStorageSuite{})
|
||||
|
||||
func Test(t *testing.T) { TestingT(t) }
|
||||
|
||||
func makeTempTestDir() (string, error) {
|
||||
return ioutil.TempDir("/tmp", "minio-test-")
|
||||
}
|
||||
|
||||
func (s *AppendStorageSuite) TestAppendStoragePutAtRootPath(c *C) {
|
||||
rootDir, err := makeTempTestDir()
|
||||
c.Assert(err, IsNil)
|
||||
defer os.RemoveAll(rootDir)
|
||||
|
||||
var objectStorage storage.ObjectStorage
|
||||
objectStorage, err = NewStorage(rootDir, 0)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
err = objectStorage.Put("path1", []byte("object1"))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
// assert object1 was created in correct path
|
||||
object1, err := objectStorage.Get("path1")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(string(object1), Equals, "object1")
|
||||
|
||||
err = objectStorage.Put("path2", []byte("object2"))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
// assert object1 was created in correct path
|
||||
object2, err := objectStorage.Get("path2")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(string(object2), Equals, "object2")
|
||||
|
||||
object1, err = objectStorage.Get("path1")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(string(object1), Equals, "object1")
|
||||
}
|
||||
|
||||
func (s *AppendStorageSuite) TestAppendStoragePutDirPath(c *C) {
|
||||
rootDir, err := makeTempTestDir()
|
||||
c.Assert(err, IsNil)
|
||||
defer os.RemoveAll(rootDir)
|
||||
|
||||
var objectStorage storage.ObjectStorage
|
||||
objectStorage, err = NewStorage(rootDir, 0)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
// add object 1
|
||||
objectStorage.Put("path1/path2/path3", []byte("object"))
|
||||
|
||||
// assert object1 was created in correct path
|
||||
object1, err := objectStorage.Get("path1/path2/path3")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(string(object1), Equals, "object")
|
||||
|
||||
// add object 2
|
||||
objectStorage.Put("path1/path1/path1", []byte("object2"))
|
||||
|
||||
// assert object1 was created in correct path
|
||||
object2, err := objectStorage.Get("path1/path1/path1")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(string(object2), Equals, "object2")
|
||||
}
|
||||
|
||||
func (s *AppendStorageSuite) TestSerialization(c *C) {
|
||||
rootDir, err := makeTempTestDir()
|
||||
c.Assert(err, IsNil)
|
||||
defer os.RemoveAll(rootDir)
|
||||
|
||||
objectStorage, err := NewStorage(rootDir, 0)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
err = objectStorage.Put("path1", []byte("object1"))
|
||||
c.Assert(err, IsNil)
|
||||
err = objectStorage.Put("path2", []byte("object2"))
|
||||
c.Assert(err, IsNil)
|
||||
err = objectStorage.Put("path3/obj3", []byte("object3"))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
es := objectStorage.(*appendStorage)
|
||||
es.file.Close()
|
||||
|
||||
objectStorage2, err := NewStorage(rootDir, 0)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
object1, err := objectStorage2.Get("path1")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(string(object1), Equals, "object1")
|
||||
|
||||
object2, err := objectStorage2.Get("path2")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(string(object2), Equals, "object2")
|
||||
|
||||
object3, err := objectStorage2.Get("path3/obj3")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(string(object3), Equals, "object3")
|
||||
}
|
||||
|
||||
func (s *AppendStorageSuite) TestSlice(c *C) {
|
||||
}
|
Loading…
Reference in New Issue
Block a user