A full restructure

pkg/storage/appendstorage/Makefile

@@ -0,0 +1,11 @@
+all: build test
+.PHONY: all
+
+build:
+	@godep go build
+
+test: build
+	@godep go test -race -coverprofile=cover.out
+
+clean:
+	@rm -v cover.out

pkg/storage/appendstorage/append_storage.go

@@ -0,0 +1,152 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package appendstorage
+
+import (
+	"bytes"
+	"encoding/gob"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/minio-io/minio/pkg/checksum/crc32c"
+	"github.com/minio-io/minio/pkg/storage"
+)
+
+type appendStorage struct {
+	RootDir     string
+	file        *os.File
+	objects     map[string]Header
+	objectsFile string
+}
+
+type Header struct {
+	Path   string
+	Offset int64
+	Length int
+	Crc    uint32
+}
+
+func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
+	rootPath := path.Join(rootDir, strconv.Itoa(slice))
+	// TODO verify and fix partial writes
+	file, err := os.OpenFile(rootPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0600)
+	if err != nil {
+		return &appendStorage{}, err
+	}
+	objectsFile := path.Join(rootDir, strconv.Itoa(slice)+".map")
+	objects := make(map[string]Header)
+	if _, err := os.Stat(objectsFile); err == nil {
+		mapFile, err := os.Open(objectsFile)
+		defer mapFile.Close()
+		if err != nil {
+			return &appendStorage{}, nil
+		}
+		dec := gob.NewDecoder(mapFile)
+		err = dec.Decode(&objects)
+		if err != nil && err != io.EOF {
+			return &appendStorage{}, nil
+		}
+	}
+	if err != nil {
+		return &appendStorage{}, err
+	}
+	return &appendStorage{
+		RootDir:     rootDir,
+		file:        file,
+		objects:     objects,
+		objectsFile: objectsFile,
+	}, nil
+}
+
+func (storage *appendStorage) Get(objectPath string) (io.Reader, error) {
+	header, ok := storage.objects[objectPath]
+	if ok == false {
+		return nil, errors.New("Object not found")
+	}
+
+	offset := header.Offset
+	length := header.Length
+	crc := header.Crc
+
+	object := make([]byte, length)
+	_, err := storage.file.ReadAt(object, offset)
+	if err != nil {
+		return nil, err
+	}
+	newcrc, err := crc32c.Crc32c(object)
+	if err != nil {
+		return nil, err
+	}
+	if newcrc != crc {
+		return nil, err
+	}
+	return bytes.NewBuffer(object), nil
+}
+
+func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
+	header := Header{
+		Path:   objectPath,
+		Offset: 0,
+		Length: 0,
+		Crc:    0,
+	}
+	offset, err := aStorage.file.Seek(0, os.SEEK_END)
+	if err != nil {
+		return err
+	}
+	objectBytes, err := ioutil.ReadAll(object)
+	if err != nil {
+		return err
+	}
+	if _, err := aStorage.file.Write(objectBytes); err != nil {
+		return err
+	}
+	header.Offset = offset
+	header.Length = len(objectBytes)
+	header.Crc, err = crc32c.Crc32c(objectBytes)
+	if err != nil {
+		return err
+	}
+	aStorage.objects[objectPath] = header
+	var mapBuffer bytes.Buffer
+	encoder := gob.NewEncoder(&mapBuffer)
+	encoder.Encode(aStorage.objects)
+	ioutil.WriteFile(aStorage.objectsFile, mapBuffer.Bytes(), 0600)
+	return nil
+}
+
+func (aStorage *appendStorage) List(objectPath string) ([]storage.ObjectDescription, error) {
+	var objectDescList []storage.ObjectDescription
+	for objectName, _ := range aStorage.objects {
+		if strings.HasPrefix(objectName, objectPath) {
+			var objectDescription storage.ObjectDescription
+			objectDescription.Name = objectName
+			objectDescription.Md5sum = ""
+			objectDescription.Murmur3 = ""
+			objectDescList = append(objectDescList, objectDescription)
+		}
+	}
+	if len(objectDescList) == 0 {
+		return nil, errors.New("No objects found")
+	}
+	return objectDescList, nil
+}

pkg/storage/appendstorage/append_storage_test.go

@@ -0,0 +1,116 @@
+package appendstorage
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/minio-io/minio/pkg/storage"
+	"github.com/minio-io/minio/pkg/utils"
+	. "gopkg.in/check.v1"
+)
+
+type AppendStorageSuite struct{}
+
+var _ = Suite(&AppendStorageSuite{})
+
+func Test(t *testing.T) { TestingT(t) }
+
+func (s *AppendStorageSuite) TestAppendStoragePutAtRootPath(c *C) {
+	rootDir, err := utils.MakeTempTestDir()
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(rootDir)
+
+	var objectStorage storage.ObjectStorage
+	objectStorage, err = NewStorage(rootDir, 0)
+	c.Assert(err, IsNil)
+
+	err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1")))
+	c.Assert(err, IsNil)
+
+	// assert object1 was created in correct path
+	objectResult1, err := objectStorage.Get("path1")
+	c.Assert(err, IsNil)
+	object1, _ := ioutil.ReadAll(objectResult1)
+	c.Assert(string(object1), Equals, "object1")
+
+	err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2")))
+	c.Assert(err, IsNil)
+
+	// assert object1 was created in correct path
+	objectResult2, err := objectStorage.Get("path2")
+	c.Assert(err, IsNil)
+	object2, _ := ioutil.ReadAll(objectResult2)
+	c.Assert(string(object2), Equals, "object2")
+
+	objectResult1, err = objectStorage.Get("path1")
+	c.Assert(err, IsNil)
+	object1, _ = ioutil.ReadAll(objectResult1)
+	c.Assert(string(object1), Equals, "object1")
+}
+
+func (s *AppendStorageSuite) TestAppendStoragePutDirPath(c *C) {
+	rootDir, err := utils.MakeTempTestDir()
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(rootDir)
+
+	var objectStorage storage.ObjectStorage
+	objectStorage, err = NewStorage(rootDir, 0)
+	c.Assert(err, IsNil)
+
+	// add object 1
+	objectStorage.Put("path1/path2/path3", bytes.NewBuffer([]byte("object")))
+
+	// assert object1 was created in correct path
+	objectResult1, err := objectStorage.Get("path1/path2/path3")
+	c.Assert(err, IsNil)
+	object1, _ := ioutil.ReadAll(objectResult1)
+	c.Assert(string(object1), Equals, "object")
+
+	// add object 2
+	objectStorage.Put("path1/path1/path1", bytes.NewBuffer([]byte("object2")))
+
+	// assert object1 was created in correct path
+	objectResult2, err := objectStorage.Get("path1/path1/path1")
+	c.Assert(err, IsNil)
+	object2, _ := ioutil.ReadAll(objectResult2)
+	c.Assert(string(object2), Equals, "object2")
+}
+
+func (s *AppendStorageSuite) TestSerialization(c *C) {
+	rootDir, err := utils.MakeTempTestDir()
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(rootDir)
+
+	objectStorage, err := NewStorage(rootDir, 0)
+	c.Assert(err, IsNil)
+
+	err = objectStorage.Put("path1", bytes.NewBuffer([]byte("object1")))
+	c.Assert(err, IsNil)
+	err = objectStorage.Put("path2", bytes.NewBuffer([]byte("object2")))
+	c.Assert(err, IsNil)
+	err = objectStorage.Put("path3/obj3", bytes.NewBuffer([]byte("object3")))
+	c.Assert(err, IsNil)
+
+	es := objectStorage.(*appendStorage)
+	es.file.Close()
+
+	objectStorage2, err := NewStorage(rootDir, 0)
+	c.Assert(err, IsNil)
+
+	objectResult1, err := objectStorage2.Get("path1")
+	c.Assert(err, IsNil)
+	object1, _ := ioutil.ReadAll(objectResult1)
+	c.Assert(string(object1), Equals, "object1")
+
+	objectResult2, err := objectStorage2.Get("path2")
+	c.Assert(err, IsNil)
+	object2, _ := ioutil.ReadAll(objectResult2)
+	c.Assert(string(object2), Equals, "object2")
+
+	objectResult3, err := objectStorage2.Get("path3/obj3")
+	c.Assert(err, IsNil)
+	object3, _ := ioutil.ReadAll(objectResult3)
+	c.Assert(string(object3), Equals, "object3")
+}