Minor refactor to prepare for file version of donut

This commit is contained in:
Frederick F. Kautz IV 2015-03-14 11:07:54 -07:00
parent dffee21cde
commit 2f4b92eb8d
5 changed files with 24 additions and 70 deletions

View File

@ -23,34 +23,17 @@ import (
"errors"
"io"
"strconv"
"time"
)
// Package Version
const Version = uint32(1)
// ObjectType is the type of object stored. It is either an Object or Multipart Object.
type ObjectType uint8
const (
// Object is a full object
Object ObjectType = iota
// MultipartObject is a collection of Objects uploaded separately that represent a large object.
MultipartObject
)
// ObjectMetadata contains information necessary to reconstruct the object and basic object metadata.
type ObjectMetadata struct {
Bucket string
Key string
ErasurePart uint16
EncodedPart uint8
ContentType string
Created time.Time
Length uint64
Md5 []byte
ObjectType ObjectType
}
// Write an encoded part to a writer

View File

@ -227,16 +227,16 @@ func (storage *Storage) ListObjects(bucket string, resources mstorage.BucketReso
return results, resources, nil
}
type byBucketName []mstorage.BucketMetadata
type ByBucketName []mstorage.BucketMetadata
// Len of bucket name
func (b byBucketName) Len() int { return len(b) }
func (b ByBucketName) Len() int { return len(b) }
// Swap bucket i, j
func (b byBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Less
func (b byBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
// ListBuckets - List buckets from memory
func (storage *Storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
@ -244,7 +244,7 @@ func (storage *Storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
for _, bucket := range storage.bucketdata {
results = append(results, bucket.metadata)
}
sort.Sort(byBucketName(results))
sort.Sort(ByBucketName(results))
return results, nil
}

View File

@ -18,6 +18,8 @@ package singledisk
import (
"bytes"
"crypto/md5"
"encoding/gob"
"errors"
"github.com/minio-io/minio/pkg/encoding/erasure"
"github.com/minio-io/minio/pkg/storage"
@ -27,17 +29,20 @@ import (
"io"
"os"
"path"
"time"
)
// StorageDriver creates a new single disk storage driver using donut without encoding.
type StorageDriver struct {
root string
donutBox DonutBox
}
// DonutBox is an interface specifying how the storage driver should interact with its underlying system.
type DonutBox interface {
Store(objectv1.ObjectMetadata, erasure1.DataHeader, io.Reader)
Store(objectv1.ObjectMetadata, erasure1.DataHeader, io.Reader) error
Get(bucket, key string, erasurePart uint16, encodedPart uint8) (objectv1.ObjectMetadata, erasure1.DataHeader, io.Reader, error)
ListObjects(bucket string) ([]string, error)
ListBuckets() ([]storage.BucketMetadata, error)
}
// Start a single disk subsystem
@ -45,7 +50,6 @@ func Start(root string, donutBox DonutBox) (chan<- string, <-chan error, storage
ctrlChannel := make(chan string)
errorChannel := make(chan error)
s := new(StorageDriver)
s.root = root
s.donutBox = donutBox
go start(ctrlChannel, errorChannel, s)
return ctrlChannel, errorChannel, s
@ -64,8 +68,7 @@ func (diskStorage StorageDriver) ListBuckets() ([]storage.BucketMetadata, error)
// CreateBucket creates a new bucket
func (diskStorage StorageDriver) CreateBucket(bucket string) error {
bucketPath := path.Join(diskStorage.root, bucket)
return os.MkdirAll(bucketPath, 0600)
return errors.New("Not Implemented")
}
// CreateBucketPolicy sets a bucket's access policy
@ -79,7 +82,7 @@ func (diskStorage StorageDriver) GetBucketPolicy(bucket string) (storage.BucketP
}
// GetObject retrieves an object and writes it to a writer
func (diskStorage StorageDriver) GetObject(w io.Writer, bucket, object string) (int64, error) {
func (diskStorage StorageDriver) GetObject(target io.Writer, bucket, key string) (int64, error) {
return 0, errors.New("Not Implemented")
}
@ -89,8 +92,12 @@ func (diskStorage StorageDriver) GetPartialObject(w io.Writer, bucket, object st
}
// GetObjectMetadata retrieves an object's metadata
func (diskStorage StorageDriver) GetObjectMetadata(bucket, object string, prefix string) (storage.ObjectMetadata, error) {
return storage.ObjectMetadata{}, errors.New("Not Implemented")
func (diskStorage StorageDriver) GetObjectMetadata(bucket, key string, prefix string) (metadata storage.ObjectMetadata, err error) {
return metadata, errors.New("Not Implemented")
}
func readHeaderGob(reader io.Reader) (header ObjectHeader, err error) {
return header, errors.New("Not Implemented")
}
// ListObjects lists objects
@ -100,42 +107,5 @@ func (diskStorage StorageDriver) ListObjects(bucket string, resources storage.Bu
// CreateObject creates a new object
func (diskStorage StorageDriver) CreateObject(bucket string, key string, contentType string, data io.Reader) error {
// test if object exists
// split object into erasure parts
erasureParts := split.Stream(data, 10*1024*1024)
// set up encoder
params, err := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
if err != nil {
return err
}
encoder := erasure.NewEncoder(params)
// for each erasure part
erasurePartIndex := 1
for erasurePart := range erasureParts {
if erasurePart.Err != nil {
return erasurePart.Err
}
// encode each erasure part into encoded parts
encodedParts, length := encoder.Encode(erasurePart.Data)
// for each encoded part
for encodedPartIndex, encodedPart := range encodedParts {
objectMetadata := objectv1.ObjectMetadata{
Bucket: bucket,
Key: key,
ErasurePart: uint16(erasurePartIndex),
EncodedPart: uint8(encodedPartIndex),
ContentType: contentType,
}
erasureMetadata := erasure1.DataHeader{
OriginalLength: uint32(length),
EncoderK: 8,
EncoderM: 8,
EncoderTechnique: erasure1.Cauchy,
}
// store encoded part
diskStorage.donutBox.Store(objectMetadata, erasureMetadata, bytes.NewBuffer(encodedPart))
erasurePartIndex = erasurePartIndex + 1
}
}
return errors.New("Not Implemented")
}

View File

@ -39,7 +39,7 @@ func (s *MySuite) TestAPISuite(c *C) {
path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
c.Check(err, IsNil)
storageList = append(storageList, path)
_, _, store := Start(path, nil) // TODO Make InMemory driver
_, _, store := Start(path, StartDonutBox()) // TODO Make InMemory driver
return store
}
mstorage.APITestSuite(c, create)

View File

@ -66,8 +66,9 @@ func testMultipleObjectCreation(c *check.C, create func() Storage) {
etags := make(map[string]string)
for key, value := range objects {
var byteBuffer bytes.Buffer
storage.GetObject(&byteBuffer, "bucket", key)
c.Assert(bytes.Equal(value, byteBuffer.Bytes()), check.Equals, true)
_, err := storage.GetObject(&byteBuffer, "bucket", key)
c.Assert(err, check.IsNil)
c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
metadata, err := storage.GetObjectMetadata("bucket", key, "")
c.Assert(err, check.IsNil)