MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-12 15:33:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #106 from harshavardhana/pr_out_ignore_io_eof_for_gob_decoding

Browse Source
This commit is contained in:
Harshavardhana 2014-12-11 01:47:49 -08:00
commit 23a3609a64
3 changed files with 36 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -33,7 +33,6 @@ build-strbyteconv:
@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/strbyteconv
build-storage: build-storage-fs build-storage-append build-storage-encoded
@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/storage
build-storage-fs:
@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/storage/fsstorage

6
pkgs/storage/appendstorage/append_storage.go
View File

@ -24,7 +24,7 @@ type Header struct {
Path string
Offset int64
Length int
Crc []byte
Crc uint32
}
func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
@ -44,7 +44,7 @@ func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
}
dec := gob.NewDecoder(mapFile)
err = dec.Decode(&objects)
if err != nil {
if err != nil && err != io.EOF {
return &appendStorage{}, nil
}
}
@ -81,7 +81,7 @@ func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
Path: objectPath,
Offset: 0,
Length: 0,
Crc: nil,
Crc: 0,
}
offset, err := aStorage.file.Seek(0, os.SEEK_END)
if err != nil {

67
pkgs/storage/encodedstorage/encoded_storage.go
View File

@ -25,10 +25,33 @@ type encodedStorage struct {
diskStorage []storage.ObjectStorage
}
type StorageEntry struct {
Path string
Md5sum string
Crc uint32
Blocks []StorageBlockEntry
Encoderparams *erasure.EncoderParams
}
type StorageBlockEntry struct {
Index int
Length int
}
type storeRequest struct {
path string
data []byte
}
type storeResponse struct {
data []byte
err error
}
func NewStorage(rootDir string, k, m int, blockSize uint64) (storage.ObjectStorage, error) {
// create storage files
storageNodes := make([]storage.ObjectStorage, 16)
for i := 0; i < 16; i++ {
storageNodes := make([]storage.ObjectStorage, k+m)
for i := 0; i < k+m; i++ {
storageNode, err := appendstorage.NewStorage(rootDir, i)
storageNodes[i] = storageNode
if err != nil {
@ -45,7 +68,7 @@ func NewStorage(rootDir string, k, m int, blockSize uint64) (storage.ObjectStora
}
encoder := gob.NewDecoder(indexFile)
err = encoder.Decode(&objects)
if err != nil {
if err != nil && err != io.EOF {
return nil, err
}
}
@ -86,10 +109,11 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
}
encoder := erasure.NewEncoder(encoderParameters)
entry := StorageEntry{
Path: objectPath,
Md5sum: "md5sum",
Crc: 24,
Blocks: make([]StorageBlockEntry, 0),
Path: objectPath,
Md5sum: "md5sum",
Crc: 0,
Blocks: make([]StorageBlockEntry, 0),
Encoderparams: encoderParameters,
}
i := 0
// encode
@ -122,28 +146,6 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
return nil
}
type storeRequest struct {
path string
data []byte
}
type storeResponse struct {
data []byte
err error
}
type StorageEntry struct {
Path string
Md5sum string
Crc uint32
Blocks []StorageBlockEntry
}
type StorageBlockEntry struct {
Index int
Length int
}
func (eStorage *encodedStorage) storeBlocks(path string, blocks [][]byte) []error {
returnChannels := make([]<-chan error, len(eStorage.diskStorage))
for i, store := range eStorage.diskStorage {
@ -161,16 +163,13 @@ func (eStorage *encodedStorage) storeBlocks(path string, blocks [][]byte) []erro
}
func (eStorage *encodedStorage) readObject(objectPath string, entry StorageEntry, writer *io.PipeWriter) {
params, err := erasure.ParseEncoderParams(eStorage.K, eStorage.M, erasure.CAUCHY)
if err != nil {
}
encoder := erasure.NewEncoder(params)
encoder := erasure.NewEncoder(entry.Encoderparams)
for i, chunk := range entry.Blocks {
blockSlices := eStorage.getBlockSlices(objectPath + "$" + strconv.Itoa(i))
var blocks [][]byte
for _, slice := range blockSlices {
if slice.err != nil {
writer.CloseWithError(err)
writer.CloseWithError(slice.err)
return
}
blocks = append(blocks, slice.data)