Make erasure matrix type not optional choose automatically

Remove option of providing Technique and handling errors based on that
choose a matrix type automatically based on number of data blocks.

INTEL recommends on using cauchy for consistent invertible matrices,
while vandermonde is faster we should default to cauchy for large
data blocks.
This commit is contained in:
Harshavardhana
2015-10-05 22:33:39 -07:00
parent cf0e1a156b
commit d5ce2f6944
13 changed files with 67 additions and 175 deletions

View File

@@ -288,7 +288,6 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, size int64,
objMetadata.ChunkCount = chunkCount
objMetadata.DataDisks = k
objMetadata.ParityDisks = m
objMetadata.ErasureTechnique = "Cauchy"
objMetadata.Size = int64(totalLength)
}
objMetadata.Bucket = b.getBucketName()
@@ -431,7 +430,7 @@ func (b bucket) getDataAndParity(totalWriters int) (k uint8, m uint8, err *probe
// writeObjectData -
func (b bucket) writeObjectData(k, m uint8, writers []io.WriteCloser, objectData io.Reader, size int64, hashWriter io.Writer) (int, int, *probe.Error) {
encoder, err := newEncoder(k, m, "Cauchy")
encoder, err := newEncoder(k, m)
if err != nil {
return 0, 0, err.Trace()
}
@@ -503,11 +502,7 @@ func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMeta
mwriter := io.MultiWriter(writer, hasher, sum512hasher)
switch len(readers) > 1 {
case true:
if objMetadata.ErasureTechnique == "" {
writer.CloseWithError(probe.WrapError(probe.NewError(MissingErasureTechnique{})))
return
}
encoder, err := newEncoder(objMetadata.DataDisks, objMetadata.ParityDisks, objMetadata.ErasureTechnique)
encoder, err := newEncoder(objMetadata.DataDisks, objMetadata.ParityDisks)
if err != nil {
writer.CloseWithError(probe.WrapError(err))
return

View File

@@ -30,11 +30,10 @@ type ObjectMetadata struct {
Size int64 `json:"size"`
// erasure
DataDisks uint8 `json:"sys.erasureK"`
ParityDisks uint8 `json:"sys.erasureM"`
ErasureTechnique string `json:"sys.erasureTechnique"`
BlockSize int `json:"sys.blockSize"`
ChunkCount int `json:"sys.chunkCount"`
DataDisks uint8 `json:"sys.erasureK"`
ParityDisks uint8 `json:"sys.erasureM"`
BlockSize int `json:"sys.blockSize"`
ChunkCount int `json:"sys.chunkCount"`
// checksums
MD5Sum string `json:"sys.md5sum"`

View File

@@ -23,41 +23,21 @@ import (
// encoder internal struct
type encoder struct {
encoder *encoding.Erasure
k, m uint8
technique encoding.Technique
}
// getErasureTechnique - convert technique string into Technique type
func getErasureTechnique(technique string) (encoding.Technique, *probe.Error) {
switch true {
case technique == "Cauchy":
return encoding.Cauchy, nil
case technique == "Vandermonde":
return encoding.Cauchy, nil
default:
return encoding.None, probe.NewError(InvalidErasureTechnique{Technique: technique})
}
encoder *encoding.Erasure
k, m uint8
}
// newEncoder - instantiate a new encoder
func newEncoder(k, m uint8, technique string) (encoder, *probe.Error) {
func newEncoder(k, m uint8) (encoder, *probe.Error) {
e := encoder{}
t, err := getErasureTechnique(technique)
params, err := encoding.ValidateParams(k, m)
if err != nil {
return encoder{}, err.Trace()
}
{
params, err := encoding.ValidateParams(k, m, t)
if err != nil {
return encoder{}, probe.NewError(err)
}
e.encoder = encoding.NewErasure(params)
e.k = k
e.m = m
e.technique = t
return e, nil
return encoder{}, probe.NewError(err)
}
e.encoder = encoding.NewErasure(params)
e.k = k
e.m = m
return e, nil
}
// TODO - think again if this is needed

View File

@@ -132,22 +132,6 @@ func (e MissingPOSTPolicy) Error() string {
return "Missing POST policy in multipart form"
}
// MissingErasureTechnique missing erasure technique
type MissingErasureTechnique struct{}
func (e MissingErasureTechnique) Error() string {
return "Missing erasure technique"
}
// InvalidErasureTechnique invalid erasure technique
type InvalidErasureTechnique struct {
Technique string
}
func (e InvalidErasureTechnique) Error() string {
return "Invalid erasure technique: " + e.Technique
}
// InternalError - generic internal error
type InternalError struct {
}