Merge pull request #409 from harshavardhana/pr_out_more_simplification_to_erasure_also_add_two_more_test_functions

pkg/encoding/erasure/cauchy_test.go

@@ -34,19 +34,33 @@ const (
 	m = 5
 )
 
-func (s *MySuite) TestCauchyDecode(c *C) {
-	ep, _ := ParseEncoderParams(k, m, Cauchy)
+func (s *MySuite) TestCauchyEncodeDecodeFailure(c *C) {
+	ep, _ := ValidateParams(k, m, Cauchy)
 
 	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
 
-	e := NewEncoder(ep)
-	chunks, _ := e.Encode(data)
+	e := NewErasure(ep)
+	chunks, err := e.Encode(data)
+	c.Assert(err, IsNil)
 
-	chunks[0] = nil
-	chunks[3] = nil
-	chunks[5] = nil
-	chunks[9] = nil
-	chunks[13] = nil
+	errorIndex := []int{0, 3, 5, 9, 11, 13}
+	chunks = corruptChunks(chunks, errorIndex)
+
+	_, err = e.Decode(chunks, len(data))
+	c.Assert(err, Not(IsNil))
+}
+
+func (s *MySuite) TestCauchyEncodeDecodeSuccess(c *C) {
+	ep, _ := ValidateParams(k, m, Cauchy)
+
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+
+	e := NewErasure(ep)
+	chunks, err := e.Encode(data)
+	c.Assert(err, IsNil)
+
+	errorIndex := []int{0, 3, 5, 9, 13}
+	chunks = corruptChunks(chunks, errorIndex)
 
 	recoveredData, err := e.Decode(chunks, len(data))
 	c.Assert(err, IsNil)

pkg/encoding/erasure/doc.go

@@ -28,8 +28,8 @@
 //  2. Create a new encoder
 //  3. Decode data
 //
-// Encoder parameters contain three configurable elements:
-//  ParseEncoderParams(k, m, technique int) (EncoderParams, error)
+// Erasure parameters contain three configurable elements:
+//  ValidateParams(k, m, technique int) (ErasureParams, error)
 //  k - Number of rows in matrix
 //  m - Number of colums in matrix
 //  technique - Matrix type, can be either Cauchy (recommended) or Vandermonde
@@ -53,15 +53,15 @@
 //
 // Creating and using an encoder
 //  var bytes []byte
-//  params := erasure.ParseEncoderParams(10, 5, erasure.Cauchy)
-//  encoder := erasure.NewEncoder(params)
+//  params := erasure.ValidateParams(10, 5, erasure.Cauchy)
+//  encoder := erasure.NewErasure(params)
 //  encodedData, length := encoder.Encode(bytes)
 //
 // Creating and using a decoder
 //  var encodedData [][]byte
 //  var length int
-//  params := erasure.ParseEncoderParams(10, 5, erasure.Cauchy)
-//  encoder := erasure.NewEncoder(params)
+//  params := erasure.ValidateParams(10, 5, erasure.Cauchy)
+//  encoder := erasure.NewErasure(params)
 //  originalData, err := encoder.Decode(encodedData, length)
 //
 package erasure

pkg/encoding/erasure/erasure_decode.go

@@ -36,7 +36,7 @@ import (
 //    are set to "nil". There must be at least "K" number of data|parity
 //    blocks.
 // "dataLen" is the length of original source data
-func (e *Encoder) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) {
+func (e *Erasure) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) {
 	var source, target **C.uint8_t
 
 	k := int(e.params.K)

pkg/encoding/erasure/erasure_encode.go

@@ -47,27 +47,27 @@ const (
 	SIMDAlign = 32
 )
 
-// EncoderParams is a configuration set for building an encoder. It is created using ValidateParams.
-type EncoderParams struct {
+// ErasureParams is a configuration set for building an encoder. It is created using ValidateParams().
+type ErasureParams struct {
 	K         uint8
 	M         uint8
 	Technique Technique // cauchy or vandermonde matrix (RS)
 }
 
-// Encoder is an object used to encode and decode data.
-type Encoder struct {
-	params                   *EncoderParams
+// Erasure is an object used to encode and decode data.
+type Erasure struct {
+	params                   *ErasureParams
 	encodeMatrix, encodeTbls *C.uint8_t
 	decodeMatrix, decodeTbls *C.uint8_t
 	decodeIndex              *C.uint32_t
 }
 
-// ParseEncoderParams creates an EncoderParams object.
+// ValidateParams creates an ErasureParams object.
 //
 // k and m represent the matrix size, which corresponds to the protection level
 // technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde.
 //
-func ParseEncoderParams(k, m uint8, technique Technique) (*EncoderParams, error) {
+func ValidateParams(k, m uint8, technique Technique) (*ErasureParams, error) {
 	if k < 1 {
 		return nil, errors.New("k cannot be zero")
 	}
@@ -89,15 +89,15 @@ func ParseEncoderParams(k, m uint8, technique Technique) (*EncoderParams, error)
 		return nil, errors.New("Technique can be either vandermonde or cauchy")
 	}
 
-	return &EncoderParams{
+	return &ErasureParams{
 		K:         k,
 		M:         m,
 		Technique: technique,
 	}, nil
 }
 
-// NewEncoder creates an encoder object with a given set of parameters.
-func NewEncoder(ep *EncoderParams) *Encoder {
+// NewErasure creates an encoder object with a given set of parameters.
+func NewErasure(ep *ErasureParams) *Erasure {
 	var k = C.int(ep.K)
 	var m = C.int(ep.M)
 
@@ -107,7 +107,7 @@ func NewEncoder(ep *EncoderParams) *Encoder {
 	C.minio_init_encoder(C.int(ep.Technique), k, m, &encodeMatrix,
 		&encodeTbls)
 
-	return &Encoder{
+	return &Erasure{
 		params:       ep,
 		encodeMatrix: encodeMatrix,
 		encodeTbls:   encodeTbls,
@@ -138,7 +138,7 @@ func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) {
 
 // Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
 // Output is [k+m][]blocks of data and parity slices.
-func (e *Encoder) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
+func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
 	k := int(e.params.K) // "k" data blocks
 	m := int(e.params.M) // "m" parity blocks
 	n := k + m           // "n" total encoded blocks

pkg/encoding/erasure/vandermonde_test.go

@@ -22,21 +22,40 @@ import (
 	. "gopkg.in/check.v1"
 )
 
-func (s *MySuite) TestVanderMondeDecode(c *C) {
-	ep, _ := ParseEncoderParams(k, m, Vandermonde)
+func corruptChunks(chunks [][]byte, errorIndex []int) [][]byte {
+	for _, err := range errorIndex {
+		chunks[err] = nil
+	}
+	return chunks
+}
+
+func (s *MySuite) TestVanderMondeEncodeDecodeFailure(c *C) {
+	ep, _ := ValidateParams(k, m, Vandermonde)
 
 	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
 
-	e := NewEncoder(ep)
+	e := NewErasure(ep)
 	chunks, err := e.Encode(data)
-	c.Logf("chunks length: %d", len(chunks))
-	c.Logf("length: %d", len(data))
+	c.Assert(err, IsNil)
 
-	chunks[0] = nil
-	chunks[3] = nil
-	chunks[5] = nil
-	chunks[9] = nil
-	chunks[13] = nil
+	errorIndex := []int{0, 3, 5, 9, 11, 13}
+	chunks = corruptChunks(chunks, errorIndex)
+
+	_, err = e.Decode(chunks, len(data))
+	c.Assert(err, Not(IsNil))
+}
+
+func (s *MySuite) TestVanderMondeEncodeDecodeSuccess(c *C) {
+	ep, _ := ValidateParams(k, m, Vandermonde)
+
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+
+	e := NewErasure(ep)
+	chunks, err := e.Encode(data)
+	c.Assert(err, IsNil)
+
+	errorIndex := []int{0, 3, 5, 9, 13}
+	chunks = corruptChunks(chunks, errorIndex)
 
 	recoveredData, err := e.Decode(chunks, len(data))
 	c.Assert(err, IsNil)

pkg/storage/donut/erasure.go

@@ -3,6 +3,7 @@ package donut
 import (
 	"bytes"
 	"errors"
+	"hash"
 	"io"
 	"strconv"
 	"strings"
@@ -12,21 +13,19 @@ import (
 	"encoding/hex"
 
 	"github.com/minio-io/iodine"
-	"github.com/minio-io/minio/pkg/encoding/erasure"
+	encoding "github.com/minio-io/minio/pkg/encoding/erasure"
 	"github.com/minio-io/minio/pkg/utils/split"
-	"hash"
-	"log"
 )
 
 // getErasureTechnique - convert technique string into Technique type
-func getErasureTechnique(technique string) (erasure.Technique, error) {
+func getErasureTechnique(technique string) (encoding.Technique, error) {
 	switch true {
 	case technique == "Cauchy":
-		return erasure.Cauchy, nil
+		return encoding.Cauchy, nil
 	case technique == "Vandermonde":
-		return erasure.Cauchy, nil
+		return encoding.Cauchy, nil
 	default:
-		return erasure.None, iodine.New(errors.New("Invalid erasure technique: "+technique), nil)
+		return encoding.None, iodine.New(errors.New("Invalid erasure technique: "+technique), nil)
 	}
 }
 
@@ -70,13 +69,12 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
 		return
 	}
 	hasher := md5.New()
-	params, err := erasure.ParseEncoderParams(k, m, technique)
+	params, err := encoding.ValidateParams(k, m, technique)
 	if err != nil {
 		writer.CloseWithError(iodine.New(err, donutMetadata))
 	}
-	encoder := erasure.NewEncoder(params)
+	encoder := encoding.NewErasure(params)
 	for i := 0; i < totalChunks; i++ {
-		log.Println(i)
 		totalLeft, err = decodeChunk(writer, readers, encoder, hasher, k, m, totalLeft, blockSize)
 		if err != nil {
 			errParams := map[string]string{
@@ -97,7 +95,7 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
 	return
 }
 
-func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *erasure.Encoder, hasher hash.Hash, k, m uint8, totalLeft int64, blockSize int) (int64, error) {
+func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *encoding.Erasure, hasher hash.Hash, k, m uint8, totalLeft int64, blockSize int) (int64, error) {
 	curBlockSize := 0
 	if int64(blockSize) < totalLeft {
 		curBlockSize = blockSize
@@ -105,7 +103,7 @@ func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *erasur
 		curBlockSize = int(totalLeft) // cast is safe, blockSize in if protects
 	}
 
-	curChunkSize := erasure.GetEncodedBlockLen(curBlockSize, uint8(k))
+	curChunkSize := encoding.GetEncodedBlockLen(curBlockSize, uint8(k))
 	encodedBytes := make([][]byte, 16)
 	for i, reader := range readers {
 		var bytesBuffer bytes.Buffer
@@ -167,8 +165,8 @@ func newErasureWriter(writers []Writer) ObjectWriter {
 
 func erasureGoroutine(r *io.PipeReader, eWriter erasureWriter, isClosed chan<- bool) {
 	chunks := split.Stream(r, 10*1024*1024)
-	params, _ := erasure.ParseEncoderParams(8, 8, erasure.Cauchy)
-	encoder := erasure.NewEncoder(params)
+	params, _ := encoding.ValidateParams(8, 8, encoding.Cauchy)
+	encoder := encoding.NewErasure(params)
 	chunkCount := 0
 	totalLength := 0
 	summer := md5.New()