Merge pull request #395 from fkautz/pr_out_refactoring_erasure_each_block_is_encoded_by_a_function_iodine_included

pkg/storage/donut/erasure.go

@@ -11,8 +11,10 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 
+	"github.com/minio-io/iodine"
 	"github.com/minio-io/minio/pkg/encoding/erasure"
 	"github.com/minio-io/minio/pkg/utils/split"
+	"hash"
 )
 
 // getErasureTechnique - convert technique string into Technique type
@@ -23,7 +25,7 @@ func getErasureTechnique(technique string) (erasure.Technique, error) {
 	case technique == "Vandermonde":
 		return erasure.Cauchy, nil
 	default:
-		return erasure.None, errors.New("Invalid erasure technique")
+		return erasure.None, iodine.Error(errors.New("Invalid erasure technique: "+technique), nil)
 	}
 }
 
@@ -31,46 +33,74 @@ func getErasureTechnique(technique string) (erasure.Technique, error) {
 func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
 	totalChunks, err := strconv.Atoi(donutMetadata["chunkCount"])
 	if err != nil {
-		writer.CloseWithError(err)
+		writer.CloseWithError(iodine.Error(err, donutMetadata))
 		return
 	}
-	totalLeft, err := strconv.Atoi(donutMetadata["size"])
+	totalLeft, err := strconv.ParseInt(donutMetadata["size"], 10, 64)
 	if err != nil {
-		writer.CloseWithError(err)
+		writer.CloseWithError(iodine.Error(err, donutMetadata))
 		return
 	}
 	blockSize, err := strconv.Atoi(donutMetadata["blockSize"])
 	if err != nil {
-		writer.CloseWithError(err)
+		writer.CloseWithError(iodine.Error(err, donutMetadata))
 		return
 	}
-	k, err := strconv.Atoi(donutMetadata["erasureK"])
+	parsedk, err := strconv.ParseUint(donutMetadata["erasureK"], 10, 8)
 	if err != nil {
-		writer.CloseWithError(err)
+		writer.CloseWithError(iodine.Error(err, donutMetadata))
 		return
 	}
-	m, err := strconv.Atoi(donutMetadata["erasureM"])
+	k := uint8(parsedk)
+	parsedm, err := strconv.ParseUint(donutMetadata["erasureM"], 10, 8)
 	if err != nil {
-		writer.CloseWithError(err)
+		writer.CloseWithError(iodine.Error(err, donutMetadata))
 		return
 	}
+	m := uint8(parsedm)
 	expectedMd5sum, err := hex.DecodeString(donutMetadata["md5"])
 	if err != nil {
-		writer.CloseWithError(err)
+		writer.CloseWithError(iodine.Error(err, donutMetadata))
 		return
 	}
 	technique, err := getErasureTechnique(donutMetadata["erasureTechnique"])
 	if err != nil {
-		writer.CloseWithError(err)
+		writer.CloseWithError(iodine.Error(err, donutMetadata))
 		return
 	}
-	summer := md5.New()
+	hasher := md5.New()
-	params, _ := erasure.ParseEncoderParams(uint8(k), uint8(m), technique)
+	params, err := erasure.ParseEncoderParams(k, m, technique)
+	if err != nil {
+		writer.CloseWithError(iodine.Error(err, donutMetadata))
+	}
 	encoder := erasure.NewEncoder(params)
 	for i := 0; i < totalChunks; i++ {
-		curBlockSize := totalLeft
+		totalLeft, err = decodeChunk(writer, readers, encoder, hasher, k, m, totalLeft, blockSize)
-		if blockSize < totalLeft {
+		if err != nil {
+			errParams := map[string]string{
+				"totalLeft": strconv.FormatInt(totalLeft, 10),
+			}
+			for k, v := range donutMetadata {
+				errParams[k] = v
+			}
+			writer.CloseWithError(iodine.Error(err, errParams))
+		}
+	}
+	actualMd5sum := hasher.Sum(nil)
+	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
+		writer.CloseWithError(iodine.Error(errors.New("decoded md5sum did not match. expected: "+string(expectedMd5sum)+" actual: "+string(actualMd5sum)), donutMetadata))
+		return
+	}
+	writer.Close()
+	return
+}
+
+func decodeChunk(writer *io.PipeWriter, readers []io.ReadCloser, encoder *erasure.Encoder, hasher hash.Hash, k, m uint8, totalLeft int64, blockSize int) (int64, error) {
+	curBlockSize := 0
+	if int64(blockSize) < totalLeft {
 		curBlockSize = blockSize
+	} else {
+		curBlockSize = int(totalLeft) // cast is safe, blockSize in if protects
 	}
 
 	curChunkSize := erasure.GetEncodedBlockLen(curBlockSize, uint8(k))
@@ -78,33 +108,36 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
 	for i, reader := range readers {
 		defer reader.Close()
 		var bytesBuffer bytes.Buffer
-			_, err := io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
+		written, err := io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
 		if err != nil {
-				writer.CloseWithError(err)
+			errParams := map[string]string{}
-				return
+			errParams["part"] = strconv.FormatInt(written, 10)
+			errParams["block.written"] = strconv.FormatInt(written, 10)
+			errParams["block.length"] = strconv.Itoa(curChunkSize)
+			return totalLeft, iodine.Error(err, errParams)
 		}
 		encodedBytes[i] = bytesBuffer.Bytes()
 	}
 	decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
 	if err != nil {
-			writer.CloseWithError(err)
+		errParams := map[string]string{}
-			return
+		errParams["block.length"] = strconv.Itoa(curChunkSize)
+		return totalLeft, iodine.Error(err, errParams)
+	}
+	_, err = hasher.Write(decodedData) // not expecting errors from hash, will also catch further down on .Sum mismatch in parent
+	if err != nil {
+		errParams := map[string]string{}
+		errParams["block.length"] = strconv.Itoa(curChunkSize)
+		return totalLeft, iodine.Error(err, errParams)
 	}
-		summer.Write(decodedData)
 	_, err = io.Copy(writer, bytes.NewBuffer(decodedData))
 	if err != nil {
-			writer.CloseWithError(err)
+		errParams := map[string]string{}
-			return
+		errParams["block.length"] = strconv.Itoa(curChunkSize)
+		return totalLeft, iodine.Error(err, errParams)
 	}
-		totalLeft = totalLeft - blockSize
+	totalLeft = totalLeft - int64(blockSize)
-	}
+	return totalLeft, nil
-	actualMd5sum := summer.Sum(nil)
-	if bytes.Compare(expectedMd5sum, actualMd5sum) != 0 {
-		writer.CloseWithError(errors.New("decoded md5sum did not match"))
-		return
-	}
-	writer.Close()
-	return
 }
 
 // erasure writer

pkg/storage/donut/object_writer.go

@@ -31,8 +31,7 @@ type donutObjectWriter struct {
 
 func (d donutObjectWriter) Write(data []byte) (int, error) {
 	written, err := d.file.Write(data)
-	iodine.Error(err, nil)
+	return written, iodine.Error(err, nil)
-	return written, err
 }
 
 func (d donutObjectWriter) Close() error {