Storage erasure encoding info at StorageEntry and also store CRC at blockheader

pkgs/erasure/encode.go

@@ -43,9 +43,9 @@ const (
 
 // EncoderParams is a configuration set for building an encoder. It is created using ValidateParams.
 type EncoderParams struct {
-	k,
-	m,
-	technique int // cauchy or vandermonde matrix (RS)
+	K,
+	M,
+	Technique int // cauchy or vandermonde matrix (RS)
 }
 
 // Encoder is an object used to encode and decode data.
@@ -87,21 +87,21 @@ func ParseEncoderParams(k, m, technique int) (*EncoderParams, error) {
 	}
 
 	return &EncoderParams{
-		k:         k,
-		m:         m,
-		technique: technique,
+		K:         k,
+		M:         m,
+		Technique: technique,
 	}, nil
 }
 
 // NewEncoder creates an encoder with a given set of parameters.
 func NewEncoder(ep *EncoderParams) *Encoder {
-	var k = C.int(ep.k)
-	var m = C.int(ep.m)
+	var k = C.int(ep.K)
+	var m = C.int(ep.M)
 
 	var encode_matrix *C.uint8_t
 	var encode_tbls *C.uint8_t
 
-	C.minio_init_encoder(C.int(ep.technique), k, m, &encode_matrix,
+	C.minio_init_encoder(C.int(ep.Technique), k, m, &encode_matrix,
 		&encode_tbls)
 
 	return &Encoder{
@@ -122,7 +122,7 @@ func (e *Encoder) Encode(block []byte) ([][]byte, int) {
 	var block_len = len(block)
 
 	chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(block_len)))
-	chunk_len := chunk_size * e.p.k
+	chunk_len := chunk_size * e.p.K
 	pad_len := chunk_len - block_len
 
 	if pad_len > 0 {
@@ -131,28 +131,28 @@ func (e *Encoder) Encode(block []byte) ([][]byte, int) {
 		block = append(block, s...)
 	}
 
-	coded_len := chunk_size * e.p.m
+	coded_len := chunk_size * e.p.M
 	c := make([]byte, coded_len)
 	block = append(block, c...)
 
 	// Allocate chunks
-	chunks := make([][]byte, e.p.k+e.p.m)
-	pointers := make([]*byte, e.p.k+e.p.m)
+	chunks := make([][]byte, e.p.K+e.p.M)
+	pointers := make([]*byte, e.p.K+e.p.M)
 
 	var i int
 	// Add data blocks to chunks
-	for i = 0; i < e.p.k; i++ {
+	for i = 0; i < e.p.K; i++ {
 		chunks[i] = block[i*chunk_size : (i+1)*chunk_size]
 		pointers[i] = &chunks[i][0]
 	}
 
-	for i = e.p.k; i < (e.p.k + e.p.m); i++ {
+	for i = e.p.K; i < (e.p.K + e.p.M); i++ {
 		chunks[i] = make([]byte, chunk_size)
 		pointers[i] = &chunks[i][0]
 	}
 
-	data := (**C.uint8_t)(unsafe.Pointer(&pointers[:e.p.k][0]))
-	coding := (**C.uint8_t)(unsafe.Pointer(&pointers[e.p.k:][0]))
+	data := (**C.uint8_t)(unsafe.Pointer(&pointers[:e.p.K][0]))
+	coding := (**C.uint8_t)(unsafe.Pointer(&pointers[e.p.K:][0]))
 
 	C.ec_encode_data(C.int(chunk_size), e.k, e.m, e.encode_tbls, data,
 		coding)

pkgs/storage/appendstorage/append_storage.go

@@ -10,6 +10,7 @@ import (
 	"path"
 	"strconv"
 
+	"github.com/minio-io/minio/pkgs/checksum/crc32c"
 	"github.com/minio-io/minio/pkgs/storage"
 )
 
@@ -96,6 +97,10 @@ func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
 	}
 	header.Offset = offset
 	header.Length = len(objectBytes)
+	header.Crc, err = crc32c.Crc32c(objectBytes)
+	if err != nil {
+		return err
+	}
 	aStorage.objects[objectPath] = header
 	var mapBuffer bytes.Buffer
 	encoder := gob.NewEncoder(&mapBuffer)

pkgs/storage/encodedstorage/encoded_storage.go

@@ -28,9 +28,8 @@ type encodedStorage struct {
 type StorageEntry struct {
 	Path          string
 	Md5sum        string
-	Crc           uint32
 	Blocks        []StorageBlockEntry
-	Encoderparams *erasure.EncoderParams
+	Encoderparams erasure.EncoderParams
 }
 
 type StorageBlockEntry struct {
@@ -109,11 +108,14 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
 	}
 	encoder := erasure.NewEncoder(encoderParameters)
 	entry := StorageEntry{
-		Path:          objectPath,
-		Md5sum:        "md5sum",
-		Crc:           0,
-		Blocks:        make([]StorageBlockEntry, 0),
-		Encoderparams: encoderParameters,
+		Path:   objectPath,
+		Md5sum: "md5sum",
+		Blocks: make([]StorageBlockEntry, 0),
+		Encoderparams: erasure.EncoderParams{
+			K:         eStorage.K,
+			M:         eStorage.M,
+			Technique: erasure.CAUCHY,
+		},
 	}
 	i := 0
 	// encode
@@ -163,7 +165,12 @@ func (eStorage *encodedStorage) storeBlocks(path string, blocks [][]byte) []erro
 }
 
 func (eStorage *encodedStorage) readObject(objectPath string, entry StorageEntry, writer *io.PipeWriter) {
-	encoder := erasure.NewEncoder(entry.Encoderparams)
+	ep, err := erasure.ParseEncoderParams(entry.Encoderparams.K, entry.Encoderparams.M, entry.Encoderparams.Technique)
+	if err != nil {
+		writer.CloseWithError(err)
+		return
+	}
+	encoder := erasure.NewEncoder(ep)
 	for i, chunk := range entry.Blocks {
 		blockSlices := eStorage.getBlockSlices(objectPath + "$" + strconv.Itoa(i))
 		var blocks [][]byte