From 3a76229551ad5178bd89abf7092d48d16e14cfea Mon Sep 17 00:00:00 2001 From: "Anand Babu (AB) Periasamy" Date: Sun, 22 Mar 2015 17:17:53 -0700 Subject: [PATCH] encoded size function is now public + some janitor duty --- pkg/encoding/erasure/cauchy_test.go | 8 ++-- pkg/encoding/erasure/erasure_decode.go | 10 ++--- pkg/encoding/erasure/erasure_encode.go | 50 +++++++++++++----------- pkg/encoding/erasure/vandermonde_test.go | 7 ++-- 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/pkg/encoding/erasure/cauchy_test.go b/pkg/encoding/erasure/cauchy_test.go index 0dda870d6..0d1ef38c6 100644 --- a/pkg/encoding/erasure/cauchy_test.go +++ b/pkg/encoding/erasure/cauchy_test.go @@ -30,13 +30,13 @@ var _ = Suite(&MySuite{}) func Test(t *testing.T) { TestingT(t) } func (s *MySuite) TestCauchyDecode(c *C) { - ep, _ := ParseEncoderParams(10, 5, Cauchy) + const k, m = 10, 5 + ep, _ := ParseEncoderParams(k, m, Cauchy) data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") e := NewEncoder(ep) - chunks, length := e.Encode(data) - c.Assert(length, Equals, len(data)) + chunks, _ := e.Encode(data) chunks[0] = nil chunks[3] = nil @@ -44,7 +44,7 @@ func (s *MySuite) TestCauchyDecode(c *C) { chunks[9] = nil chunks[13] = nil - recoveredData, err := e.Decode(chunks, length) + recoveredData, err := e.Decode(chunks, len(data)) c.Assert(err, IsNil) if !bytes.Equal(data, recoveredData) { diff --git a/pkg/encoding/erasure/erasure_decode.go b/pkg/encoding/erasure/erasure_decode.go index 8d79a5109..f12d9fc4e 100644 --- a/pkg/encoding/erasure/erasure_decode.go +++ b/pkg/encoding/erasure/erasure_decode.go @@ -39,10 +39,10 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { k := int(e.k) n := int(e.k + e.m) - if len(chunks) != n { + if len(chunks) != int(n) { return nil, errors.New(fmt.Sprintf("chunks length must be %d", n)) } - chunk_size := getChunkSize(k, length) + chunk_size := GetEncodedChunkLen(length, k) error_index := make([]int, n+1) var err_count int = 0 @@ -58,7 +58,7 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { err_count++ // Too many missing chunks, cannot be more than parity `m` - if err_count-1 > (n - k) { + if err_count-1 > int(n-k) { return nil, errors.New("too many erasures requested, can't decode") } @@ -90,8 +90,8 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls, source, target) - recovered_output := make([]byte, 0, chunk_size*k) - for i := 0; i < k; i++ { + recovered_output := make([]byte, 0, chunk_size*int(k)) + for i := 0; i < int(k); i++ { recovered_output = append(recovered_output, chunks[i]...) } diff --git a/pkg/encoding/erasure/erasure_encode.go b/pkg/encoding/erasure/erasure_encode.go index 1a8301e42..ea1a286db 100644 --- a/pkg/encoding/erasure/erasure_encode.go +++ b/pkg/encoding/erasure/erasure_encode.go @@ -116,38 +116,44 @@ func NewEncoder(ep *EncoderParams) *Encoder { } } -func getChunkSize(k, split_len int) int { - var alignment, remainder, padded_len int +func GetEncodedLen(inputLen, k, m int) (outputLen int) { + outputLen = GetEncodedChunkLen(inputLen, k) * (k + m) + return outputLen +} - alignment = k * SimdAlign - remainder = split_len % alignment +func GetEncodedChunkLen(inputLen, k int) (outputChunkLen int) { + alignment := k * SimdAlign + remainder := inputLen % alignment - padded_len = split_len + paddedInputLen := inputLen if remainder != 0 { - padded_len = split_len + (alignment - remainder) + paddedInputLen = inputLen + (alignment - remainder) } - return padded_len / k + outputChunkLen = paddedInputLen / k + return outputChunkLen } // Encode encodes a block of data. The input is the original data. The output // is a 2 tuple containing (k + m) chunks of erasure encoded data and the // length of the original object. -func (e *Encoder) Encode(block []byte) ([][]byte, int) { - var block_len = len(block) +func (e *Encoder) Encode(input []byte) ([][]byte, error) { + var inputLen = len(input) - chunk_size := getChunkSize(int(e.k), block_len) - chunk_len := chunk_size * int(e.k) - pad_len := int(chunk_len) - block_len + chunkLen := GetEncodedChunkLen(inputLen, int(e.k)) + encodedDataLen := chunkLen * int(e.k) + paddedDataLen := int(encodedDataLen) - inputLen - if pad_len > 0 { - s := make([]byte, pad_len) + if paddedDataLen > 0 { + s := make([]byte, paddedDataLen) // Expand with new padded blocks to the byte array - block = append(block, s...) + input = append(input, s...) } - coded_len := chunk_size * int(e.p.M) - c := make([]byte, coded_len) - block = append(block, c...) + encodedParityLen := chunkLen * int(e.p.M) + c := make([]byte, encodedParityLen) + input = append(input, c...) + + // encodedOutLen := encodedDataLen + encodedParityLen // Allocate chunks chunks := make([][]byte, e.p.K+e.p.M) @@ -156,19 +162,19 @@ func (e *Encoder) Encode(block []byte) ([][]byte, int) { var i int // Add data blocks to chunks for i = 0; i < int(e.p.K); i++ { - chunks[i] = block[i*chunk_size : (i+1)*chunk_size] + chunks[i] = input[i*chunkLen : (i+1)*chunkLen] pointers[i] = &chunks[i][0] } for i = int(e.p.K); i < int(e.p.K+e.p.M); i++ { - chunks[i] = make([]byte, chunk_size) + chunks[i] = make([]byte, chunkLen) pointers[i] = &chunks[i][0] } data := (**C.uint8_t)(unsafe.Pointer(&pointers[:e.p.K][0])) coding := (**C.uint8_t)(unsafe.Pointer(&pointers[e.p.K:][0])) - C.ec_encode_data(C.int(chunk_size), e.k, e.m, e.encode_tbls, data, + C.ec_encode_data(C.int(chunkLen), e.k, e.m, e.encode_tbls, data, coding) - return chunks, block_len + return chunks, nil } diff --git a/pkg/encoding/erasure/vandermonde_test.go b/pkg/encoding/erasure/vandermonde_test.go index 50ea52fed..5ed5ea309 100644 --- a/pkg/encoding/erasure/vandermonde_test.go +++ b/pkg/encoding/erasure/vandermonde_test.go @@ -28,10 +28,9 @@ func (s *MySuite) TestVanderMondeDecode(c *C) { data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") e := NewEncoder(ep) - chunks, length := e.Encode(data) + chunks, err := e.Encode(data) c.Logf("chunks length: %d", len(chunks)) - c.Logf("length: %d", length) - c.Assert(length, Equals, len(data)) + c.Logf("length: %d", len(data)) chunks[0] = nil chunks[3] = nil @@ -39,7 +38,7 @@ func (s *MySuite) TestVanderMondeDecode(c *C) { chunks[9] = nil chunks[13] = nil - recoveredData, err := e.Decode(chunks, length) + recoveredData, err := e.Decode(chunks, len(data)) c.Assert(err, IsNil) if !bytes.Equal(recoveredData, data) {