MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #71 - some crazy races inside erasure and simplify the code

Browse Source 
- This change also brings in changing 'unsigned char' to 'uint8_t'
for brevity
This commit is contained in:
Harshavardhana 2014-12-07 00:09:24 -08:00
parent d57700922d
commit 7314b5e37d
18 changed files with 205 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -12,8 +12,7 @@ getdeps: checkdeps
build-erasure: build-erasure:
@$(MAKE) $(MAKE_OPTIONS) -C pkgs/erasure/isal lib @$(MAKE) $(MAKE_OPTIONS) -C pkgs/erasure/isal lib
@godep go test -race github.com/minio-io/minio/pkgs/erasure @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/erasure
@godep go test -coverprofile=cover.out github.com/minio-io/minio/pkgs/erasure
build-signify: build-signify:
@$(MAKE) $(MAKE_OPTIONS) -C pkgs/signify @$(MAKE) $(MAKE_OPTIONS) -C pkgs/signify

6
cmd/erasure-demo/decode.go
View File

@ -76,11 +76,15 @@ func decode(c *cli.Context) {
// set up encoder // set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY) erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
// Get decoder
encoder := erasure.NewEncoder(erasureParameters)
// decode data // decode data
decodedData, err := erasure.Decode(chunks, erasureParameters, length) decodedData, err := encoder.Decode(chunks, length)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
// append decoded data // append decoded data
length, err = outputFile.Write(decodedData) length, err = outputFile.Write(decodedData)
if err != nil { if err != nil {

8
cmd/erasure-demo/encode.go
View File

@ -39,9 +39,13 @@ func encode(c *cli.Context) {
// set up encoder // set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY) erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
// Init new encoder
encoder := erasure.NewEncoder(erasureParameters)
// encode data // encode data
if config.blockSize == 0 { if config.blockSize == 0 {
encodedData, length := erasure.Encode(input, erasureParameters) encodedData, length := encoder.Encode(input)
for key, data := range encodedData { for key, data := range encodedData {
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600) ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600) ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
@ -55,7 +59,7 @@ func encode(c *cli.Context) {
if chunk.Err != nil { if chunk.Err != nil {
log.Fatal(chunk.Err) log.Fatal(chunk.Err)
} }
encodedData, length := erasure.Encode(chunk.Data, erasureParameters) encodedData, length := encoder.Encode(chunk.Data)
for key, data := range encodedData { for key, data := range encodedData {
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600) ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600) ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)

1
pkgs/erasure/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
erasure.test

9
pkgs/erasure/Makefile
View File

@ -1,13 +1,8 @@
all: build test all: build test
.PHONY: all .PHONY: all
test: test-race test-cover test:
@godep go test -race -coverprofile=cover.out
test-race:
@godep go test -race
test-cover:
@godep go test -coverprofile=cover.out
isal/isal-l.a: isal/isal-l.a:
@$(MAKE) -C isal lib @$(MAKE) -C isal lib

16
pkgs/erasure/cauchy_test.go
View File

@ -28,23 +28,13 @@ var _ = Suite(&MySuite{})
func Test(t *testing.T) { TestingT(t) } func Test(t *testing.T) { TestingT(t) }
func (s *MySuite) TestCauchyEncode(c *C) {
ep, _ := ParseEncoderParams(10, 5, CAUCHY)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
chunks, length := Encode(data, ep)
c.Logf("chunks length: %d;\nlength: %d\n", len(chunks), length)
c.Assert(length, Equals, len(data))
}
func (s *MySuite) TestCauchyDecode(c *C) { func (s *MySuite) TestCauchyDecode(c *C) {
ep, _ := ParseEncoderParams(10, 5, CAUCHY) ep, _ := ParseEncoderParams(10, 5, CAUCHY)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
chunks, length := Encode(data, ep) e := NewEncoder(ep)
chunks, length := e.Encode(data)
c.Assert(length, Equals, len(data)) c.Assert(length, Equals, len(data))
chunks[0] = nil chunks[0] = nil
@ -53,7 +43,7 @@ func (s *MySuite) TestCauchyDecode(c *C) {
chunks[9] = nil chunks[9] = nil
chunks[13] = nil chunks[13] = nil
recovered_data, err := Decode(chunks, ep, length) recovered_data, err := e.Decode(chunks, length)
c.Assert(err, IsNil) c.Assert(err, IsNil)
if !bytes.Equal(data, recovered_data) { if !bytes.Equal(data, recovered_data) {

32
pkgs/erasure/common.h
View File

@ -22,24 +22,22 @@
#define SIMD_ALIGN 32 #define SIMD_ALIGN 32
int32_t minio_init_encoder (int technique, int k, int m, int32_t minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix, uint8_t **encode_matrix,
unsigned char **encode_tbls); uint8_t **encode_tbls);
uint32_t minio_calc_chunk_size (int k, uint32_t minio_calc_chunk_size (int k, uint32_t split_len);
uint32_t split_len);
int32_t minio_init_decoder (int *src_err_list, int32_t minio_init_decoder (int32_t *error_index,
unsigned char *encoding_matrix, int k, int n, int errs,
unsigned char **decode_matrix, uint8_t *encoding_matrix,
unsigned char **decode_tbls, uint8_t **decode_matrix,
int k, int n, int errs); uint8_t **decode_tbls,
uint32_t **decode_index);
int32_t minio_src_in_err (int r, int *src_err_list); int32_t minio_get_source_target (int errs, int k, int m,
int32_t *error_index,
int32_t minio_get_source_target(int *src_err_list, uint32_t *decode_index,
int errs, int k, int m, uint8_t **buffs,
unsigned char **data, uint8_t ***source,
unsigned char **coding, uint8_t ***target);
unsigned char ***source,
unsigned char ***target);
#endif /* __COMMON_H__ */ #endif /* __COMMON_H__ */

12
pkgs/erasure/ctypes.go
View File

@ -23,35 +23,35 @@ import (
) )
// Integer to Int conversion // Integer to Int conversion
func int2cInt(src_err_list []int) *C.int { func int2cInt(src_err_list []int) *C.int32_t {
var sizeErrInt = int(unsafe.Sizeof(src_err_list[0])) var sizeErrInt = int(unsafe.Sizeof(src_err_list[0]))
switch sizeInt { switch sizeInt {
case sizeErrInt: case sizeErrInt:
return (*C.int)(unsafe.Pointer(&src_err_list[0])) return (*C.int32_t)(unsafe.Pointer(&src_err_list[0]))
case sizeInt8: case sizeInt8:
int8Array := make([]int8, len(src_err_list)) int8Array := make([]int8, len(src_err_list))
for i, v := range src_err_list { for i, v := range src_err_list {
int8Array[i] = int8(v) int8Array[i] = int8(v)
} }
return (*C.int)(unsafe.Pointer(&int8Array[0])) return (*C.int32_t)(unsafe.Pointer(&int8Array[0]))
case sizeInt16: case sizeInt16:
int16Array := make([]int16, len(src_err_list)) int16Array := make([]int16, len(src_err_list))
for i, v := range src_err_list { for i, v := range src_err_list {
int16Array[i] = int16(v) int16Array[i] = int16(v)
} }
return (*C.int)(unsafe.Pointer(&int16Array[0])) return (*C.int32_t)(unsafe.Pointer(&int16Array[0]))
case sizeInt32: case sizeInt32:
int32Array := make([]int32, len(src_err_list)) int32Array := make([]int32, len(src_err_list))
for i, v := range src_err_list { for i, v := range src_err_list {
int32Array[i] = int32(v) int32Array[i] = int32(v)
} }
return (*C.int)(unsafe.Pointer(&int32Array[0])) return (*C.int32_t)(unsafe.Pointer(&int32Array[0]))
case sizeInt64: case sizeInt64:
int64Array := make([]int64, len(src_err_list)) int64Array := make([]int64, len(src_err_list))
for i, v := range src_err_list { for i, v := range src_err_list {
int64Array[i] = int64(v) int64Array[i] = int64(v)
} }
return (*C.int)(unsafe.Pointer(&int64Array[0])) return (*C.int32_t)(unsafe.Pointer(&int64Array[0]))
default: default:
panic(fmt.Sprintf("Unsupported: %d", sizeInt)) panic(fmt.Sprintf("Unsupported: %d", sizeInt))
} }

104
pkgs/erasure/decode.c
View File

@ -21,11 +21,12 @@
#include <erasure-code.h> #include <erasure-code.h>
#include "common.h" #include "common.h"
int32_t minio_src_in_err (int r, int *src_err_list) static
int32_t _minio_src_index_in_error (int r, int32_t *error_index)
{ {
int i; int i;
for (i = 0; src_err_list[i] != -1; i++) { for (i = 0; error_index[i] != -1; i++) {
if (src_err_list[i] == r) { if (error_index[i] == r) {
// true // true
return 1; return 1;
} }
@ -34,47 +35,29 @@ int32_t minio_src_in_err (int r, int *src_err_list)
return 0; return 0;
} }
int32_t minio_get_source_target(int *src_err_list, int32_t minio_get_source_target (int errs, int k, int m,
int errs, int k, int m, int32_t *error_index,
unsigned char **data, uint32_t *decode_index,
unsigned char **coding, uint8_t **buffs,
unsigned char ***source, uint8_t ***source,
unsigned char ***target) uint8_t ***target)
{ {
int i, j, l; int i;
unsigned char *tmp_source[k]; uint8_t *tmp_source[k];
unsigned char *tmp_target[m]; uint8_t *tmp_target[m];
// Fill zeroes memset (tmp_source, 0, k);
memset (tmp_source, 0, sizeof(tmp_source)); memset (tmp_target, 0, m);
memset (tmp_target, 0, sizeof(tmp_target));
// Separate out source and target buffers from input data/coding chunks for (i = 0; i < k; i++) {
// This separation needs to happen at error chunks from input chunks tmp_source[i] = (uint8_t *) buffs[decode_index[i]];
for (i = 0, j = 0, l = 0;
((l < k) || (j < errs)) && (i < (k + m)); i++) {
if (!minio_src_in_err(i, src_err_list)) {
if (l < k) {
if (i < k)
tmp_source[l] =
(unsigned char *) data[i];
else
tmp_source[l] =
(unsigned char *) coding[i - k];
l++;
}
} else {
if (j < m) {
if (i < k)
tmp_target[j] =
(unsigned char *) data[i];
else
tmp_target[j] =
(unsigned char *) coding[i - k];
j++;
}
}
} }
for (i = 0; i < m; i++) {
if (i < errs)
tmp_target[i] = (uint8_t *) buffs[error_index[i]];
}
*source = tmp_source; *source = tmp_source;
*target = tmp_target; *target = tmp_target;
} }
@ -83,32 +66,27 @@ int32_t minio_get_source_target(int *src_err_list,
Generate decode matrix during the decoding phase Generate decode matrix during the decoding phase
*/ */
int minio_init_decoder (int *src_err_list, int minio_init_decoder (int32_t *error_index,
unsigned char *encode_matrix, int k, int n, int errs,
unsigned char **decode_matrix, uint8_t *encode_matrix,
unsigned char **decode_tbls, uint8_t **decode_matrix,
int k, int n, int errs) uint8_t **decode_tbls,
uint32_t **decode_index)
{ {
int i, j, r, s, l, z; int i, j, r, s, l, z;
unsigned char input_matrix[k * n]; uint8_t input_matrix[k * n];
unsigned char inverse_matrix[k * n]; uint8_t inverse_matrix[k * n];
unsigned char *tmp_decode_matrix; uint8_t tmp_decode_matrix[k * n];
unsigned char *tmp_decode_tbls; uint8_t tmp_decode_tbls[k * n * 32];
uint32_t tmp_decode_index[k];
tmp_decode_matrix = (unsigned char *) malloc (k * n);
if (!tmp_decode_matrix)
return -1;
tmp_decode_tbls = (unsigned char *) malloc (k * n * 32);
if (!tmp_decode_tbls)
return -1;
for (i = 0, r = 0; i < k; i++, r++) { for (i = 0, r = 0; i < k; i++, r++) {
while (minio_src_in_err(r, src_err_list)) while (_minio_src_index_in_error(r, error_index))
r++; r++;
for (j = 0; j < k; j++) { for (j = 0; j < k; j++) {
input_matrix[k * i + j] = encode_matrix[k * r + j]; input_matrix[k * i + j] = encode_matrix[k * r + j];
} }
tmp_decode_index[i] = r;
} }
// Not all Vandermonde matrix can be inverted // Not all Vandermonde matrix can be inverted
@ -117,12 +95,12 @@ int minio_init_decoder (int *src_err_list,
} }
for (l = 0; l < errs; l++) { for (l = 0; l < errs; l++) {
if (src_err_list[l] < k) { if (error_index[l] < k) {
// decoding matrix elements for data chunks // decoding matrix elements for data chunks
for (j = 0; j < k; j++) { for (j = 0; j < k; j++) {
tmp_decode_matrix[k * l + j] = tmp_decode_matrix[k * l + j] =
inverse_matrix[k * inverse_matrix[k *
src_err_list[l] + j]; error_index[l] + j];
} }
} else { } else {
int s = 0; int s = 0;
@ -132,16 +110,18 @@ int minio_init_decoder (int *src_err_list,
for (j = 0; j < k; j++) { for (j = 0; j < k; j++) {
s ^= gf_mul(inverse_matrix[j * k + i], s ^= gf_mul(inverse_matrix[j * k + i],
encode_matrix[k * encode_matrix[k *
src_err_list[l] + j]); error_index[l] + j]);
} }
tmp_decode_matrix[k * l + i] = s; tmp_decode_matrix[k * l + i] = s;
} }
} }
} }
ec_init_tables(k, errs, tmp_decode_matrix, tmp_decode_tbls); ec_init_tables (k, errs, tmp_decode_matrix, tmp_decode_tbls);
*decode_matrix = tmp_decode_matrix; *decode_matrix = tmp_decode_matrix;
*decode_tbls = tmp_decode_tbls; *decode_tbls = tmp_decode_tbls;
*decode_index = tmp_decode_index;
return 0; return 0;
} }

42
pkgs/erasure/decode.go
View File

@ -21,7 +21,6 @@ package erasure
// #cgo CPPFLAGS: -Iisal/include // #cgo CPPFLAGS: -Iisal/include
// #include <stdlib.h> // #include <stdlib.h>
// #include <erasure-code.h> // #include <erasure-code.h>
// #include <stdlib.h>
// //
// #include "common.h" // #include "common.h"
import "C" import "C"
@ -32,12 +31,13 @@ import (
) )
func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
var decode_matrix *C.uchar var decode_matrix *C.uint8_t
var decode_tbls *C.uchar var decode_tbls *C.uint8_t
var source, target **C.uchar var decode_index *C.uint32_t
var source, target **C.uint8_t
k := int(e.p.k) k := int(e.k)
n := int(e.p.k + e.p.m) n := int(e.k + e.m)
if len(chunks) != n { if len(chunks) != n {
return nil, errors.New(fmt.Sprintf("chunks length must be %d", n)) return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
@ -45,25 +45,25 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(length))) chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(length)))
src_err_list := make([]int, n+1) error_index := make([]int, n+1)
var err_count int = 0 var err_count int = 0
for i := range chunks { for i := range chunks {
// Check of chunks are really null // Check of chunks are really null
if chunks[i] == nil || len(chunks[i]) == 0 { if chunks[i] == nil || len(chunks[i]) == 0 {
src_err_list[err_count] = i error_index[err_count] = i
err_count++ err_count++
} }
} }
src_err_list[err_count] = -1 error_index[err_count] = -1
err_count++ err_count++
// Too many missing chunks, cannot be more than parity `m` // Too many missing chunks, cannot be more than parity `m`
if err_count-1 > e.p.m { if err_count-1 > (n - k) {
return nil, errors.New("too many erasures requested, can't decode") return nil, errors.New("too many erasures requested, can't decode")
} }
src_err_list_ptr := int2cInt(src_err_list[:err_count]) error_index_ptr := int2cInt(error_index[:err_count])
for i := range chunks { for i := range chunks {
if chunks[i] == nil || len(chunks[i]) == 0 { if chunks[i] == nil || len(chunks[i]) == 0 {
@ -71,23 +71,18 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
} }
} }
C.minio_init_decoder(src_err_list_ptr, e.encode_matrix, C.minio_init_decoder(error_index_ptr, e.k, e.k+e.m, C.int(err_count-1),
&decode_matrix, &decode_tbls, e.encode_matrix, &decode_matrix, &decode_tbls, &decode_index)
e.k, e.k+e.m, C.int(err_count-1))
defer C.free(unsafe.Pointer(decode_matrix))
defer C.free(unsafe.Pointer(decode_tbls))
pointers := make([]*byte, n) pointers := make([]*byte, n)
for i := range chunks { for i := range chunks {
pointers[i] = &chunks[i][0] pointers[i] = &chunks[i][0]
} }
data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0])) data := (**C.uint8_t)(unsafe.Pointer(&pointers[0]))
coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0]))
C.minio_get_source_target(src_err_list_ptr, C.int(err_count-1), C.minio_get_source_target(C.int(err_count-1), e.k, e.m, error_index_ptr,
e.k, e.m, data, coding, &source, &target) decode_index, data, &source, &target)
C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls, C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls,
source, target) source, target)
@ -102,8 +97,3 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
return recovered_output[:length], nil return recovered_output[:length], nil
} }
func Decode(block [][]byte, ep *EncoderParams, length int) ([]byte, error) {
encoder := NewEncoder(ep)
return encoder.Decode(block, length)
}

14
pkgs/erasure/encode.c
View File

@ -21,18 +21,16 @@
#include "common.h" #include "common.h"
int32_t minio_init_encoder (int technique, int k, int m, int32_t minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix, uint8_t **encode_matrix,
unsigned char **encode_tbls) uint8_t **encode_tbls)
{ {
size_t encode_matrix_size; size_t encode_matrix_size;
size_t encode_tbls_size; size_t encode_tbls_size;
unsigned char *tmp_matrix, *tmp_tbls; uint8_t *tmp_matrix;
uint8_t *tmp_tbls;
encode_matrix_size = k * (k + m); tmp_matrix = (uint8_t *) malloc (k * (k + m));
encode_tbls_size = k * (k + m) * 32; tmp_tbls = (uint8_t *) malloc (k * (k + m) * 32);
tmp_matrix = (unsigned char *) malloc (encode_matrix_size);
tmp_tbls = (unsigned char *) malloc (encode_tbls_size);
if (technique == 0) { if (technique == 0) {
/* /*

19
pkgs/erasure/encode.go
View File

@ -22,7 +22,6 @@ package erasure
// #cgo LDFLAGS: isal/isa-l.a // #cgo LDFLAGS: isal/isa-l.a
// #include <stdlib.h> // #include <stdlib.h>
// #include <erasure-code.h> // #include <erasure-code.h>
// #include <stdlib.h>
// //
// #include "common.h" // #include "common.h"
import "C" import "C"
@ -57,7 +56,7 @@ type Encoder struct {
encode_matrix, encode_matrix,
encode_tbls, encode_tbls,
decode_matrix, decode_matrix,
decode_tbls *C.uchar decode_tbls *C.uint8_t
} }
// ParseEncoderParams creates an EncoderParams object. // ParseEncoderParams creates an EncoderParams object.
@ -99,13 +98,12 @@ func NewEncoder(ep *EncoderParams) *Encoder {
var k = C.int(ep.k) var k = C.int(ep.k)
var m = C.int(ep.m) var m = C.int(ep.m)
var encode_matrix *C.uchar var encode_matrix *C.uint8_t
var encode_tbls *C.uchar var encode_tbls *C.uint8_t
C.minio_init_encoder(C.int(ep.technique), k, m, &encode_matrix, C.minio_init_encoder(C.int(ep.technique), k, m, &encode_matrix,
&encode_tbls) &encode_tbls)
defer C.free(unsafe.Pointer(encode_matrix))
defer C.free(unsafe.Pointer(encode_tbls))
return &Encoder{ return &Encoder{
p: ep, p: ep,
k: k, k: k,
@ -153,15 +151,10 @@ func (e *Encoder) Encode(block []byte) ([][]byte, int) {
pointers[i] = &chunks[i][0] pointers[i] = &chunks[i][0]
} }
data := (**C.uchar)(unsafe.Pointer(&pointers[:e.p.k][0])) data := (**C.uint8_t)(unsafe.Pointer(&pointers[:e.p.k][0]))
coding := (**C.uchar)(unsafe.Pointer(&pointers[e.p.k:][0])) coding := (**C.uint8_t)(unsafe.Pointer(&pointers[e.p.k:][0]))
C.ec_encode_data(C.int(chunk_size), e.k, e.m, e.encode_tbls, data, C.ec_encode_data(C.int(chunk_size), e.k, e.m, e.encode_tbls, data,
coding) coding)
return chunks, block_len return chunks, block_len
} }
func Encode(block []byte, ep *EncoderParams) ([][]byte, int) {
encoder := NewEncoder(ep)
return encoder.Encode(block)
}

10
pkgs/erasure/isal/include/ec-base.h
View File

@ -2,7 +2,7 @@
Copyright(c) 2011-2014 Intel Corporation All rights reserved. Copyright(c) 2011-2014 Intel Corporation All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions modification, are permitted provided that the following conditions
are met: are met:
* Redistributions of source code must retain the above copyright * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
@ -32,7 +32,7 @@
// Global GF(256) tables // Global GF(256) tables
#ifndef GF_LARGE_TABLES #ifndef GF_LARGE_TABLES
unsigned char gff_base[] = { uint8_t gff_base[] = {
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1d, 0x3a, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1d, 0x3a,
0x74, 0xe8, 0xcd, 0x87, 0x13, 0x26, 0x4c, 0x98, 0x2d, 0x5a, 0x74, 0xe8, 0xcd, 0x87, 0x13, 0x26, 0x4c, 0x98, 0x2d, 0x5a,
0xb4, 0x75, 0xea, 0xc9, 0x8f, 0x03, 0x06, 0x0c, 0x18, 0x30, 0xb4, 0x75, 0xea, 0xc9, 0x8f, 0x03, 0x06, 0x0c, 0x18, 0x30,
@ -61,7 +61,7 @@ unsigned char gff_base[] = {
0x6c, 0xd8, 0xad, 0x47, 0x8e, 0x01 0x6c, 0xd8, 0xad, 0x47, 0x8e, 0x01
}; };
unsigned char gflog_base[] = { uint8_t gflog_base[] = {
0x00, 0xff, 0x01, 0x19, 0x02, 0x32, 0x1a, 0xc6, 0x03, 0xdf, 0x00, 0xff, 0x01, 0x19, 0x02, 0x32, 0x1a, 0xc6, 0x03, 0xdf,
0x33, 0xee, 0x1b, 0x68, 0xc7, 0x4b, 0x04, 0x64, 0xe0, 0x0e, 0x33, 0xee, 0x1b, 0x68, 0xc7, 0x4b, 0x04, 0x64, 0xe0, 0x0e,
0x34, 0x8d, 0xef, 0x81, 0x1c, 0xc1, 0x69, 0xf8, 0xc8, 0x08, 0x34, 0x8d, 0xef, 0x81, 0x1c, 0xc1, 0x69, 0xf8, 0xc8, 0x08,
@ -90,7 +90,7 @@ unsigned char gflog_base[] = {
0xf4, 0xea, 0xa8, 0x50, 0x58, 0xaf 0xf4, 0xea, 0xa8, 0x50, 0x58, 0xaf
}; };
#else #else
unsigned char gf_mul_table_base[] = { uint8_t gf_mul_table_base[] = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@ -6647,7 +6647,7 @@ unsigned char gf_mul_table_base[] = {
0xc6, 0x39, 0xfe, 0x01, 0x1d, 0xe2 0xc6, 0x39, 0xfe, 0x01, 0x1d, 0xe2
}; };
unsigned char gf_inv_table_base[] = { uint8_t gf_inv_table_base[] = {
0x00, 0x01, 0x8e, 0xf4, 0x47, 0xa7, 0x7a, 0xba, 0xad, 0x9d, 0x00, 0x01, 0x8e, 0xf4, 0x47, 0xa7, 0x7a, 0xba, 0xad, 0x9d,
0xdd, 0x98, 0x3d, 0xaa, 0x5d, 0x96, 0xd8, 0x72, 0xc0, 0x58, 0xdd, 0x98, 0x3d, 0xaa, 0x5d, 0x96, 0xd8, 0x72, 0xc0, 0x58,
0xe0, 0x3e, 0x4c, 0x66, 0x90, 0xde, 0x55, 0x80, 0xa0, 0x83, 0xe0, 0x3e, 0x4c, 0x66, 0x90, 0xde, 0x55, 0x80, 0xa0, 0x83,

99
pkgs/erasure/isal/include/erasure-code.h
View File

@ -31,6 +31,7 @@
#ifndef _ERASURE_CODE_H_ #ifndef _ERASURE_CODE_H_
#define _ERASURE_CODE_H_ #define _ERASURE_CODE_H_
#include <stdint.h>
/** /**
* @file erasure_code.h * @file erasure_code.h
* @brief Interface to functions supporting erasure code encode and decode. * @brief Interface to functions supporting erasure code encode and decode.
@ -71,7 +72,7 @@ extern "C" {
* @returns none * @returns none
*/ */
void ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls); void ec_init_tables(int k, int rows, uint8_t* a, uint8_t* gftbls);
/** /**
* @brief Generate or decode erasure codes on blocks of data. * @brief Generate or decode erasure codes on blocks of data.
@ -93,7 +94,7 @@ void ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls);
* @returns none * @returns none
*/ */
void ec_encode_data_sse(int len, int k, int rows, unsigned char *gftbls, unsigned char **data, unsigned char **coding); void ec_encode_data_sse(int len, int k, int rows, uint8_t *gftbls, uint8_t **data, uint8_t **coding);
/** /**
@ -118,7 +119,7 @@ void ec_encode_data_sse(int len, int k, int rows, unsigned char *gftbls, unsigne
* @returns none * @returns none
*/ */
void ec_encode_data(int len, int k, int rows, unsigned char *gftbls, unsigned char **data, unsigned char **coding); void ec_encode_data(int len, int k, int rows, uint8_t *gftbls, uint8_t **data, uint8_t **coding);
/** /**
@ -140,7 +141,7 @@ void ec_encode_data(int len, int k, int rows, unsigned char *gftbls, unsigned ch
* @returns none * @returns none
*/ */
void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v, unsigned char **src, unsigned char **dest); void ec_encode_data_base(int len, int srcs, int dests, uint8_t *v, uint8_t **src, uint8_t **dest);
/** /**
@ -161,8 +162,8 @@ void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v, unsigne
* @returns none * @returns none
*/ */
void gf_vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls, void gf_vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char *dest); uint8_t **src, uint8_t *dest);
/** /**
* @brief GF(2^8) vector dot product. * @brief GF(2^8) vector dot product.
@ -182,8 +183,8 @@ void gf_vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls, void gf_vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char *dest); uint8_t **src, uint8_t *dest);
/** /**
* @brief GF(2^8) vector dot product. * @brief GF(2^8) vector dot product.
@ -203,8 +204,8 @@ void gf_vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls, void gf_vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char *dest); uint8_t **src, uint8_t *dest);
/** /**
* @brief GF(2^8) vector dot product with two outputs. * @brief GF(2^8) vector dot product with two outputs.
@ -225,8 +226,8 @@ void gf_vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_2vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls, void gf_2vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with two outputs. * @brief GF(2^8) vector dot product with two outputs.
@ -247,8 +248,8 @@ void gf_2vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_2vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls, void gf_2vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with two outputs. * @brief GF(2^8) vector dot product with two outputs.
@ -269,8 +270,8 @@ void gf_2vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_2vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls, void gf_2vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with three outputs. * @brief GF(2^8) vector dot product with three outputs.
@ -291,8 +292,8 @@ void gf_2vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_3vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls, void gf_3vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with three outputs. * @brief GF(2^8) vector dot product with three outputs.
@ -313,8 +314,8 @@ void gf_3vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_3vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls, void gf_3vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with three outputs. * @brief GF(2^8) vector dot product with three outputs.
@ -335,8 +336,8 @@ void gf_3vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_3vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls, void gf_3vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with four outputs. * @brief GF(2^8) vector dot product with four outputs.
@ -357,8 +358,8 @@ void gf_3vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_4vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls, void gf_4vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with four outputs. * @brief GF(2^8) vector dot product with four outputs.
@ -379,8 +380,8 @@ void gf_4vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_4vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls, void gf_4vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with four outputs. * @brief GF(2^8) vector dot product with four outputs.
@ -401,8 +402,8 @@ void gf_4vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_4vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls, void gf_4vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with five outputs. * @brief GF(2^8) vector dot product with five outputs.
@ -423,8 +424,8 @@ void gf_4vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_5vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls, void gf_5vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with five outputs. * @brief GF(2^8) vector dot product with five outputs.
@ -445,8 +446,8 @@ void gf_5vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_5vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls, void gf_5vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with five outputs. * @brief GF(2^8) vector dot product with five outputs.
@ -467,8 +468,8 @@ void gf_5vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_5vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls, void gf_5vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with six outputs. * @brief GF(2^8) vector dot product with six outputs.
@ -489,8 +490,8 @@ void gf_5vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_6vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls, void gf_6vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with six outputs. * @brief GF(2^8) vector dot product with six outputs.
@ -511,8 +512,8 @@ void gf_6vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_6vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls, void gf_6vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product with six outputs. * @brief GF(2^8) vector dot product with six outputs.
@ -533,8 +534,8 @@ void gf_6vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_6vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls, void gf_6vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char **dest); uint8_t **src, uint8_t **dest);
/** /**
* @brief GF(2^8) vector dot product, runs baseline version. * @brief GF(2^8) vector dot product, runs baseline version.
@ -556,8 +557,8 @@ void gf_6vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_vect_dot_prod_base(int len, int vlen, unsigned char *gftbls, void gf_vect_dot_prod_base(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char *dest); uint8_t **src, uint8_t *dest);
/** /**
* @brief GF(2^8) vector dot product, runs appropriate version. * @brief GF(2^8) vector dot product, runs appropriate version.
@ -579,8 +580,8 @@ void gf_vect_dot_prod_base(int len, int vlen, unsigned char *gftbls,
* @returns none * @returns none
*/ */
void gf_vect_dot_prod(int len, int vlen, unsigned char *gftbls, void gf_vect_dot_prod(int len, int vlen, uint8_t *gftbls,
unsigned char **src, unsigned char *dest); uint8_t **src, uint8_t *dest);
/********************************************************************** /**********************************************************************
* The remaining are lib support functions used in GF(2^8) operations. * The remaining are lib support functions used in GF(2^8) operations.
@ -594,7 +595,7 @@ void gf_vect_dot_prod(int len, int vlen, unsigned char *gftbls,
* @returns Product of a and b in GF(2^8) * @returns Product of a and b in GF(2^8)
*/ */
unsigned char gf_mul(unsigned char a, unsigned char b); uint8_t gf_mul(uint8_t a, uint8_t b);
/** /**
* @brief Single element GF(2^8) inverse. * @brief Single element GF(2^8) inverse.
@ -603,7 +604,7 @@ unsigned char gf_mul(unsigned char a, unsigned char b);
* @returns Field element b such that a x b = {1} * @returns Field element b such that a x b = {1}
*/ */
unsigned char gf_inv(unsigned char a); uint8_t gf_inv(uint8_t a);
/** /**
* @brief Generate a matrix of coefficients to be used for encoding. * @brief Generate a matrix of coefficients to be used for encoding.
@ -622,7 +623,7 @@ unsigned char gf_inv(unsigned char a);
* @returns none * @returns none
*/ */
void gf_gen_rs_matrix(unsigned char *a, int m, int k); void gf_gen_rs_matrix(uint8_t *a, int m, int k);
/** /**
* @brief Generate a Cauchy matrix of coefficients to be used for encoding. * @brief Generate a Cauchy matrix of coefficients to be used for encoding.
@ -637,7 +638,7 @@ void gf_gen_rs_matrix(unsigned char *a, int m, int k);
* @returns none * @returns none
*/ */
void gf_gen_cauchy1_matrix(unsigned char *a, int m, int k); void gf_gen_cauchy1_matrix(uint8_t *a, int m, int k);
/** /**
* @brief Invert a matrix in GF(2^8) * @brief Invert a matrix in GF(2^8)
@ -648,7 +649,7 @@ void gf_gen_cauchy1_matrix(unsigned char *a, int m, int k);
* @returns 0 successful, other fail on singular input matrix * @returns 0 successful, other fail on singular input matrix
*/ */
int gf_invert_matrix(unsigned char *in, unsigned char *out, const int n); int gf_invert_matrix(uint8_t *in, uint8_t *out, const int n);
/*************************************************************/ /*************************************************************/

12
pkgs/erasure/isal/include/gf-vect-mul.h
View File

@ -61,7 +61,7 @@ extern "C" {
* @returns 0 pass, other fail * @returns 0 pass, other fail
*/ */
int gf_vect_mul_sse(int len, unsigned char *gftbl, void *src, void *dest); int gf_vect_mul_sse(int len, uint8_t *gftbl, void *src, void *dest);
/** /**
@ -82,7 +82,7 @@ int gf_vect_mul_sse(int len, unsigned char *gftbl, void *src, void *dest);
* @returns 0 pass, other fail * @returns 0 pass, other fail
*/ */
int gf_vect_mul_avx(int len, unsigned char *gftbl, void *src, void *dest); int gf_vect_mul_avx(int len, uint8_t *gftbl, void *src, void *dest);
/** /**
@ -105,7 +105,7 @@ int gf_vect_mul_avx(int len, unsigned char *gftbl, void *src, void *dest);
* @returns 0 pass, other fail * @returns 0 pass, other fail
*/ */
int gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest); int gf_vect_mul(int len, uint8_t *gftbl, void *src, void *dest);
/** /**
@ -118,7 +118,7 @@ int gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest);
* @param gftbl Table output. * @param gftbl Table output.
*/ */
void gf_vect_mul_init(unsigned char c, unsigned char* gftbl); void gf_vect_mul_init(uint8_t c, uint8_t* gftbl);
/** /**
@ -138,8 +138,8 @@ void gf_vect_mul_init(unsigned char c, unsigned char* gftbl);
* @param dest Pointer to destination data array. Must be aligned to 32B. * @param dest Pointer to destination data array. Must be aligned to 32B.
*/ */
void gf_vect_mul_base(int len, unsigned char *a, unsigned char *src, void gf_vect_mul_base(int len, uint8_t *a, uint8_t *src,
unsigned char *dest); uint8_t *dest);
#ifdef __cplusplus #ifdef __cplusplus
} }

47
pkgs/erasure/isal/src/ec-base.c
View File

@ -28,12 +28,13 @@
**********************************************************************/ **********************************************************************/
#include <limits.h> #include <limits.h>
#include <stdint.h>
#include <string.h> // for memset #include <string.h> // for memset
#include "erasure-code.h" #include "erasure-code.h"
#include "ec-base.h" // for GF tables #include "ec-base.h" // for GF tables
#include "erasure/types.h" #include "erasure/types.h"
unsigned char gf_mul(unsigned char a, unsigned char b) uint8_t gf_mul(uint8_t a, uint8_t b)
{ {
#ifndef GF_LARGE_TABLES #ifndef GF_LARGE_TABLES
int i; int i;
@ -47,7 +48,7 @@ unsigned char gf_mul(unsigned char a, unsigned char b)
#endif #endif
} }
unsigned char gf_inv(unsigned char a) uint8_t gf_inv(uint8_t a)
{ {
#ifndef GF_LARGE_TABLES #ifndef GF_LARGE_TABLES
if (a == 0) if (a == 0)
@ -59,10 +60,10 @@ unsigned char gf_inv(unsigned char a)
#endif #endif
} }
void gf_gen_rs_matrix(unsigned char *a, int m, int k) void gf_gen_rs_matrix(uint8_t *a, int m, int k)
{ {
int i, j; int i, j;
unsigned char p, gen = 1; uint8_t p, gen = 1;
memset(a, 0, k * m); memset(a, 0, k * m);
for (i = 0; i < k; i++) for (i = 0; i < k; i++)
@ -78,10 +79,10 @@ void gf_gen_rs_matrix(unsigned char *a, int m, int k)
} }
} }
void gf_gen_cauchy1_matrix(unsigned char *a, int m, int k) void gf_gen_cauchy1_matrix(uint8_t *a, int m, int k)
{ {
int i, j; int i, j;
unsigned char *p; uint8_t *p;
// Identity matrix in high position // Identity matrix in high position
memset(a, 0, k * m); memset(a, 0, k * m);
@ -96,10 +97,10 @@ void gf_gen_cauchy1_matrix(unsigned char *a, int m, int k)
} }
int gf_invert_matrix(unsigned char *in_mat, unsigned char *out_mat, const int n) int gf_invert_matrix(uint8_t *in_mat, uint8_t *out_mat, const int n)
{ {
int i, j, k; int i, j, k;
unsigned char temp; uint8_t temp;
// Set out_mat[] to the identity matrix // Set out_mat[] to the identity matrix
for (i = 0; i < n * n; i++) // memset(out_mat, 0, n*n) for (i = 0; i < n * n; i++) // memset(out_mat, 0, n*n)
@ -154,16 +155,16 @@ int gf_invert_matrix(unsigned char *in_mat, unsigned char *out_mat, const int n)
// Calculates const table gftbl in GF(2^8) from single input A // Calculates const table gftbl in GF(2^8) from single input A
// gftbl(A) = {A{00}, A{01}, A{02}, ... , A{0f} }, {A{00}, A{10}, A{20}, ... , A{f0} } // gftbl(A) = {A{00}, A{01}, A{02}, ... , A{0f} }, {A{00}, A{10}, A{20}, ... , A{f0} }
void gf_vect_mul_init(unsigned char c, unsigned char *tbl) void gf_vect_mul_init(uint8_t c, uint8_t *tbl)
{ {
unsigned char c2 = (c << 1) ^ ((c & 0x80) ? 0x1d : 0); //Mult by GF{2} uint8_t c2 = (c << 1) ^ ((c & 0x80) ? 0x1d : 0); //Mult by GF{2}
unsigned char c4 = (c2 << 1) ^ ((c2 & 0x80) ? 0x1d : 0); //Mult by GF{2} uint8_t c4 = (c2 << 1) ^ ((c2 & 0x80) ? 0x1d : 0); //Mult by GF{2}
unsigned char c8 = (c4 << 1) ^ ((c4 & 0x80) ? 0x1d : 0); //Mult by GF{2} uint8_t c8 = (c4 << 1) ^ ((c4 & 0x80) ? 0x1d : 0); //Mult by GF{2}
#if __WORDSIZE == 64 || _WIN64 || __x86_64__ #if __WORDSIZE == 64 || _WIN64 || __x86_64__
unsigned long long v1, v2, v4, v8, *t; unsigned long long v1, v2, v4, v8, *t;
unsigned long long v10, v20, v40, v80; unsigned long long v10, v20, v40, v80;
unsigned char c17, c18, c20, c24; uint8_t c17, c18, c20, c24;
t = (unsigned long long *)tbl; t = (unsigned long long *)tbl;
@ -191,8 +192,8 @@ void gf_vect_mul_init(unsigned char c, unsigned char *tbl)
t[3] = v80 ^ v40; t[3] = v80 ^ v40;
#else // 32-bit or other #else // 32-bit or other
unsigned char c3, c5, c6, c7, c9, c10, c11, c12, c13, c14, c15; uint8_t c3, c5, c6, c7, c9, c10, c11, c12, c13, c14, c15;
unsigned char c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30, uint8_t c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30,
c31; c31;
c3 = c2 ^ c; c3 = c2 ^ c;
@ -261,11 +262,11 @@ void gf_vect_mul_init(unsigned char c, unsigned char *tbl)
#endif //__WORDSIZE == 64 || _WIN64 || __x86_64__ #endif //__WORDSIZE == 64 || _WIN64 || __x86_64__
} }
void gf_vect_dot_prod_base(int len, int vlen, unsigned char *v, void gf_vect_dot_prod_base(int len, int vlen, uint8_t *v,
unsigned char **src, unsigned char *dest) uint8_t **src, uint8_t *dest)
{ {
int i, j; int i, j;
unsigned char s; uint8_t s;
for (i = 0; i < len; i++) { for (i = 0; i < len; i++) {
s = 0; s = 0;
for (j = 0; j < vlen; j++) for (j = 0; j < vlen; j++)
@ -275,11 +276,11 @@ void gf_vect_dot_prod_base(int len, int vlen, unsigned char *v,
} }
} }
void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v, void ec_encode_data_base(int len, int srcs, int dests, uint8_t *v,
unsigned char **src, unsigned char **dest) uint8_t **src, uint8_t **dest)
{ {
int i, j, l; int i, j, l;
unsigned char s; uint8_t s;
for (l = 0; l < dests; l++) { for (l = 0; l < dests; l++) {
for (i = 0; i < len; i++) { for (i = 0; i < len; i++) {
@ -292,10 +293,10 @@ void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v,
} }
} }
void gf_vect_mul_base(int len, unsigned char *a, unsigned char *src, unsigned char *dest) void gf_vect_mul_base(int len, uint8_t *a, uint8_t *src, uint8_t *dest)
{ {
//2nd element of table array is ref value used to fill it in //2nd element of table array is ref value used to fill it in
unsigned char c = a[1]; uint8_t c = a[1];
while (len-- > 0) while (len-- > 0)
*dest++ = gf_mul(c, *src++); *dest++ = gf_mul(c, *src++);
} }

15
pkgs/erasure/isal/src/ec-highlevel-func.c
View File

@ -27,10 +27,11 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**********************************************************************/ **********************************************************************/
#include <limits.h> #include <limits.h>
#include <stdint.h>
#include "erasure-code.h" #include "erasure-code.h"
#include "erasure/types.h" #include "erasure/types.h"
void ec_init_tables(int k, int rows, unsigned char *a, unsigned char *g_tbls) void ec_init_tables(int k, int rows, uint8_t *a, uint8_t *g_tbls)
{ {
int i, j; int i, j;
@ -43,8 +44,8 @@ void ec_init_tables(int k, int rows, unsigned char *a, unsigned char *g_tbls)
} }
#if __WORDSIZE == 64 || _WIN64 || __x86_64__ #if __WORDSIZE == 64 || _WIN64 || __x86_64__
void ec_encode_data_sse(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data, void ec_encode_data_sse(int len, int k, int rows, uint8_t *g_tbls, uint8_t **data,
unsigned char **coding) uint8_t **coding)
{ {
if (len < 16) { if (len < 16) {
@ -74,8 +75,8 @@ void ec_encode_data_sse(int len, int k, int rows, unsigned char *g_tbls, unsigne
} }
void ec_encode_data_avx(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data, void ec_encode_data_avx(int len, int k, int rows, uint8_t *g_tbls, uint8_t **data,
unsigned char **coding) uint8_t **coding)
{ {
if (len < 16) { if (len < 16) {
@ -105,8 +106,8 @@ void ec_encode_data_avx(int len, int k, int rows, unsigned char *g_tbls, unsigne
} }
void ec_encode_data_avx2(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data, void ec_encode_data_avx2(int len, int k, int rows, uint8_t *g_tbls, uint8_t **data,
unsigned char **coding) uint8_t **coding)
{ {
if (len < 32) { if (len < 32) {

16
pkgs/erasure/vandermonde_test.go
View File

@ -21,23 +21,13 @@ import (
. "gopkg.in/check.v1" . "gopkg.in/check.v1"
) )
func (s *MySuite) TestVanderMondeEncode(c *C) {
ep, _ := ParseEncoderParams(10, 5, VANDERMONDE)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
chunks, length := Encode(data, ep)
c.Logf("chunks length: %d;\nlength: %d\n", len(chunks), length)
c.Assert(length, Equals, len(data))
}
func (s *MySuite) TestVanderMondeDecode(c *C) { func (s *MySuite) TestVanderMondeDecode(c *C) {
ep, _ := ParseEncoderParams(10, 5, VANDERMONDE) ep, _ := ParseEncoderParams(10, 5, VANDERMONDE)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.") data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
chunks, length := Encode(data, ep) e := NewEncoder(ep)
chunks, length := e.Encode(data)
c.Logf("chunks length: %d", len(chunks)) c.Logf("chunks length: %d", len(chunks))
c.Logf("length: %d", length) c.Logf("length: %d", length)
c.Assert(length, Equals, len(data)) c.Assert(length, Equals, len(data))
@ -48,7 +38,7 @@ func (s *MySuite) TestVanderMondeDecode(c *C) {
chunks[9] = nil chunks[9] = nil
chunks[13] = nil chunks[13] = nil
recovered_data, err := Decode(chunks, ep, length) recovered_data, err := e.Decode(chunks, length)
c.Assert(err, IsNil) c.Assert(err, IsNil)
if !bytes.Equal(recovered_data, data) { if !bytes.Equal(recovered_data, data) {