mirror of https://github.com/minio/minio.git
Merge pull request #342 from abperiasamy/janitor_remove_k_m
This commit is contained in:
commit
601f5c77fe
|
@ -37,12 +37,13 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
|
||||||
var decode_index *C.uint32_t
|
var decode_index *C.uint32_t
|
||||||
var source, target **C.uint8_t
|
var source, target **C.uint8_t
|
||||||
|
|
||||||
k := int(e.k)
|
k := e.parms.K
|
||||||
n := int(e.k + e.m)
|
m := e.parms.M
|
||||||
|
n := k + m
|
||||||
if len(chunks) != int(n) {
|
if len(chunks) != int(n) {
|
||||||
return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
|
return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
|
||||||
}
|
}
|
||||||
chunk_size := GetEncodedChunkLen(length, k)
|
chunk_size := GetEncodedChunkLen(length, uint8(k))
|
||||||
|
|
||||||
error_index := make([]int, n+1)
|
error_index := make([]int, n+1)
|
||||||
var err_count int = 0
|
var err_count int = 0
|
||||||
|
@ -70,7 +71,7 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
C.minio_init_decoder(error_index_ptr, e.k, e.k+e.m, C.int(err_count-1),
|
C.minio_init_decoder(error_index_ptr, C.int(k), C.int(n), C.int(err_count-1),
|
||||||
e.encode_matrix, &decode_matrix, &decode_tbls, &decode_index)
|
e.encode_matrix, &decode_matrix, &decode_tbls, &decode_index)
|
||||||
|
|
||||||
pointers := make([]*byte, n)
|
pointers := make([]*byte, n)
|
||||||
|
@ -80,14 +81,14 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
|
||||||
|
|
||||||
data := (**C.uint8_t)(unsafe.Pointer(&pointers[0]))
|
data := (**C.uint8_t)(unsafe.Pointer(&pointers[0]))
|
||||||
|
|
||||||
ret := C.minio_get_source_target(C.int(err_count-1), e.k, e.m, error_index_ptr,
|
ret := C.minio_get_source_target(C.int(err_count-1), C.int(k), C.int(m), error_index_ptr,
|
||||||
decode_index, data, &source, &target)
|
decode_index, data, &source, &target)
|
||||||
|
|
||||||
if int(ret) == -1 {
|
if int(ret) == -1 {
|
||||||
return nil, errors.New("Decoding source target failed")
|
return nil, errors.New("Decoding source target failed")
|
||||||
}
|
}
|
||||||
|
|
||||||
C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls,
|
C.ec_encode_data(C.int(chunk_size), C.int(k), C.int(err_count-1), decode_tbls,
|
||||||
source, target)
|
source, target)
|
||||||
|
|
||||||
recovered_output := make([]byte, 0, chunk_size*int(k))
|
recovered_output := make([]byte, 0, chunk_size*int(k))
|
||||||
|
|
|
@ -39,7 +39,7 @@ const (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
SimdAlign = 32
|
SIMDAlign = 32
|
||||||
)
|
)
|
||||||
|
|
||||||
// EncoderParams is a configuration set for building an encoder. It is created using ValidateParams.
|
// EncoderParams is a configuration set for building an encoder. It is created using ValidateParams.
|
||||||
|
@ -51,9 +51,7 @@ type EncoderParams struct {
|
||||||
|
|
||||||
// Encoder is an object used to encode and decode data.
|
// Encoder is an object used to encode and decode data.
|
||||||
type Encoder struct {
|
type Encoder struct {
|
||||||
p *EncoderParams
|
parms *EncoderParams
|
||||||
k,
|
|
||||||
m C.int
|
|
||||||
encode_matrix,
|
encode_matrix,
|
||||||
encode_tbls,
|
encode_tbls,
|
||||||
decode_matrix,
|
decode_matrix,
|
||||||
|
@ -96,19 +94,17 @@ func ParseEncoderParams(k, m uint8, technique Technique) (*EncoderParams, error)
|
||||||
|
|
||||||
// NewEncoder creates an encoder object with a given set of parameters.
|
// NewEncoder creates an encoder object with a given set of parameters.
|
||||||
func NewEncoder(ep *EncoderParams) *Encoder {
|
func NewEncoder(ep *EncoderParams) *Encoder {
|
||||||
var k = C.int(ep.K)
|
|
||||||
var m = C.int(ep.M)
|
|
||||||
|
|
||||||
var encode_matrix *C.uint8_t
|
var encode_matrix *C.uint8_t
|
||||||
var encode_tbls *C.uint8_t
|
var encode_tbls *C.uint8_t
|
||||||
|
|
||||||
|
k := C.int(ep.K)
|
||||||
|
m := C.int(ep.M)
|
||||||
|
|
||||||
C.minio_init_encoder(C.int(ep.Technique), k, m, &encode_matrix,
|
C.minio_init_encoder(C.int(ep.Technique), k, m, &encode_matrix,
|
||||||
&encode_tbls)
|
&encode_tbls)
|
||||||
|
|
||||||
return &Encoder{
|
return &Encoder{
|
||||||
p: ep,
|
parms: ep,
|
||||||
k: k,
|
|
||||||
m: m,
|
|
||||||
encode_matrix: encode_matrix,
|
encode_matrix: encode_matrix,
|
||||||
encode_tbls: encode_tbls,
|
encode_tbls: encode_tbls,
|
||||||
decode_matrix: nil,
|
decode_matrix: nil,
|
||||||
|
@ -116,20 +112,20 @@ func NewEncoder(ep *EncoderParams) *Encoder {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetEncodedLen(inputLen, k, m int) (outputLen int) {
|
func GetEncodedLen(inputLen int, k, m uint8) (outputLen int) {
|
||||||
outputLen = GetEncodedChunkLen(inputLen, k) * (k + m)
|
outputLen = GetEncodedChunkLen(inputLen, k) * int(k+m)
|
||||||
return outputLen
|
return outputLen
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetEncodedChunkLen(inputLen, k int) (outputChunkLen int) {
|
func GetEncodedChunkLen(inputLen int, k uint8) (outputChunkLen int) {
|
||||||
alignment := k * SimdAlign
|
alignment := int(k) * SIMDAlign
|
||||||
remainder := inputLen % alignment
|
remainder := inputLen % alignment
|
||||||
|
|
||||||
paddedInputLen := inputLen
|
paddedInputLen := inputLen
|
||||||
if remainder != 0 {
|
if remainder != 0 {
|
||||||
paddedInputLen = inputLen + (alignment - remainder)
|
paddedInputLen = inputLen + (alignment - remainder)
|
||||||
}
|
}
|
||||||
outputChunkLen = paddedInputLen / k
|
outputChunkLen = paddedInputLen / int(k)
|
||||||
return outputChunkLen
|
return outputChunkLen
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -137,10 +133,13 @@ func GetEncodedChunkLen(inputLen, k int) (outputChunkLen int) {
|
||||||
// is a 2 tuple containing (k + m) chunks of erasure encoded data and the
|
// is a 2 tuple containing (k + m) chunks of erasure encoded data and the
|
||||||
// length of the original object.
|
// length of the original object.
|
||||||
func (e *Encoder) Encode(input []byte) ([][]byte, error) {
|
func (e *Encoder) Encode(input []byte) ([][]byte, error) {
|
||||||
var inputLen = len(input)
|
inputLen := len(input)
|
||||||
|
k := C.int(e.parms.K)
|
||||||
|
m := C.int(e.parms.M)
|
||||||
|
n := k + m
|
||||||
|
|
||||||
chunkLen := GetEncodedChunkLen(inputLen, int(e.k))
|
chunkLen := GetEncodedChunkLen(inputLen, e.parms.K)
|
||||||
encodedDataLen := chunkLen * int(e.k)
|
encodedDataLen := chunkLen * int(k)
|
||||||
paddedDataLen := int(encodedDataLen) - inputLen
|
paddedDataLen := int(encodedDataLen) - inputLen
|
||||||
|
|
||||||
if paddedDataLen > 0 {
|
if paddedDataLen > 0 {
|
||||||
|
@ -149,32 +148,32 @@ func (e *Encoder) Encode(input []byte) ([][]byte, error) {
|
||||||
input = append(input, s...)
|
input = append(input, s...)
|
||||||
}
|
}
|
||||||
|
|
||||||
encodedParityLen := chunkLen * int(e.p.M)
|
encodedParityLen := chunkLen * int(e.parms.M)
|
||||||
c := make([]byte, encodedParityLen)
|
c := make([]byte, encodedParityLen)
|
||||||
input = append(input, c...)
|
input = append(input, c...)
|
||||||
|
|
||||||
// encodedOutLen := encodedDataLen + encodedParityLen
|
// encodedOutLen := encodedDataLen + encodedParityLen
|
||||||
|
|
||||||
// Allocate chunks
|
// Allocate chunks
|
||||||
chunks := make([][]byte, e.p.K+e.p.M)
|
chunks := make([][]byte, k+m)
|
||||||
pointers := make([]*byte, e.p.K+e.p.M)
|
pointers := make([]*byte, k+m)
|
||||||
|
|
||||||
var i int
|
var i int
|
||||||
// Add data blocks to chunks
|
// Add data blocks to chunks
|
||||||
for i = 0; i < int(e.p.K); i++ {
|
for i = 0; i < int(k); i++ {
|
||||||
chunks[i] = input[i*chunkLen : (i+1)*chunkLen]
|
chunks[i] = input[i*chunkLen : (i+1)*chunkLen]
|
||||||
pointers[i] = &chunks[i][0]
|
pointers[i] = &chunks[i][0]
|
||||||
}
|
}
|
||||||
|
|
||||||
for i = int(e.p.K); i < int(e.p.K+e.p.M); i++ {
|
for i = int(k); i < int(n); i++ {
|
||||||
chunks[i] = make([]byte, chunkLen)
|
chunks[i] = make([]byte, chunkLen)
|
||||||
pointers[i] = &chunks[i][0]
|
pointers[i] = &chunks[i][0]
|
||||||
}
|
}
|
||||||
|
|
||||||
data := (**C.uint8_t)(unsafe.Pointer(&pointers[:e.p.K][0]))
|
data := (**C.uint8_t)(unsafe.Pointer(&pointers[:k][0]))
|
||||||
coding := (**C.uint8_t)(unsafe.Pointer(&pointers[e.p.K:][0]))
|
coding := (**C.uint8_t)(unsafe.Pointer(&pointers[k:][0]))
|
||||||
|
|
||||||
C.ec_encode_data(C.int(chunkLen), e.k, e.m, e.encode_tbls, data,
|
C.ec_encode_data(C.int(chunkLen), k, m, e.encode_tbls, data,
|
||||||
coding)
|
coding)
|
||||||
return chunks, nil
|
return chunks, nil
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue