mirror of
https://github.com/minio/minio.git
synced 2024-12-24 22:25:54 -05:00
Merge pull request #55 from harshavardhana/pr_out_decoding_bug_fixes_42
Decoding bug fixes #42
This commit is contained in:
commit
980ad3b373
45
pkgs/erasure/common.h
Normal file
45
pkgs/erasure/common.h
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
/*
|
||||||
|
* Mini Object Storage, (C) 2014 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef __COMMON_H__
|
||||||
|
#define __COMMON_H__
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
#define SIMD_ALIGN 32
|
||||||
|
|
||||||
|
int32_t minio_init_encoder (int technique, int k, int m,
|
||||||
|
unsigned char **encode_matrix,
|
||||||
|
unsigned char **encode_tbls);
|
||||||
|
|
||||||
|
uint32_t minio_calc_chunk_size (int k,
|
||||||
|
uint32_t split_len);
|
||||||
|
|
||||||
|
int32_t minio_init_decoder (int *src_err_list,
|
||||||
|
unsigned char *encoding_matrix,
|
||||||
|
unsigned char **decode_matrix,
|
||||||
|
unsigned char **decode_tbls,
|
||||||
|
int k, int n, int errs);
|
||||||
|
|
||||||
|
int32_t minio_src_in_err (int r, int *src_err_list);
|
||||||
|
|
||||||
|
int32_t minio_get_source_target(int *src_err_list,
|
||||||
|
int errs, int k, int m,
|
||||||
|
unsigned char **data,
|
||||||
|
unsigned char **coding,
|
||||||
|
unsigned char ***source,
|
||||||
|
unsigned char ***target);
|
||||||
|
#endif /* __COMMON_H__ */
|
@ -19,13 +19,14 @@
|
|||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
#include <erasure-code.h>
|
#include <erasure-code.h>
|
||||||
#include "decode.h"
|
#include "common.h"
|
||||||
|
|
||||||
static int src_in_err (int r, int *src_err_list)
|
int32_t minio_src_in_err (int r, int *src_err_list)
|
||||||
{
|
{
|
||||||
int i;
|
int i;
|
||||||
for (i = 0; src_err_list[i] != -1; i++) {
|
for (i = 0; src_err_list[i] != -1; i++) {
|
||||||
if (src_err_list[i] == r) {
|
if (src_err_list[i] == r) {
|
||||||
|
// true
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -33,6 +34,51 @@ static int src_in_err (int r, int *src_err_list)
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int32_t minio_get_source_target(int *src_err_list,
|
||||||
|
int errs, int k, int m,
|
||||||
|
unsigned char **data,
|
||||||
|
unsigned char **coding,
|
||||||
|
unsigned char ***source,
|
||||||
|
unsigned char ***target)
|
||||||
|
{
|
||||||
|
int i, j, l;
|
||||||
|
unsigned char *tmp_source[k];
|
||||||
|
unsigned char *tmp_target[m];
|
||||||
|
|
||||||
|
// Fill zeroes
|
||||||
|
memset (tmp_source, 0, sizeof(tmp_source));
|
||||||
|
memset (tmp_target, 0, sizeof(tmp_target));
|
||||||
|
|
||||||
|
// Separate out source and target buffers from input data/coding chunks
|
||||||
|
// This separation needs to happen at error chunks from input chunks
|
||||||
|
for (i = 0, j = 0, l = 0;
|
||||||
|
((l < k) || (j < errs)) && (i < (k + m)); i++) {
|
||||||
|
if (!minio_src_in_err(i, src_err_list)) {
|
||||||
|
if (l < k) {
|
||||||
|
if (i < k)
|
||||||
|
tmp_source[l] =
|
||||||
|
(unsigned char *) data[i];
|
||||||
|
else
|
||||||
|
tmp_source[l] =
|
||||||
|
(unsigned char *) coding[i - k];
|
||||||
|
l++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (j < m) {
|
||||||
|
if (i < k)
|
||||||
|
tmp_target[j] =
|
||||||
|
(unsigned char *) data[i];
|
||||||
|
else
|
||||||
|
tmp_target[j] =
|
||||||
|
(unsigned char *) coding[i - k];
|
||||||
|
j++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*source = tmp_source;
|
||||||
|
*target = tmp_target;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Generate decode matrix during the decoding phase
|
Generate decode matrix during the decoding phase
|
||||||
*/
|
*/
|
||||||
@ -58,7 +104,7 @@ int minio_init_decoder (int *src_err_list,
|
|||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
for (i = 0, r = 0; i < k; i++, r++) {
|
for (i = 0, r = 0; i < k; i++, r++) {
|
||||||
while (src_in_err(r, src_err_list))
|
while (minio_src_in_err(r, src_err_list))
|
||||||
r++;
|
r++;
|
||||||
for (j = 0; j < k; j++) {
|
for (j = 0; j < k; j++) {
|
||||||
input_matrix[k * i + j] = encode_matrix[k * r + j];
|
input_matrix[k * i + j] = encode_matrix[k * r + j];
|
||||||
|
@ -24,8 +24,7 @@ package erasure
|
|||||||
// #include <erasure-code.h>
|
// #include <erasure-code.h>
|
||||||
// #include <stdlib.h>
|
// #include <stdlib.h>
|
||||||
//
|
//
|
||||||
// #include "decode.h"
|
// #include "common.h"
|
||||||
// #include "encode.h"
|
|
||||||
import "C"
|
import "C"
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@ -36,6 +35,7 @@ import (
|
|||||||
func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
|
func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
|
||||||
var decode_matrix *C.uchar
|
var decode_matrix *C.uchar
|
||||||
var decode_tbls *C.uchar
|
var decode_tbls *C.uchar
|
||||||
|
var source, target **C.uchar
|
||||||
|
|
||||||
k := int(e.p.k)
|
k := int(e.p.k)
|
||||||
n := int(e.p.k + e.p.m)
|
n := int(e.p.k + e.p.m)
|
||||||
@ -44,7 +44,7 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
|
|||||||
return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
|
return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
|
||||||
}
|
}
|
||||||
|
|
||||||
chunk_size := int(C.calc_chunk_size(e.k, C.uint(length)))
|
chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(length)))
|
||||||
|
|
||||||
src_err_list := make([]int, n+1)
|
src_err_list := make([]int, n+1)
|
||||||
var err_count int = 0
|
var err_count int = 0
|
||||||
@ -84,21 +84,14 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
|
|||||||
pointers[i] = &chunks[i][0]
|
pointers[i] = &chunks[i][0]
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
// Pack recovery array as list of valid sources
|
|
||||||
// Its order must be the same as the order
|
|
||||||
// to generate matrix b in gf_gen_decode_matrix
|
|
||||||
var i int
|
|
||||||
for i = 0; i < e.p.k; i++ {
|
|
||||||
recov[i] = buffs[decode_index[i]]
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0]))
|
data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0]))
|
||||||
coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0]))
|
coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0]))
|
||||||
|
|
||||||
|
C.minio_get_source_target(src_err_list_ptr, C.int(err_count-1),
|
||||||
|
e.k, e.m, data, coding, &source, &target)
|
||||||
|
|
||||||
C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls,
|
C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls,
|
||||||
data, coding)
|
source, target)
|
||||||
|
|
||||||
recovered_output := make([]byte, 0, chunk_size*k)
|
recovered_output := make([]byte, 0, chunk_size*k)
|
||||||
for i := 0; i < k; i++ {
|
for i := 0; i < k; i++ {
|
||||||
|
@ -1,25 +0,0 @@
|
|||||||
/*
|
|
||||||
* Mini Object Storage, (C) 2014 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef __DECODE_H__
|
|
||||||
#define __DECODE_H__
|
|
||||||
|
|
||||||
int minio_init_decoder (int *src_err_list,
|
|
||||||
unsigned char *encoding_matrix,
|
|
||||||
unsigned char **decode_matrix,
|
|
||||||
unsigned char **decode_tbls,
|
|
||||||
int k, int n, int errs);
|
|
||||||
#endif /* __DECODE_H__ */
|
|
@ -13,37 +13,36 @@
|
|||||||
//
|
//
|
||||||
// Encoding data may be performed in 3 steps.
|
// Encoding data may be performed in 3 steps.
|
||||||
//
|
//
|
||||||
// 1. Create a validated set of encoder parameters
|
// 1. Create a parse set of encoder parameters
|
||||||
// 2. Create a new encoder
|
// 2. Create a new encoder
|
||||||
// 3. Encode data
|
// 3. Encode data
|
||||||
//
|
//
|
||||||
// Decoding data is also performed in 3 steps.
|
// Decoding data is also performed in 3 steps.
|
||||||
//
|
//
|
||||||
// 1. Create a validated set of encoder parameters
|
// 1. Create a parse set of encoder parameters for validation
|
||||||
// 2. Create a new encoder
|
// 2. Create a new encoder
|
||||||
// 3. Decode data
|
// 3. Decode data
|
||||||
//
|
//
|
||||||
// Encoder parameters contain four configurable elements:
|
// Encoder parameters contain four configurable elements:
|
||||||
// ValidateParams(k, m, w, technique int) (EncoderParams, error)
|
// ParseEncoderParams(k, m, technique int) (EncoderParams, error)
|
||||||
// k - Number of rows in matrix
|
// k - Number of rows in matrix
|
||||||
// m - Number of colums in matrix
|
// m - Number of colums in matrix
|
||||||
// w - Word size, typically 8 or aligned by 8.
|
|
||||||
// technique - Matrix type, can be either CAUCHY (recommended) or VANDERMONDE
|
// technique - Matrix type, can be either CAUCHY (recommended) or VANDERMONDE
|
||||||
// constraints: k + m < 256
|
// constraints: k + m < Galois Field (2^8)
|
||||||
//
|
//
|
||||||
// Example
|
// Example
|
||||||
//
|
//
|
||||||
//
|
//
|
||||||
// Creating and using an encoder
|
// Creating and using an encoder
|
||||||
// var bytes []byte
|
// var bytes []byte
|
||||||
// params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY)
|
// params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY)
|
||||||
// encoder := erasure.NewEncoder(params)
|
// encoder := erasure.NewEncoder(params)
|
||||||
// encodedData, length := encoder.Encode(bytes)
|
// encodedData, length := encoder.Encode(bytes)
|
||||||
//
|
//
|
||||||
// Creating and using a decoder
|
// Creating and using a decoder
|
||||||
// var encodedData [][]byte
|
// var encodedData [][]byte
|
||||||
// var length int
|
// var length int
|
||||||
// params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY)
|
// params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY)
|
||||||
// encoder := erasure.NewEncoder(params)
|
// encoder := erasure.NewEncoder(params)
|
||||||
// originalData, err := encoder.Decode(encodedData, length)
|
// originalData, err := encoder.Decode(encodedData, length)
|
||||||
package erasure
|
package erasure
|
||||||
|
@ -18,11 +18,11 @@
|
|||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
|
||||||
#include <erasure-code.h>
|
#include <erasure-code.h>
|
||||||
#include "encode.h"
|
#include "common.h"
|
||||||
|
|
||||||
void minio_init_encoder (int technique, int k, int m,
|
int32_t minio_init_encoder (int technique, int k, int m,
|
||||||
unsigned char **encode_matrix,
|
unsigned char **encode_matrix,
|
||||||
unsigned char **encode_tbls)
|
unsigned char **encode_tbls)
|
||||||
{
|
{
|
||||||
size_t encode_matrix_size;
|
size_t encode_matrix_size;
|
||||||
size_t encode_tbls_size;
|
size_t encode_tbls_size;
|
||||||
@ -52,9 +52,11 @@ void minio_init_encoder (int technique, int k, int m,
|
|||||||
|
|
||||||
*encode_matrix = tmp_matrix;
|
*encode_matrix = tmp_matrix;
|
||||||
*encode_tbls = tmp_tbls;
|
*encode_tbls = tmp_tbls;
|
||||||
|
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned int calc_chunk_size (int k, unsigned int split_len)
|
uint32_t minio_calc_chunk_size (int k, uint32_t split_len)
|
||||||
{
|
{
|
||||||
int alignment;
|
int alignment;
|
||||||
int remainder;
|
int remainder;
|
||||||
@ -69,7 +71,3 @@ unsigned int calc_chunk_size (int k, unsigned int split_len)
|
|||||||
}
|
}
|
||||||
return padded_len / k;
|
return padded_len / k;
|
||||||
}
|
}
|
||||||
/*
|
|
||||||
void minio_encode (int k, int m, )
|
|
||||||
{
|
|
||||||
*/
|
|
||||||
|
@ -24,7 +24,7 @@ package erasure
|
|||||||
// #include <erasure-code.h>
|
// #include <erasure-code.h>
|
||||||
// #include <stdlib.h>
|
// #include <stdlib.h>
|
||||||
//
|
//
|
||||||
// #include "encode.h"
|
// #include "common.h"
|
||||||
import "C"
|
import "C"
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@ -123,7 +123,7 @@ func NewEncoder(ep *EncoderParams) *Encoder {
|
|||||||
func (e *Encoder) Encode(block []byte) ([][]byte, int) {
|
func (e *Encoder) Encode(block []byte) ([][]byte, int) {
|
||||||
var block_len = len(block)
|
var block_len = len(block)
|
||||||
|
|
||||||
chunk_size := int(C.calc_chunk_size(e.k, C.uint(block_len)))
|
chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(block_len)))
|
||||||
chunk_len := chunk_size * e.p.k
|
chunk_len := chunk_size * e.p.k
|
||||||
pad_len := chunk_len - block_len
|
pad_len := chunk_len - block_len
|
||||||
|
|
||||||
|
@ -1,25 +0,0 @@
|
|||||||
/*
|
|
||||||
* Mini Object Storage, (C) 2014 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef __ENCODE_H__
|
|
||||||
#define __ENCODE_H__
|
|
||||||
|
|
||||||
#define SIMD_ALIGN 32
|
|
||||||
void minio_init_encoder (int technique, int k, int m,
|
|
||||||
unsigned char **encode_matrix,
|
|
||||||
unsigned char **encode_tbls);
|
|
||||||
unsigned int calc_chunk_size (int k, unsigned int split_len);
|
|
||||||
#endif /* __ENCODE_H__ */
|
|
Loading…
Reference in New Issue
Block a user