MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make erasure matrix type not optional choose automatically 

Remove option of providing Technique and handling errors based on that
choose a matrix type automatically based on number of data blocks.

INTEL recommends on using cauchy for consistent invertible matrices,
while vandermonde is faster we should default to cauchy for large
data blocks.
This commit is contained in:
Harshavardhana 2015-10-05 22:33:39 -07:00
parent cf0e1a156b
commit d5ce2f6944
13 changed files with 67 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pkg/donut/bucket.go
View File

@ -288,7 +288,6 @@ func (b bucket) WriteObject(objectName string, objectData io.Reader, size int64,
objMetadata.ChunkCount = chunkCount
objMetadata.DataDisks = k
objMetadata.ParityDisks = m
objMetadata.ErasureTechnique = "Cauchy"
objMetadata.Size = int64(totalLength)
}
objMetadata.Bucket = b.getBucketName()
@ -431,7 +430,7 @@ func (b bucket) getDataAndParity(totalWriters int) (k uint8, m uint8, err *probe
// writeObjectData -
func (b bucket) writeObjectData(k, m uint8, writers []io.WriteCloser, objectData io.Reader, size int64, hashWriter io.Writer) (int, int, *probe.Error) {
encoder, err := newEncoder(k, m, "Cauchy")
encoder, err := newEncoder(k, m)
if err != nil {
return 0, 0, err.Trace()
}
@ -503,11 +502,7 @@ func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMeta
mwriter := io.MultiWriter(writer, hasher, sum512hasher)
switch len(readers) > 1 {
case true:
if objMetadata.ErasureTechnique == "" {
writer.CloseWithError(probe.WrapError(probe.NewError(MissingErasureTechnique{})))
return
}
encoder, err := newEncoder(objMetadata.DataDisks, objMetadata.ParityDisks, objMetadata.ErasureTechnique)
encoder, err := newEncoder(objMetadata.DataDisks, objMetadata.ParityDisks)
if err != nil {
writer.CloseWithError(probe.WrapError(err))
return

1
pkg/donut/definitions.go
View File

@ -32,7 +32,6 @@ type ObjectMetadata struct {
// erasure
DataDisks uint8 `json:"sys.erasureK"`
ParityDisks uint8 `json:"sys.erasureM"`
ErasureTechnique string `json:"sys.erasureTechnique"`
BlockSize int `json:"sys.blockSize"`
ChunkCount int `json:"sys.chunkCount"`

24
pkg/donut/encoder.go
View File

@ -25,39 +25,19 @@ import (
type encoder struct {
encoder *encoding.Erasure
k, m uint8
technique encoding.Technique
}
// getErasureTechnique - convert technique string into Technique type
func getErasureTechnique(technique string) (encoding.Technique, *probe.Error) {
switch true {
case technique == "Cauchy":
return encoding.Cauchy, nil
case technique == "Vandermonde":
return encoding.Cauchy, nil
default:
return encoding.None, probe.NewError(InvalidErasureTechnique{Technique: technique})
}
}
// newEncoder - instantiate a new encoder
func newEncoder(k, m uint8, technique string) (encoder, *probe.Error) {
func newEncoder(k, m uint8) (encoder, *probe.Error) {
e := encoder{}
t, err := getErasureTechnique(technique)
if err != nil {
return encoder{}, err.Trace()
}
{
params, err := encoding.ValidateParams(k, m, t)
params, err := encoding.ValidateParams(k, m)
if err != nil {
return encoder{}, probe.NewError(err)
}
e.encoder = encoding.NewErasure(params)
e.k = k
e.m = m
e.technique = t
return e, nil
}
}
// TODO - think again if this is needed

16
pkg/donut/errors.go
View File

@ -132,22 +132,6 @@ func (e MissingPOSTPolicy) Error() string {
return "Missing POST policy in multipart form"
}
// MissingErasureTechnique missing erasure technique
type MissingErasureTechnique struct{}
func (e MissingErasureTechnique) Error() string {
return "Missing erasure technique"
}
// InvalidErasureTechnique invalid erasure technique
type InvalidErasureTechnique struct {
Technique string
}
func (e InvalidErasureTechnique) Error() string {
return "Invalid erasure technique: " + e.Technique
}
// InternalError - generic internal error
type InternalError struct {
}

7
pkg/erasure/doc.go
View File

@ -39,10 +39,9 @@
//
// But here are the few points to keep in mind
//
// Techniques:
// Matrix Type:
// - Vandermonde is most commonly used method for choosing coefficients in erasure
// encoding but does not guarantee invertable for every sub matrix.
// Users may want to adjust for k > 5. (k is data blocks)
// - Whereas Cauchy is our recommended method for choosing coefficients in erasure coding.
// Since any sub-matrix of a Cauchy matrix is invertable.
//
@ -53,14 +52,14 @@
//
// Creating and using an encoder
// var bytes []byte
// params := erasure.ValidateParams(10, 5, erasure.Cauchy)
// params := erasure.ValidateParams(10, 5)
// encoder := erasure.NewErasure(params)
// encodedData, length := encoder.Encode(bytes)
//
// Creating and using a decoder
// var encodedData [][]byte
// var length int
// params := erasure.ValidateParams(10, 5, erasure.Cauchy)
// params := erasure.ValidateParams(10, 5)
// encoder := erasure.NewErasure(params)
// originalData, err := encoder.Decode(encodedData, length)
//

0
pkg/erasure/ec_isal-l.h → pkg/erasure/ec.h
View File

2
pkg/erasure/ec_minio_common.h
View File

@ -19,7 +19,7 @@
#include <stdint.h>
int32_t minio_init_encoder (int technique, int k, int m,
int32_t minio_init_encoder (int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls);

2
pkg/erasure/ec_minio_decode.c
View File

@ -18,7 +18,7 @@
#include <stdlib.h>
#include <string.h>
#include "ec_isal-l.h"
#include "ec.h"
#include "ec_minio_common.h"
static

10
pkg/erasure/ec_minio_encode.c
View File

@ -17,16 +17,14 @@
#include <stdlib.h>
#include <stdio.h>
#include "ec_isal-l.h"
#include "ec.h"
#include "ec_minio_common.h"
/*
Generate encode matrix during the encoding phase
*/
int32_t minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls)
int32_t minio_init_encoder (int k, int m, unsigned char **encode_matrix, unsigned char **encode_tbls)
{
unsigned char *tmp_matrix;
unsigned char *tmp_tbls;
@ -34,7 +32,7 @@ int32_t minio_init_encoder (int technique, int k, int m,
tmp_matrix = (unsigned char *) malloc (k * (k + m));
tmp_tbls = (unsigned char *) malloc (k * (k + m) * 32);
if (technique == 0) {
if (k < 5) {
/*
Commonly used method for choosing coefficients in erasure
encoding but does not guarantee invertable for every sub
@ -44,7 +42,7 @@ int32_t minio_init_encoder (int technique, int k, int m,
-- Intel
*/
gf_gen_rs_matrix (tmp_matrix, k + m, k);
} else if (technique == 1) {
} else {
gf_gen_cauchy1_matrix (tmp_matrix, k + m, k);
}

2
pkg/erasure/erasure_decode.go
View File

@ -18,7 +18,7 @@ package erasure
// #cgo CFLAGS: -O0
// #include <stdlib.h>
// #include "ec_isal-l.h"
// #include "ec.h"
// #include "ec_minio_common.h"
import "C"
import (

34
pkg/erasure/erasure_encode.go
View File

@ -17,7 +17,7 @@
package erasure
// #include <stdlib.h>
// #include "ec_isal-l.h"
// #include "ec.h"
// #include "ec_minio_common.h"
import "C"
import (
@ -25,22 +25,6 @@ import (
"unsafe"
)
// Technique - type of matrix type used in encoding
type Technique uint8
// Different types of supported matrix types
const (
Vandermonde Technique = iota
Cauchy
None
)
// Default Data and Parity blocks
const (
K = 10
M = 3
)
// Block alignment
const (
SIMDAlign = 32
@ -50,7 +34,6 @@ const (
type Params struct {
K uint8
M uint8
Technique Technique // cauchy or vandermonde matrix (RS)
}
// Erasure is an object used to encode and decode data.
@ -66,7 +49,7 @@ type Erasure struct {
// k and m represent the matrix size, which corresponds to the protection level
// technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde.
//
func ValidateParams(k, m uint8, technique Technique) (*Params, error) {
func ValidateParams(k, m uint8) (*Params, error) {
if k < 1 {
return nil, errors.New("k cannot be zero")
}
@ -79,19 +62,9 @@ func ValidateParams(k, m uint8, technique Technique) (*Params, error) {
return nil, errors.New("(k + m) cannot be bigger than Galois field GF(2^8) - 1")
}
switch technique {
case Vandermonde:
break
case Cauchy:
break
default:
return nil, errors.New("Technique can be either vandermonde or cauchy")
}
return &Params{
K: k,
M: m,
Technique: technique,
}, nil
}
@ -103,8 +76,7 @@ func NewErasure(ep *Params) *Erasure {
var encodeMatrix *C.uchar
var encodeTbls *C.uchar
C.minio_init_encoder(C.int(ep.Technique), k, m, &encodeMatrix,
&encodeTbls)
C.minio_init_encoder(k, m, &encodeMatrix, &encodeTbls)
return &Erasure{
params: ep,

39
pkg/erasure/cauchy_test.go → pkg/erasure/erasure_test.go
View File

@ -34,8 +34,15 @@ const (
m = 5
)
func (s *MySuite) TestCauchyEncodeDecodeFailure(c *C) {
ep, _ := ValidateParams(k, m, Cauchy)
func corruptChunks(chunks [][]byte, errorIndex []int) [][]byte {
for _, err := range errorIndex {
chunks[err] = nil
}
return chunks
}
func (s *MySuite) TestEncodeDecodeFailure(c *C) {
ep, _ := ValidateParams(k, m)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
@ -50,8 +57,8 @@ func (s *MySuite) TestCauchyEncodeDecodeFailure(c *C) {
c.Assert(err, Not(IsNil))
}
func (s *MySuite) TestCauchyEncodeDecodeSuccess(c *C) {
ep, _ := ValidateParams(k, m, Cauchy)
func (s *MySuite) TestEncodeDecodeSuccess(c *C) {
ep, _ := ValidateParams(k, m)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
@ -69,3 +76,27 @@ func (s *MySuite) TestCauchyEncodeDecodeSuccess(c *C) {
c.Fatalf("Recovered data mismatches with original data")
}
}
func (s *MySuite) TestEncodeDecodeSuccessBuffer(c *C) {
ep, _ := ValidateParams(k, m)
tmpBuffer := new(bytes.Buffer)
for i := 0; i < 1024*1024; i++ {
tmpBuffer.Write([]byte("Hello world, hello world"))
}
e := NewErasure(ep)
chunks, err := e.Encode(tmpBuffer.Bytes())
c.Assert(err, IsNil)
errorIndex := []int{0, 3, 5, 9, 13}
chunks = corruptChunks(chunks, errorIndex)
recoveredData, err := e.Decode(chunks, len(tmpBuffer.Bytes()))
c.Assert(err, IsNil)
if !bytes.Equal(tmpBuffer.Bytes(), recoveredData) {
c.Fatalf("Recovered data mismatches with original data")
}
}

66
pkg/erasure/vandermonde_test.go
View File

@ -1,66 +0,0 @@
/*
* Minio Cloud Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package erasure
import (
"bytes"
. "gopkg.in/check.v1"
)
func corruptChunks(chunks [][]byte, errorIndex []int) [][]byte {
for _, err := range errorIndex {
chunks[err] = nil
}
return chunks
}
func (s *MySuite) TestVanderMondeEncodeDecodeFailure(c *C) {
ep, _ := ValidateParams(k, m, Vandermonde)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
e := NewErasure(ep)
chunks, err := e.Encode(data)
c.Assert(err, IsNil)
errorIndex := []int{0, 3, 5, 9, 11, 13}
chunks = corruptChunks(chunks, errorIndex)
_, err = e.Decode(chunks, len(data))
c.Assert(err, Not(IsNil))
}
func (s *MySuite) TestVanderMondeEncodeDecodeSuccess(c *C) {
ep, _ := ValidateParams(k, m, Vandermonde)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
e := NewErasure(ep)
chunks, err := e.Encode(data)
c.Assert(err, IsNil)
errorIndex := []int{0, 3, 5, 9, 13}
chunks = corruptChunks(chunks, errorIndex)
recoveredData, err := e.Decode(chunks, len(data))
c.Assert(err, IsNil)
if !bytes.Equal(recoveredData, data) {
c.Fatalf("Recovered data mismatches with original data")
}
}