minio/pkg/erasure/erasure_encode.go

/*
 * Minio Cloud Storage, (C) 2014 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package erasure

// #include <stdlib.h>
// #include "ec_isal-l.h"
// #include "ec_minio_common.h"
import "C"
import (
	"errors"
	"io"
	"unsafe"
)

// Technique - type of matrix type used in encoding
type Technique uint8

// Different types of supported matrix types
const (
	Vandermonde Technique = iota
	Cauchy
	None
)

// Default Data and Parity blocks
const (
	K = 10
	M = 3
)

// Block alignment
const (
	SIMDAlign = 32
)

// Params is a configuration set for building an encoder. It is created using ValidateParams().
type Params struct {
	K         uint8
	M         uint8
	Technique Technique // cauchy or vandermonde matrix (RS)
}

// Erasure is an object used to encode and decode data.
type Erasure struct {
	params                   *Params
	encodeMatrix, encodeTbls *C.uchar
	decodeMatrix, decodeTbls *C.uchar
	decodeIndex              *C.uint32_t
}

// ValidateParams creates an Params object.
//
// k and m represent the matrix size, which corresponds to the protection level
// technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde.
//
func ValidateParams(k, m uint8, technique Technique) (*Params, error) {
	if k < 1 {
		return nil, errors.New("k cannot be zero")
	}

	if m < 1 {
		return nil, errors.New("m cannot be zero")
	}

	if k+m > 255 {
		return nil, errors.New("(k + m) cannot be bigger than Galois field GF(2^8) - 1")
	}

	switch technique {
	case Vandermonde:
		break
	case Cauchy:
		break
	default:
		return nil, errors.New("Technique can be either vandermonde or cauchy")
	}

	return &Params{
		K:         k,
		M:         m,
		Technique: technique,
	}, nil
}

// NewErasure creates an encoder object with a given set of parameters.
func NewErasure(ep *Params) *Erasure {
	var k = C.int(ep.K)
	var m = C.int(ep.M)

	var encodeMatrix *C.uchar
	var encodeTbls *C.uchar

	C.minio_init_encoder(C.int(ep.Technique), k, m, &encodeMatrix,
		&encodeTbls)

	return &Erasure{
		params:       ep,
		encodeMatrix: encodeMatrix,
		encodeTbls:   encodeTbls,
		decodeMatrix: nil,
		decodeTbls:   nil,
		decodeIndex:  nil,
	}
}

// GetEncodedBlocksLen - total length of all encoded blocks
func GetEncodedBlocksLen(inputLen int, k, m uint8) (outputLen int) {
	outputLen = GetEncodedBlockLen(inputLen, k) * int(k+m)
	return outputLen
}

// GetEncodedBlockLen - length per block of encoded blocks
func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) {
	alignment := int(k) * SIMDAlign
	remainder := inputLen % alignment

	paddedInputLen := inputLen
	if remainder != 0 {
		paddedInputLen = inputLen + (alignment - remainder)
	}
	encodedOutputLen = paddedInputLen / int(k)
	return encodedOutputLen
}

// Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
// Output is [k+m][]blocks of data and parity slices.
func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
	k := int(e.params.K) // "k" data blocks
	m := int(e.params.M) // "m" parity blocks
	n := k + m           // "n" total encoded blocks

	// Length of a single encoded chunk.
	// Total number of encoded chunks = "k" data  + "m" parity blocks
	encodedBlockLen := GetEncodedBlockLen(len(inputData), uint8(k))

	// Length of total number of "k" data chunks
	encodedDataBlocksLen := encodedBlockLen * k

	// Length of extra padding required for the data blocks.
	encodedDataBlocksPadLen := encodedDataBlocksLen - len(inputData)

	// Extend inputData buffer to accommodate coded data blocks if necesssary
	if encodedDataBlocksPadLen > 0 {
		padding := make([]byte, encodedDataBlocksPadLen)
		// Expand with new padded blocks to the byte array
		inputData = append(inputData, padding...)
	}

	// Extend inputData buffer to accommodate coded parity blocks
	{ // Local Scope
		encodedParityBlocksLen := encodedBlockLen * m
		parityBlocks := make([]byte, encodedParityBlocksLen)
		inputData = append(inputData, parityBlocks...)
	}

	// Allocate memory to the "encoded blocks" return buffer
	encodedBlocks = make([][]byte, n) // Return buffer

	// Neccessary to bridge Go to the C world. C requires 2D arry of pointers to
	// byte array. "encodedBlocks" is a 2D slice.
	pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.

	// Copy data block slices to encoded block buffer
	for i := 0; i < k; i++ {
		encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
	}

	// Copy erasure block slices to encoded block buffer
	for i := k; i < n; i++ {
		encodedBlocks[i] = make([]byte, encodedBlockLen)
		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
	}

	// Erasure code the data into K data blocks and M parity
	// blocks. Only the parity blocks are filled. Data blocks remain
	// intact.
	C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks

	return encodedBlocks, nil
}

// EncodeStream erasure codes a block of data in "k" data blocks and "m" parity blocks.
// Output is [k+m][]blocks of data and parity slices.
func (e *Erasure) EncodeStream(data io.Reader, size int64) ([][]byte, []byte, error) {
	k := int(e.params.K) // "k" data blocks
	m := int(e.params.M) // "m" parity blocks
	n := k + m           // "n" total encoded blocks

	// Length of a single encoded chunk.
	// Total number of encoded chunks = "k" data  + "m" parity blocks
	encodedBlockLen := GetEncodedBlockLen(int(size), uint8(k))

	// Length of total number of "n" data chunks
	encodedDataBlocksLen := encodedBlockLen * n

	// allocate byte array for encodedBlock length
	inputData := make([]byte, size, encodedDataBlocksLen)

	_, err := io.ReadFull(data, inputData)
	if err != nil {
		// do not check for io.ErrUnexpectedEOF, we know the right amount of size
		// to be read if its a short read we need to throw error since reader could
		// have been prematurely closed.
		if err != io.EOF {
			return nil, nil, err
		}
	}

	// Allocate memory to the "encoded blocks" return buffer
	encodedBlocks := make([][]byte, n) // Return buffer

	// Neccessary to bridge Go to the C world. C requires 2D arry of pointers to
	// byte array. "encodedBlocks" is a 2D slice.
	pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.

	// Copy data block slices to encoded block buffer
	for i := 0; i < n; i++ {
		encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
	}

	// Erasure code the data into K data blocks and M parity
	// blocks. Only the parity blocks are filled. Data blocks remain
	// intact.
	C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks

	return encodedBlocks, inputData[0:size], nil
}