mirror of
https://github.com/minio/minio.git
synced 2025-11-07 12:52:58 -05:00
This change provides new implementations of the XL backend operations: - create file - read file - heal file Further this change adds table based tests for all three operations. This affects also the bitrot algorithm integration. Algorithms are now integrated in an idiomatic way (like crypto.Hash). Fixes #4696 Fixes #4649 Fixes #4359
This commit is contained in:
Andreas Auernhammer
committed by
Dee Koder
Dee Koder
parent
617f2394fb
commit
85fcee1919
@@ -18,161 +18,91 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mustEncodeData - encodes data slice and provides encoded 2 dimensional slice.
|
||||
func mustEncodeData(data []byte, dataBlocks, parityBlocks int) [][]byte {
|
||||
encodedData, err := encodeData(data, dataBlocks, parityBlocks)
|
||||
if err != nil {
|
||||
// Upon failure panic this function.
|
||||
panic(err)
|
||||
}
|
||||
return encodedData
|
||||
var erasureDecodeTests = []struct {
|
||||
dataBlocks, parityBlocks int
|
||||
missingData, missingParity int
|
||||
reconstructParity bool
|
||||
shouldFail bool
|
||||
}{
|
||||
{dataBlocks: 2, parityBlocks: 2, missingData: 0, missingParity: 0, reconstructParity: true, shouldFail: false},
|
||||
{dataBlocks: 3, parityBlocks: 3, missingData: 1, missingParity: 0, reconstructParity: true, shouldFail: false},
|
||||
{dataBlocks: 4, parityBlocks: 4, missingData: 2, missingParity: 0, reconstructParity: false, shouldFail: false},
|
||||
{dataBlocks: 5, parityBlocks: 5, missingData: 0, missingParity: 1, reconstructParity: true, shouldFail: false},
|
||||
{dataBlocks: 6, parityBlocks: 6, missingData: 0, missingParity: 2, reconstructParity: true, shouldFail: false},
|
||||
{dataBlocks: 7, parityBlocks: 7, missingData: 1, missingParity: 1, reconstructParity: false, shouldFail: false},
|
||||
{dataBlocks: 8, parityBlocks: 8, missingData: 3, missingParity: 2, reconstructParity: false, shouldFail: false},
|
||||
{dataBlocks: 2, parityBlocks: 2, missingData: 2, missingParity: 1, reconstructParity: true, shouldFail: true},
|
||||
{dataBlocks: 4, parityBlocks: 2, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: true},
|
||||
{dataBlocks: 8, parityBlocks: 4, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: false},
|
||||
}
|
||||
|
||||
// Generates good encoded data with one parity block and data block missing.
|
||||
func getGoodEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
|
||||
encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
|
||||
encodedData[3] = nil
|
||||
encodedData[1] = nil
|
||||
return encodedData
|
||||
}
|
||||
|
||||
// Generates bad encoded data with one parity block and data block with garbage data.
|
||||
func getBadEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
|
||||
encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
|
||||
encodedData[3] = []byte("garbage")
|
||||
encodedData[1] = []byte("garbage")
|
||||
return encodedData
|
||||
}
|
||||
|
||||
// Generates encoded data with all data blocks missing.
|
||||
func getMissingData(data []byte, dataBlocks, parityBlocks int) [][]byte {
|
||||
encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
|
||||
for i := 0; i < dataBlocks+1; i++ {
|
||||
encodedData[i] = nil
|
||||
}
|
||||
return encodedData
|
||||
}
|
||||
|
||||
// Generates encoded data with less number of blocks than expected data blocks.
|
||||
func getInsufficientData(data []byte, dataBlocks, parityBlocks int) [][]byte {
|
||||
encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
|
||||
// Return half of the data.
|
||||
return encodedData[:dataBlocks/2]
|
||||
}
|
||||
|
||||
// Represents erasure encoding matrix dataBlocks and paritBlocks.
|
||||
type encodingMatrix struct {
|
||||
dataBlocks int
|
||||
parityBlocks int
|
||||
}
|
||||
|
||||
// List of encoding matrices the tests will run on.
|
||||
var encodingMatrices = []encodingMatrix{
|
||||
{3, 3}, // 3 data, 3 parity blocks.
|
||||
{4, 4}, // 4 data, 4 parity blocks.
|
||||
{5, 5}, // 5 data, 5 parity blocks.
|
||||
{6, 6}, // 6 data, 6 parity blocks.
|
||||
{7, 7}, // 7 data, 7 parity blocks.
|
||||
{8, 8}, // 8 data, 8 parity blocks.
|
||||
}
|
||||
|
||||
// Tests erasure decoding functionality for various types of inputs.
|
||||
func TestErasureDecode(t *testing.T) {
|
||||
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
|
||||
|
||||
// List of decoding cases
|
||||
// - validates bad encoded data.
|
||||
// - validates good encoded data.
|
||||
// - validates insufficient data.
|
||||
testDecodeCases := []struct {
|
||||
enFn func([]byte, int, int) [][]byte
|
||||
shouldPass bool
|
||||
}{
|
||||
// Generates bad encoded data.
|
||||
{
|
||||
enFn: getBadEncodedData,
|
||||
shouldPass: false,
|
||||
},
|
||||
// Generates good encoded data.
|
||||
{
|
||||
enFn: getGoodEncodedData,
|
||||
shouldPass: true,
|
||||
},
|
||||
// Generates missing data.
|
||||
{
|
||||
enFn: getMissingData,
|
||||
shouldPass: false,
|
||||
},
|
||||
// Generates short data.
|
||||
{
|
||||
enFn: getInsufficientData,
|
||||
shouldPass: false,
|
||||
},
|
||||
data := make([]byte, 256)
|
||||
if _, err := io.ReadFull(rand.Reader, data); err != nil {
|
||||
t.Fatalf("Failed to read random data: %v", err)
|
||||
}
|
||||
for i, test := range erasureDecodeTests {
|
||||
buffer := make([]byte, len(data), 2*len(data))
|
||||
copy(buffer, data)
|
||||
|
||||
// Validates all decode tests.
|
||||
for i, testCase := range testDecodeCases {
|
||||
for _, encodingMatrix := range encodingMatrices {
|
||||
disks := make([]StorageAPI, test.dataBlocks+test.parityBlocks)
|
||||
storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to create erasure storage: %v", i, err)
|
||||
}
|
||||
encoded, err := storage.ErasureEncode(buffer)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to encode data: %v", i, err)
|
||||
}
|
||||
|
||||
// Encoding matrix.
|
||||
dataBlocks := encodingMatrix.dataBlocks
|
||||
parityBlocks := encodingMatrix.parityBlocks
|
||||
for j := range encoded[:test.missingData] {
|
||||
encoded[j] = nil
|
||||
}
|
||||
for j := test.dataBlocks; j < test.dataBlocks+test.missingParity; j++ {
|
||||
encoded[j] = nil
|
||||
}
|
||||
|
||||
// Data block size.
|
||||
blockSize := len(data)
|
||||
if test.reconstructParity {
|
||||
err = storage.ErasureDecodeDataAndParityBlocks(encoded)
|
||||
} else {
|
||||
err = storage.ErasureDecodeDataBlocks(encoded)
|
||||
}
|
||||
|
||||
// Test decoder for just the missing data blocks
|
||||
{
|
||||
// Generates encoded data based on type of testCase function.
|
||||
encodedData := testCase.enFn(data, dataBlocks, parityBlocks)
|
||||
if err == nil && test.shouldFail {
|
||||
t.Errorf("Test %d: test should fail but it passed", i)
|
||||
}
|
||||
if err != nil && !test.shouldFail {
|
||||
t.Errorf("Test %d: test should pass but it failed: %v", i, err)
|
||||
}
|
||||
|
||||
// Decodes the data.
|
||||
err := decodeMissingData(encodedData, dataBlocks, parityBlocks)
|
||||
if err != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
|
||||
decoded := encoded
|
||||
if !test.shouldFail {
|
||||
if test.reconstructParity {
|
||||
for j := range decoded {
|
||||
if decoded[j] == nil {
|
||||
t.Errorf("Test %d: failed to reconstruct shard %d", i, j)
|
||||
}
|
||||
}
|
||||
|
||||
// Proceed to extract the data blocks.
|
||||
decodedDataWriter := new(bytes.Buffer)
|
||||
_, err = writeDataBlocks(decodedDataWriter, encodedData, dataBlocks, 0, int64(blockSize))
|
||||
if err != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
|
||||
}
|
||||
|
||||
// Validate if decoded data is what we expected.
|
||||
if bytes.Equal(decodedDataWriter.Bytes(), data) != testCase.shouldPass {
|
||||
err := errUnexpected
|
||||
t.Errorf("Test %d: Expected to pass by failed instead %s", i+1, err)
|
||||
} else {
|
||||
for j := range decoded[:test.dataBlocks] {
|
||||
if decoded[j] == nil {
|
||||
t.Errorf("Test %d: failed to reconstruct data shard %d", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test decoder for all missing data and parity blocks
|
||||
{
|
||||
// Generates encoded data based on type of testCase function.
|
||||
encodedData := testCase.enFn(data, dataBlocks, parityBlocks)
|
||||
|
||||
// Decodes the data.
|
||||
err := decodeDataAndParity(encodedData, dataBlocks, parityBlocks)
|
||||
if err != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
|
||||
}
|
||||
|
||||
// Proceed to extract the data blocks.
|
||||
decodedDataWriter := new(bytes.Buffer)
|
||||
_, err = writeDataBlocks(decodedDataWriter, encodedData, dataBlocks, 0, int64(blockSize))
|
||||
if err != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
|
||||
}
|
||||
|
||||
// Validate if decoded data is what we expected.
|
||||
if bytes.Equal(decodedDataWriter.Bytes(), data) != testCase.shouldPass {
|
||||
err := errUnexpected
|
||||
t.Errorf("Test %d: Expected to pass by failed instead %s", i+1, err)
|
||||
}
|
||||
decodedData := new(bytes.Buffer)
|
||||
if _, err = writeDataBlocks(decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil {
|
||||
t.Errorf("Test %d: failed to write data blocks: %v", i, err)
|
||||
}
|
||||
if !bytes.Equal(decodedData.Bytes(), data) {
|
||||
t.Errorf("Test %d: Decoded data does not match original data: got: %v want: %v", i, decodedData.Bytes(), data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user