mirror of
https://github.com/minio/minio.git
synced 2025-11-09 13:39:46 -05:00
Fix bug in ErasureStorage.HealFile (#4913)
This commit is contained in:
Aditya Manthramurthy
committed by
Dee Koder
Dee Koder
parent
f7ae3be586
commit
3c0d3f7510
@@ -25,38 +25,51 @@ import (
|
||||
)
|
||||
|
||||
var erasureHealFileTests = []struct {
|
||||
dataBlocks int
|
||||
disks, offDisks, badDisks, badOffDisks int
|
||||
blocksize, size int64
|
||||
algorithm BitrotAlgorithm
|
||||
shouldFail bool
|
||||
shouldFailQuorum bool
|
||||
dataBlocks, disks int
|
||||
|
||||
// number of offline disks is also number of staleDisks for
|
||||
// erasure reconstruction in this test
|
||||
offDisks int
|
||||
|
||||
// bad disks are online disks which return errors
|
||||
badDisks, badStaleDisks int
|
||||
|
||||
blocksize, size int64
|
||||
algorithm BitrotAlgorithm
|
||||
shouldFail bool
|
||||
}{
|
||||
{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false}, // 0
|
||||
{dataBlocks: 3, disks: 6, offDisks: 2, badDisks: 0, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 1
|
||||
{dataBlocks: 4, disks: 8, offDisks: 2, badDisks: 1, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 2
|
||||
{dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 1, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 3
|
||||
{dataBlocks: 6, disks: 12, offDisks: 2, badDisks: 3, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false}, // 4
|
||||
{dataBlocks: 7, disks: 14, offDisks: 4, badDisks: 1, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 5
|
||||
{dataBlocks: 8, disks: 16, offDisks: 6, badDisks: 1, badOffDisks: 1, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 6
|
||||
{dataBlocks: 7, disks: 14, offDisks: 2, badDisks: 3, badOffDisks: 0, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: true, shouldFailQuorum: false}, // 7
|
||||
{dataBlocks: 6, disks: 12, offDisks: 1, badDisks: 0, badOffDisks: 1, blocksize: int64(oneMiByte - 1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false}, // 8
|
||||
{dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 0, badOffDisks: 3, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: SHA256, shouldFail: true, shouldFailQuorum: false}, // 9
|
||||
{dataBlocks: 4, disks: 8, offDisks: 1, badDisks: 1, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 10
|
||||
{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badOffDisks: 1, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 11
|
||||
{dataBlocks: 6, disks: 12, offDisks: 8, badDisks: 3, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 12
|
||||
{dataBlocks: 7, disks: 14, offDisks: 3, badDisks: 4, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 13
|
||||
{dataBlocks: 7, disks: 14, offDisks: 6, badDisks: 1, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 14
|
||||
{dataBlocks: 8, disks: 16, offDisks: 4, badDisks: 5, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 15
|
||||
{dataBlocks: 2, disks: 4, offDisks: 0, badDisks: 0, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 16
|
||||
{dataBlocks: 2, disks: 4, offDisks: 0, badDisks: 0, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: 0, shouldFail: true, shouldFailQuorum: false}, // 17
|
||||
{dataBlocks: 12, disks: 16, offDisks: 2, badDisks: 1, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 18
|
||||
{dataBlocks: 6, disks: 8, offDisks: 1, badDisks: 0, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 19
|
||||
{dataBlocks: 7, disks: 10, offDisks: 1, badDisks: 0, badOffDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: 0, shouldFail: true, shouldFailQuorum: false}, // 20
|
||||
{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: SHA256, shouldFail: false}, // 0
|
||||
{dataBlocks: 3, disks: 6, offDisks: 2, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 1
|
||||
{dataBlocks: 4, disks: 8, offDisks: 2, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 2
|
||||
{dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 3
|
||||
{dataBlocks: 6, disks: 12, offDisks: 2, badDisks: 3, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: SHA256, shouldFail: false}, // 4
|
||||
{dataBlocks: 7, disks: 14, offDisks: 4, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 5
|
||||
{dataBlocks: 8, disks: 16, offDisks: 6, badDisks: 1, badStaleDisks: 1, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 6
|
||||
{dataBlocks: 7, disks: 14, offDisks: 2, badDisks: 3, badStaleDisks: 0, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 7
|
||||
{dataBlocks: 6, disks: 12, offDisks: 1, badDisks: 0, badStaleDisks: 1, blocksize: int64(oneMiByte - 1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 8
|
||||
{dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 0, badStaleDisks: 3, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: SHA256, shouldFail: true}, // 9
|
||||
{dataBlocks: 4, disks: 8, offDisks: 1, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 10
|
||||
{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 1, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 11
|
||||
{dataBlocks: 6, disks: 12, offDisks: 8, badDisks: 3, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 12
|
||||
{dataBlocks: 7, disks: 14, offDisks: 3, badDisks: 4, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 13
|
||||
{dataBlocks: 7, disks: 14, offDisks: 6, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 14
|
||||
{dataBlocks: 8, disks: 16, offDisks: 4, badDisks: 5, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 15
|
||||
{dataBlocks: 2, disks: 4, offDisks: 0, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 16
|
||||
{dataBlocks: 2, disks: 4, offDisks: 0, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: 0, shouldFail: true}, // 17
|
||||
{dataBlocks: 12, disks: 16, offDisks: 2, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 18
|
||||
{dataBlocks: 6, disks: 8, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 19
|
||||
{dataBlocks: 7, disks: 10, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: 0, shouldFail: true}, // 20
|
||||
{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte * 64, algorithm: SHA256, shouldFail: false}, // 21
|
||||
}
|
||||
|
||||
func TestErasureHealFile(t *testing.T) {
|
||||
for i, test := range erasureHealFileTests {
|
||||
if test.offDisks < test.badStaleDisks {
|
||||
// test case sanity check
|
||||
t.Fatalf("Test %d: Bad test case - number of stale disks cannot be less than number of badstale disks", i)
|
||||
}
|
||||
|
||||
// create some test data
|
||||
setup, err := newErasureTestSetup(test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
|
||||
@@ -66,15 +79,11 @@ func TestErasureHealFile(t *testing.T) {
|
||||
setup.Remove()
|
||||
t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
|
||||
}
|
||||
offline := make([]StorageAPI, len(storage.disks))
|
||||
copy(offline, storage.disks)
|
||||
|
||||
data := make([]byte, test.size)
|
||||
if _, err = io.ReadFull(rand.Reader, data); err != nil {
|
||||
setup.Remove()
|
||||
t.Fatalf("Test %d: failed to create random test data: %v", i, err)
|
||||
}
|
||||
|
||||
algorithm := test.algorithm
|
||||
if !algorithm.Available() {
|
||||
algorithm = DefaultBitrotAlgorithm
|
||||
@@ -86,7 +95,25 @@ func TestErasureHealFile(t *testing.T) {
|
||||
t.Fatalf("Test %d: failed to create random test data: %v", i, err)
|
||||
}
|
||||
|
||||
info, err := storage.HealFile(offline, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums)
|
||||
// setup stale disks for the test case
|
||||
staleDisks := make([]StorageAPI, len(storage.disks))
|
||||
copy(staleDisks, storage.disks)
|
||||
for j := 0; j < len(storage.disks); j++ {
|
||||
if j < test.offDisks {
|
||||
storage.disks[j] = OfflineDisk
|
||||
} else {
|
||||
staleDisks[j] = nil
|
||||
}
|
||||
}
|
||||
for j := 0; j < test.badDisks; j++ {
|
||||
storage.disks[test.offDisks+j] = badDisk{nil}
|
||||
}
|
||||
for j := 0; j < test.badStaleDisks; j++ {
|
||||
staleDisks[j] = badDisk{nil}
|
||||
}
|
||||
|
||||
// test case setup is complete - now call Healfile()
|
||||
info, err := storage.HealFile(staleDisks, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums)
|
||||
if err != nil && !test.shouldFail {
|
||||
t.Errorf("Test %d: should pass but it failed with: %v", i, err)
|
||||
}
|
||||
@@ -100,39 +127,13 @@ func TestErasureHealFile(t *testing.T) {
|
||||
if info.Algorithm != test.algorithm {
|
||||
t.Errorf("Test %d: healed with wrong algorithm: got: %v want: %v", i, info.Algorithm, test.algorithm)
|
||||
}
|
||||
if !reflect.DeepEqual(info.Checksums, file.Checksums) {
|
||||
t.Errorf("Test %d: heal returned different bitrot keys", i)
|
||||
}
|
||||
}
|
||||
if err == nil && !test.shouldFail {
|
||||
for j := 0; j < len(storage.disks); j++ {
|
||||
if j < test.offDisks {
|
||||
storage.disks[j] = OfflineDisk
|
||||
} else {
|
||||
offline[j] = OfflineDisk
|
||||
// Verify that checksums of staleDisks
|
||||
// match expected values
|
||||
for i, disk := range staleDisks {
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
for j := 0; j < test.badDisks; j++ {
|
||||
storage.disks[test.offDisks+j] = badDisk{nil}
|
||||
}
|
||||
for j := 0; j < test.badOffDisks; j++ {
|
||||
offline[j] = badDisk{nil}
|
||||
}
|
||||
info, err := storage.HealFile(offline, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums)
|
||||
if err != nil && !test.shouldFailQuorum {
|
||||
t.Errorf("Test %d: should pass but it failed with: %v", i, err)
|
||||
}
|
||||
if err == nil && test.shouldFailQuorum {
|
||||
t.Errorf("Test %d: should fail but it passed", i)
|
||||
}
|
||||
if err == nil {
|
||||
if info.Size != test.size {
|
||||
t.Errorf("Test %d: healed wrong number of bytes: got: #%d want: #%d", i, info.Size, test.size)
|
||||
}
|
||||
if info.Algorithm != test.algorithm {
|
||||
t.Errorf("Test %d: healed with wrong algorithm: got: %v want: %v", i, info.Algorithm, test.algorithm)
|
||||
}
|
||||
if !reflect.DeepEqual(info.Checksums, file.Checksums) {
|
||||
if !reflect.DeepEqual(info.Checksums[i], file.Checksums[i]) {
|
||||
t.Errorf("Test %d: heal returned different bitrot checksums", i)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user