mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Add crc32c checksum support for erasure chunks
Now upon conditions - if checksum '.cksum' file is missing - or corrupted chunk (mismatching chunk) - or missing chunk entirely Decoding works consistently
This commit is contained in:
parent
3fe2c2af01
commit
01c0e45b5c
@ -7,6 +7,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/codegangsta/cli"
|
"github.com/codegangsta/cli"
|
||||||
|
"github.com/minio-io/minio/pkgs/checksum/crc32c"
|
||||||
"github.com/minio-io/minio/pkgs/erasure"
|
"github.com/minio-io/minio/pkgs/erasure"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -58,8 +59,23 @@ func decode(c *cli.Context) {
|
|||||||
for _, inputFile := range inputFiles {
|
for _, inputFile := range inputFiles {
|
||||||
// get chunks
|
// get chunks
|
||||||
chunks := make([][]byte, k+m)
|
chunks := make([][]byte, k+m)
|
||||||
|
// get checksum
|
||||||
|
cksums := make([][]byte, k+m)
|
||||||
for i := 0; i < k+m; i++ {
|
for i := 0; i < k+m; i++ {
|
||||||
chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
|
chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
|
||||||
|
cksums[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i) + ".cksum")
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < k+m; i++ {
|
||||||
|
crcChunk, err := crc32c.Crc32c(chunks[i])
|
||||||
|
if err != nil {
|
||||||
|
chunks[i] = nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
crcChunkStr := strconv.Itoa(int(crcChunk))
|
||||||
|
if string(cksums[i]) != crcChunkStr {
|
||||||
|
chunks[i] = nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// get length
|
// get length
|
||||||
@ -67,6 +83,7 @@ func decode(c *cli.Context) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
lengthString := string(lengthBytes)
|
lengthString := string(lengthBytes)
|
||||||
length, err := strconv.Atoi(lengthString)
|
length, err := strconv.Atoi(lengthString)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -8,6 +8,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/codegangsta/cli"
|
"github.com/codegangsta/cli"
|
||||||
|
"github.com/minio-io/minio/pkgs/checksum/crc32c"
|
||||||
"github.com/minio-io/minio/pkgs/erasure"
|
"github.com/minio-io/minio/pkgs/erasure"
|
||||||
"github.com/minio-io/minio/pkgs/split"
|
"github.com/minio-io/minio/pkgs/split"
|
||||||
)
|
)
|
||||||
@ -47,6 +48,12 @@ func encode(c *cli.Context) {
|
|||||||
if config.blockSize == 0 {
|
if config.blockSize == 0 {
|
||||||
encodedData, length := encoder.Encode(input)
|
encodedData, length := encoder.Encode(input)
|
||||||
for key, data := range encodedData {
|
for key, data := range encodedData {
|
||||||
|
crc, err := crc32c.Crc32c(data)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
crcString := strconv.Itoa(int(crc))
|
||||||
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
|
||||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
|
||||||
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
|
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
|
||||||
}
|
}
|
||||||
@ -61,9 +68,15 @@ func encode(c *cli.Context) {
|
|||||||
}
|
}
|
||||||
encodedData, length := encoder.Encode(chunk.Data)
|
encodedData, length := encoder.Encode(chunk.Data)
|
||||||
for key, data := range encodedData {
|
for key, data := range encodedData {
|
||||||
|
crc, err := crc32c.Crc32c(data)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
crcString := strconv.Itoa(int(crc))
|
||||||
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
|
||||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
|
||||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
|
|
||||||
}
|
}
|
||||||
|
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
|
||||||
chunkCount += 1
|
chunkCount += 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user