mirror of
https://github.com/minio/minio.git
synced 2024-12-23 21:55:53 -05:00
Adding chunking by block to erasure-demo via --block-size parameter
This commit is contained in:
parent
59c1197f47
commit
46b08681a4
@ -24,36 +24,68 @@ func decode(c *cli.Context) {
|
||||
|
||||
k := config.k
|
||||
m := config.m
|
||||
// get chunks
|
||||
chunks := make([][]byte, k+m)
|
||||
for i := 0; i < k+m; i++ {
|
||||
chunks[i], _ = ioutil.ReadFile(config.input + "." + strconv.Itoa(i))
|
||||
|
||||
// check if output file exists, fail if so
|
||||
if _, err := os.Stat(config.output); !os.IsNotExist(err) {
|
||||
log.Fatal("Output file exists")
|
||||
}
|
||||
|
||||
// get length
|
||||
lengthBytes, err := ioutil.ReadFile(config.input + ".length")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
lengthString := string(lengthBytes)
|
||||
length, err := strconv.Atoi(lengthString)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
|
||||
|
||||
// decode data
|
||||
decodedData, err := erasure.Decode(chunks, erasureParameters, length)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// write decode data out
|
||||
if _, err := os.Stat(config.output); os.IsNotExist(err) {
|
||||
ioutil.WriteFile(config.output, decodedData, 0600)
|
||||
// get list of files
|
||||
var inputFiles []string
|
||||
if _, err := os.Stat(config.input + ".length"); os.IsNotExist(err) {
|
||||
err = nil
|
||||
chunkCount := 0
|
||||
for !os.IsNotExist(err) {
|
||||
_, err = os.Stat(config.input + "." + strconv.Itoa(chunkCount) + ".length")
|
||||
chunkCount += 1
|
||||
}
|
||||
chunkCount = chunkCount - 1
|
||||
inputFiles = make([]string, chunkCount)
|
||||
for i := 0; i < chunkCount; i++ {
|
||||
inputFiles[i] = config.input + "." + strconv.Itoa(i)
|
||||
}
|
||||
} else {
|
||||
log.Fatal("Output file already exists")
|
||||
inputFiles = []string{config.input}
|
||||
}
|
||||
|
||||
// open file to write
|
||||
outputFile, err := os.OpenFile(config.output, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
defer outputFile.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for _, inputFile := range inputFiles {
|
||||
// get chunks
|
||||
chunks := make([][]byte, k+m)
|
||||
for i := 0; i < k+m; i++ {
|
||||
chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
|
||||
}
|
||||
|
||||
// get length
|
||||
lengthBytes, err := ioutil.ReadFile(inputFile + ".length")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
lengthString := string(lengthBytes)
|
||||
length, err := strconv.Atoi(lengthString)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
|
||||
|
||||
// decode data
|
||||
decodedData, err := erasure.Decode(chunks, erasureParameters, length)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
// append decoded data
|
||||
length, err = outputFile.Write(decodedData)
|
||||
if err != nil {
|
||||
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
@ -8,6 +9,7 @@ import (
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/erasure"
|
||||
"github.com/minio-io/minio/pkgs/split"
|
||||
)
|
||||
|
||||
func encode(c *cli.Context) {
|
||||
@ -24,6 +26,7 @@ func encode(c *cli.Context) {
|
||||
|
||||
// get file
|
||||
inputFile, err := os.Open(config.input)
|
||||
defer inputFile.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@ -37,11 +40,27 @@ func encode(c *cli.Context) {
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
|
||||
// encode data
|
||||
encodedData, length := erasure.Encode(input, erasureParameters)
|
||||
|
||||
// write encoded data out
|
||||
for key, data := range encodedData {
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
|
||||
if config.blockSize == 0 {
|
||||
encodedData, length := erasure.Encode(input, erasureParameters)
|
||||
for key, data := range encodedData {
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
|
||||
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
|
||||
}
|
||||
} else {
|
||||
chunkCount := 0
|
||||
splitChannel := make(chan split.ByteMessage)
|
||||
inputReader := bytes.NewReader(input)
|
||||
go split.SplitStream(inputReader, config.blockSize, splitChannel)
|
||||
for chunk := range splitChannel {
|
||||
if chunk.Err != nil {
|
||||
log.Fatal(chunk.Err)
|
||||
}
|
||||
encodedData, length := erasure.Encode(chunk.Data, erasureParameters)
|
||||
for key, data := range encodedData {
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
|
||||
}
|
||||
chunkCount += 1
|
||||
}
|
||||
}
|
||||
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
|
||||
}
|
||||
|
@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/strbyteconv"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -29,6 +30,11 @@ func main() {
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -58,7 +64,7 @@ type inputConfig struct {
|
||||
output string
|
||||
k int
|
||||
m int
|
||||
blockSize int
|
||||
blockSize uint64
|
||||
}
|
||||
|
||||
// parses input and returns an inputConfig with parsed input
|
||||
@ -88,10 +94,22 @@ func parseInput(c *cli.Context) (inputConfig, error) {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
|
||||
var blockSize uint64
|
||||
blockSize = 0
|
||||
if c.String("block-size") != "" {
|
||||
if c.String("block-size") != "full" {
|
||||
blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return inputConfig{
|
||||
input: inputFilePath,
|
||||
output: outputFilePath,
|
||||
k: k,
|
||||
m: m,
|
||||
input: inputFilePath,
|
||||
output: outputFilePath,
|
||||
k: k,
|
||||
m: m,
|
||||
blockSize: blockSize,
|
||||
}, nil
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user