mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
Merge pull request #57 from fkautz/pr_out_merging_encode_and_decode_to_single_minio_demo_binary
Merging encode and decode to single minio-demo binary
This commit is contained in:
Harshavardhana
commit
07e1199ae1
3
Makefile
3
Makefile
@ -13,8 +13,7 @@ test: build-erasure build-signify
|
||||
godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/gateway
|
||||
|
||||
install: build-erasure
|
||||
godep go install github.com/minio-io/minio/cmd/minio-encode
|
||||
godep go install github.com/minio-io/minio/cmd/minio-decode
|
||||
godep go install github.com/minio-io/minio/cmd/minio-demo
|
||||
|
||||
save:
|
||||
godep save ./...
|
||||
|
||||
@ -1,90 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/erasure"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := cli.NewApp()
|
||||
app.Name = "minio-decode"
|
||||
app.Usage = "erasure decode a byte stream"
|
||||
app.Flags = []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "input,i",
|
||||
Value: "",
|
||||
Usage: "Input file",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "output,o",
|
||||
Value: "",
|
||||
Usage: "Output file",
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "k",
|
||||
Value: 10,
|
||||
Usage: "k value of encoder parameters",
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "m",
|
||||
Value: 5,
|
||||
Usage: "m value of encoder parameters",
|
||||
},
|
||||
}
|
||||
app.Action = func(c *cli.Context) {
|
||||
// check if minio-encode called without parameters
|
||||
if len(c.Args()) == 1 {
|
||||
cli.ShowAppHelp(c)
|
||||
}
|
||||
|
||||
// get input path
|
||||
if c.String("input") == "" {
|
||||
log.Fatal("No input specified")
|
||||
}
|
||||
inputFilePath := c.String("input")
|
||||
|
||||
// get output path
|
||||
outputFilePath := inputFilePath
|
||||
if c.String("output") != "" {
|
||||
outputFilePath = c.String("output")
|
||||
}
|
||||
|
||||
k := c.Int("k")
|
||||
m := c.Int("m")
|
||||
|
||||
// get chunks
|
||||
chunks := make([][]byte, k+m)
|
||||
for i := 0; i < k+m; i++ {
|
||||
chunks[i], _ = ioutil.ReadFile(inputFilePath + "." + strconv.Itoa(i))
|
||||
}
|
||||
|
||||
// get length
|
||||
lengthBytes, err := ioutil.ReadFile(inputFilePath + ".length")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
lengthString := string(lengthBytes)
|
||||
length, err := strconv.Atoi(lengthString)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
|
||||
|
||||
// decode data
|
||||
decodedData, err := erasure.Decode(chunks, erasureParameters, length)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// write decode data out
|
||||
ioutil.WriteFile(outputFilePath, decodedData, 0600)
|
||||
}
|
||||
app.Run(os.Args)
|
||||
}
|
||||
78
cmd/minio-demo/decode.go
Normal file
78
cmd/minio-demo/decode.go
Normal file
@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/erasure"
|
||||
)
|
||||
|
||||
func decode(c *cli.Context) {
|
||||
// check if minio-encode called without parameters
|
||||
if len(c.Args()) != 1 {
|
||||
cli.ShowCommandHelp(c, "decode")
|
||||
return
|
||||
}
|
||||
|
||||
// get input path
|
||||
inputFilePath := c.Args().Get(0)
|
||||
|
||||
// get output path
|
||||
outputFilePath := inputFilePath
|
||||
if c.String("output") != "" {
|
||||
outputFilePath = c.String("output")
|
||||
}
|
||||
|
||||
protectionLevel := c.String("protection-level")
|
||||
protectionLevelSplit := strings.Split(protectionLevel, ",")
|
||||
if len(protectionLevelSplit) != 2 {
|
||||
log.Fatal("Malformed input for protection-level")
|
||||
}
|
||||
|
||||
k, err := strconv.Atoi(protectionLevelSplit[0])
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
m, err := strconv.Atoi(protectionLevelSplit[1])
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// get chunks
|
||||
chunks := make([][]byte, k+m)
|
||||
for i := 0; i < k+m; i++ {
|
||||
chunks[i], _ = ioutil.ReadFile(inputFilePath + "." + strconv.Itoa(i))
|
||||
}
|
||||
|
||||
// get length
|
||||
lengthBytes, err := ioutil.ReadFile(inputFilePath + ".length")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
lengthString := string(lengthBytes)
|
||||
length, err := strconv.Atoi(lengthString)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
|
||||
|
||||
// decode data
|
||||
decodedData, err := erasure.Decode(chunks, erasureParameters, length)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// write decode data out
|
||||
if _, err := os.Stat(outputFilePath); os.IsNotExist(err) {
|
||||
ioutil.WriteFile(outputFilePath, decodedData, 0600)
|
||||
} else {
|
||||
log.Fatal("Output file already exists")
|
||||
}
|
||||
}
|
||||
68
cmd/minio-demo/encode.go
Normal file
68
cmd/minio-demo/encode.go
Normal file
@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/erasure"
|
||||
)
|
||||
|
||||
func encode(c *cli.Context) {
|
||||
// check if minio-encode called without parameters
|
||||
if len(c.Args()) != 1 {
|
||||
cli.ShowCommandHelp(c, "encode")
|
||||
return
|
||||
}
|
||||
|
||||
// get input path
|
||||
inputFilePath := c.Args().Get(0)
|
||||
|
||||
// get output path
|
||||
outputFilePath := inputFilePath
|
||||
if c.String("output") != "" {
|
||||
outputFilePath = c.String("output")
|
||||
}
|
||||
|
||||
protectionLevel := c.String("protection-level")
|
||||
protectionLevelSplit := strings.Split(protectionLevel, ",")
|
||||
if len(protectionLevelSplit) != 2 {
|
||||
log.Fatal("Malformed input for protection-level")
|
||||
}
|
||||
|
||||
k, err := strconv.Atoi(protectionLevelSplit[0])
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
m, err := strconv.Atoi(protectionLevelSplit[1])
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// get file
|
||||
inputFile, err := os.Open(inputFilePath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// read file
|
||||
input, err := ioutil.ReadAll(inputFile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
|
||||
// encode data
|
||||
encodedData, length := erasure.Encode(input, erasureParameters)
|
||||
|
||||
// write encoded data out
|
||||
for key, data := range encodedData {
|
||||
ioutil.WriteFile(outputFilePath+"."+strconv.Itoa(key), data, 0600)
|
||||
}
|
||||
ioutil.WriteFile(outputFilePath+".length", []byte(strconv.Itoa(length)), 0600)
|
||||
}
|
||||
50
cmd/minio-demo/main.go
Normal file
50
cmd/minio-demo/main.go
Normal file
@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := cli.NewApp()
|
||||
app.Name = "minio-encode"
|
||||
app.Usage = "erasure encode a byte stream"
|
||||
app.Commands = []cli.Command{
|
||||
{
|
||||
Name: "encode",
|
||||
Usage: "erasure encode a byte stream",
|
||||
Action: encode,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "output,o",
|
||||
Value: "",
|
||||
Usage: "Output file",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,5",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "decode",
|
||||
Usage: "erasure decode a byte stream",
|
||||
Action: decode,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "output,o",
|
||||
Value: "",
|
||||
Usage: "Output file",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,5",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
app.Run(os.Args)
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/erasure"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := cli.NewApp()
|
||||
app.Name = "minio-encode"
|
||||
app.Usage = "erasure encode a byte stream"
|
||||
app.Flags = []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "input,i",
|
||||
Value: "",
|
||||
Usage: "Input file",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "output,o",
|
||||
Value: "",
|
||||
Usage: "Output file",
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "k",
|
||||
Value: 10,
|
||||
Usage: "k value of encoder parameters",
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "m",
|
||||
Value: 5,
|
||||
Usage: "m value of encoder parameters",
|
||||
},
|
||||
}
|
||||
app.Action = func(c *cli.Context) {
|
||||
// check if minio-encode called without parameters
|
||||
if len(c.Args()) == 1 {
|
||||
cli.ShowAppHelp(c)
|
||||
}
|
||||
|
||||
// get input path
|
||||
if c.String("input") == "" {
|
||||
log.Fatal("No input specified")
|
||||
}
|
||||
inputFilePath := c.String("input")
|
||||
|
||||
// get output path
|
||||
outputFilePath := inputFilePath
|
||||
if c.String("output") != "" {
|
||||
outputFilePath = c.String("output")
|
||||
}
|
||||
|
||||
k := c.Int("k")
|
||||
m := c.Int("m")
|
||||
|
||||
// get file
|
||||
inputFile, err := os.Open(inputFilePath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// read file
|
||||
input, err := ioutil.ReadAll(inputFile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
|
||||
// encode data
|
||||
encodedData, length := erasure.Encode(input, erasureParameters)
|
||||
|
||||
// write encoded data out
|
||||
for key, data := range encodedData {
|
||||
ioutil.WriteFile(outputFilePath+"."+strconv.Itoa(key), data, 0600)
|
||||
}
|
||||
ioutil.WriteFile(outputFilePath+".length", []byte(strconv.Itoa(length)), 0600)
|
||||
}
|
||||
app.Run(os.Args)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user