mirror of
https://github.com/minio/minio.git
synced 2024-12-24 22:25:54 -05:00
Merge pull request #128 from harshavardhana/pr_out_implement_commands
This commit is contained in:
commit
1f8806b44f
2
Makefile
2
Makefile
@ -44,7 +44,7 @@ cover: build-erasure build-signify build-split build-crc32c build-cpu build-sha1
|
||||
@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/gateway
|
||||
|
||||
install: build-erasure
|
||||
@godep go install github.com/minio-io/minio/cmd/erasure-demo && echo "Installed erasure-demo into ${GOPATH}/bin"
|
||||
@godep go install github.com/minio-io/minio/cmd/minio && echo "Installed minio into ${GOPATH}/bin"
|
||||
|
||||
save: restore
|
||||
@godep save ./...
|
||||
|
@ -1,112 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/checksum/crc32c"
|
||||
"github.com/minio-io/minio/pkgs/erasure"
|
||||
)
|
||||
|
||||
func decode(c *cli.Context) {
|
||||
// check if minio-encode called without parameters
|
||||
if len(c.Args()) != 1 {
|
||||
cli.ShowCommandHelp(c, "decode")
|
||||
return
|
||||
}
|
||||
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
k := config.k
|
||||
m := config.m
|
||||
|
||||
// check if output file exists, fail if so
|
||||
if _, err := os.Stat(config.output); !os.IsNotExist(err) {
|
||||
log.Fatal("Output file exists")
|
||||
}
|
||||
|
||||
// get list of files
|
||||
var inputFiles []string
|
||||
if _, err := os.Stat(config.input + ".length"); os.IsNotExist(err) {
|
||||
err = nil
|
||||
chunkCount := 0
|
||||
for !os.IsNotExist(err) {
|
||||
_, err = os.Stat(config.input + "." + strconv.Itoa(chunkCount) + ".length")
|
||||
chunkCount += 1
|
||||
}
|
||||
chunkCount = chunkCount - 1
|
||||
inputFiles = make([]string, chunkCount)
|
||||
for i := 0; i < chunkCount; i++ {
|
||||
inputFiles[i] = config.input + "." + strconv.Itoa(i)
|
||||
}
|
||||
} else {
|
||||
inputFiles = []string{config.input}
|
||||
}
|
||||
|
||||
// open file to write
|
||||
outputFile, err := os.OpenFile(config.output, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
defer outputFile.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for _, inputFile := range inputFiles {
|
||||
// get chunks
|
||||
chunks := make([][]byte, k+m)
|
||||
// get checksum
|
||||
cksums := make([][]byte, k+m)
|
||||
for i := 0; i < k+m; i++ {
|
||||
chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
|
||||
cksums[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i) + ".cksum")
|
||||
}
|
||||
|
||||
for i := 0; i < k+m; i++ {
|
||||
crcChunk, err := crc32c.Crc32c(chunks[i])
|
||||
if err != nil {
|
||||
chunks[i] = nil
|
||||
continue
|
||||
}
|
||||
crcChunkStr := strconv.Itoa(int(crcChunk))
|
||||
if string(cksums[i]) != crcChunkStr {
|
||||
chunks[i] = nil
|
||||
}
|
||||
}
|
||||
|
||||
// get length
|
||||
lengthBytes, err := ioutil.ReadFile(inputFile + ".length")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
lengthString := string(lengthBytes)
|
||||
length, err := strconv.Atoi(lengthString)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
|
||||
|
||||
// Get decoder
|
||||
encoder := erasure.NewEncoder(erasureParameters)
|
||||
|
||||
// decode data
|
||||
decodedData, err := encoder.Decode(chunks, length)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// append decoded data
|
||||
length, err = outputFile.Write(decodedData)
|
||||
if err != nil {
|
||||
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
@ -1,83 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/checksum/crc32c"
|
||||
"github.com/minio-io/minio/pkgs/erasure"
|
||||
"github.com/minio-io/minio/pkgs/split"
|
||||
)
|
||||
|
||||
func encode(c *cli.Context) {
|
||||
// check if minio-encode called without parameters
|
||||
if len(c.Args()) != 1 {
|
||||
cli.ShowCommandHelp(c, "encode")
|
||||
return
|
||||
}
|
||||
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// get file
|
||||
inputFile, err := os.Open(config.input)
|
||||
defer inputFile.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// read file
|
||||
input, err := ioutil.ReadAll(inputFile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
|
||||
|
||||
// Init new encoder
|
||||
encoder := erasure.NewEncoder(erasureParameters)
|
||||
|
||||
// encode data
|
||||
if config.blockSize == 0 {
|
||||
encodedData, length := encoder.Encode(input)
|
||||
for key, data := range encodedData {
|
||||
crc, err := crc32c.Crc32c(data)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
crcString := strconv.Itoa(int(crc))
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
|
||||
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
|
||||
}
|
||||
} else {
|
||||
chunkCount := 0
|
||||
splitChannel := make(chan split.SplitMessage)
|
||||
inputReader := bytes.NewReader(input)
|
||||
go split.SplitStream(inputReader, config.blockSize, splitChannel)
|
||||
for chunk := range splitChannel {
|
||||
if chunk.Err != nil {
|
||||
log.Fatal(chunk.Err)
|
||||
}
|
||||
encodedData, length := encoder.Encode(chunk.Data)
|
||||
for key, data := range encodedData {
|
||||
crc, err := crc32c.Crc32c(data)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
crcString := strconv.Itoa(int(crc))
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
|
||||
}
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
|
||||
chunkCount += 1
|
||||
}
|
||||
}
|
||||
}
|
@ -1,41 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
)
|
||||
|
||||
func get(c *cli.Context) {
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
var objectReader io.Reader
|
||||
objectName := c.Args().Get(0)
|
||||
|
||||
switch config.storageDriver {
|
||||
case "erasure":
|
||||
{
|
||||
if len(objectName) == 0 {
|
||||
if objectReader, err = erasureGetList(config, ""); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
if objectReader, err = erasureGet(config, objectName); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
{
|
||||
log.Fatal("Unknown driver")
|
||||
}
|
||||
}
|
||||
if objectReader == nil {
|
||||
log.Fatal("Object not found")
|
||||
}
|
||||
io.Copy(os.Stdout, objectReader)
|
||||
}
|
@ -1,218 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/strbyteconv"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := cli.NewApp()
|
||||
app.Name = "erasure-demo"
|
||||
app.Usage = "erasure encode a byte stream"
|
||||
app.Commands = []cli.Command{
|
||||
{
|
||||
Name: "encode",
|
||||
Usage: "erasure encode a byte stream",
|
||||
Action: encode,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "output,o",
|
||||
Value: "",
|
||||
Usage: "Output file",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "decode",
|
||||
Usage: "erasure decode a byte stream",
|
||||
Action: decode,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "output,o",
|
||||
Value: "",
|
||||
Usage: "Output file",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "get",
|
||||
Usage: "get an object",
|
||||
Action: get,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: getMinioDir(),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "driver",
|
||||
Value: "erasure",
|
||||
Usage: "erasure",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "put",
|
||||
Usage: "put an object",
|
||||
Action: put,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: getMinioDir(),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "driver",
|
||||
Value: "erasure",
|
||||
Usage: "erasure",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
app.Run(os.Args)
|
||||
}
|
||||
|
||||
// config representing cli input
|
||||
type inputConfig struct {
|
||||
input string
|
||||
output string
|
||||
k int
|
||||
m int
|
||||
blockSize uint64
|
||||
rootDir string
|
||||
storageDriver string
|
||||
}
|
||||
|
||||
// parses input and returns an inputConfig with parsed input
|
||||
func parseInput(c *cli.Context) (inputConfig, error) {
|
||||
// get input path
|
||||
inputFilePath := c.Args().Get(0)
|
||||
|
||||
// get output path
|
||||
outputFilePath := inputFilePath
|
||||
if c.String("output") != "" {
|
||||
outputFilePath = c.String("output")
|
||||
}
|
||||
|
||||
var k, m int
|
||||
if c.String("protection-level") != "" {
|
||||
protectionLevel := c.String("protection-level")
|
||||
protectionLevelSplit := strings.Split(protectionLevel, ",")
|
||||
if len(protectionLevelSplit) != 2 {
|
||||
return inputConfig{}, errors.New("Malformed input for protection-level")
|
||||
}
|
||||
|
||||
var err error
|
||||
k, err = strconv.Atoi(protectionLevelSplit[0])
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
m, err = strconv.Atoi(protectionLevelSplit[1])
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var blockSize uint64
|
||||
blockSize = 0
|
||||
if c.String("block-size") != "" {
|
||||
if c.String("block-size") != "full" {
|
||||
var err error
|
||||
blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var rootDir string
|
||||
if c.String("root") != "" {
|
||||
rootDir = c.String("root")
|
||||
}
|
||||
|
||||
var storageDriver string
|
||||
if c.String("driver") != "" {
|
||||
storageDriver = c.String("driver")
|
||||
}
|
||||
|
||||
config := inputConfig{
|
||||
input: inputFilePath,
|
||||
output: outputFilePath,
|
||||
k: k,
|
||||
m: m,
|
||||
blockSize: blockSize,
|
||||
rootDir: rootDir,
|
||||
storageDriver: storageDriver,
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func getMinioDir() string {
|
||||
user, err := user.Current()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
homePath := user.HomeDir
|
||||
minioPath := path.Join(homePath, ".minio")
|
||||
err = _initMinioDir(minioPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return minioPath
|
||||
}
|
||||
|
||||
func _initMinioDir(dirPath string) error {
|
||||
_, err := os.Lstat(dirPath)
|
||||
if err != nil {
|
||||
log.Printf("%s not found, creating a new-one for the first time",
|
||||
dirPath)
|
||||
err = os.Mkdir(dirPath, 0700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
95
cmd/minio/minio.go
Normal file
95
cmd/minio/minio.go
Normal file
@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/minio"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := cli.NewApp()
|
||||
app.Name = "minio"
|
||||
app.Usage = "minio - object storage"
|
||||
app.Commands = []cli.Command{
|
||||
{
|
||||
Name: "encode",
|
||||
Usage: "erasure encode a byte stream",
|
||||
Action: minio.Encode,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: minio.Getobjectdir(".minio/erasure"),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "staging",
|
||||
Value: minio.Getobjectdir(".minio/staging"),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "get",
|
||||
Usage: "get an object",
|
||||
Action: minio.Get,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: minio.Getobjectdir(".minio/erasure"),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "put",
|
||||
Usage: "put an object",
|
||||
Action: minio.Put,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "staging",
|
||||
Value: minio.Getobjectdir(".minio/staging"),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "list",
|
||||
Usage: "list objects",
|
||||
Action: minio.List,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: minio.Getobjectdir(".minio/erasure"),
|
||||
Usage: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
app.Run(os.Args)
|
||||
}
|
103
pkgs/minio/common.go
Normal file
103
pkgs/minio/common.go
Normal file
@ -0,0 +1,103 @@
|
||||
package minio
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/strbyteconv"
|
||||
)
|
||||
|
||||
// config representing cli input
|
||||
type inputConfig struct {
|
||||
k int
|
||||
m int
|
||||
blockSize uint64
|
||||
rootDir string
|
||||
stagingDir string
|
||||
}
|
||||
|
||||
// parses input and returns an inputConfig with parsed input
|
||||
func parseInput(c *cli.Context) (inputConfig, error) {
|
||||
var k, m int
|
||||
if c.String("protection-level") != "" {
|
||||
protectionLevel := c.String("protection-level")
|
||||
protectionLevelSplit := strings.Split(protectionLevel, ",")
|
||||
if len(protectionLevelSplit) != 2 {
|
||||
return inputConfig{}, errors.New("Malformed input for protection-level")
|
||||
}
|
||||
|
||||
var err error
|
||||
k, err = strconv.Atoi(protectionLevelSplit[0])
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
m, err = strconv.Atoi(protectionLevelSplit[1])
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var blockSize uint64
|
||||
blockSize = 0
|
||||
if c.String("block-size") != "" {
|
||||
if c.String("block-size") != "full" {
|
||||
var err error
|
||||
blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var rootDir string
|
||||
if c.String("root") != "" {
|
||||
rootDir = c.String("root")
|
||||
}
|
||||
|
||||
var stagingDir string
|
||||
if c.String("staging") != "" {
|
||||
stagingDir = c.String("staging")
|
||||
}
|
||||
|
||||
config := inputConfig{
|
||||
k: k,
|
||||
m: m,
|
||||
blockSize: blockSize,
|
||||
rootDir: rootDir,
|
||||
stagingDir: stagingDir,
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func Getobjectdir(basename string) string {
|
||||
user, err := user.Current()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
homePath := user.HomeDir
|
||||
minioPath := path.Join(homePath, basename)
|
||||
err = _initDir(minioPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return minioPath
|
||||
}
|
||||
|
||||
func _initDir(dirPath string) error {
|
||||
_, err := os.Lstat(dirPath)
|
||||
if err != nil {
|
||||
log.Printf("%s not found, creating a new-one for the first time",
|
||||
dirPath)
|
||||
err = os.MkdirAll(dirPath, 0700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
37
pkgs/minio/encode.go
Normal file
37
pkgs/minio/encode.go
Normal file
@ -0,0 +1,37 @@
|
||||
package minio
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
)
|
||||
|
||||
func Encode(c *cli.Context) {
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
var objectName string
|
||||
switch len(c.Args()) {
|
||||
case 1:
|
||||
objectName = c.Args().Get(0)
|
||||
default:
|
||||
log.Fatal("Please specify a valid object name \n # erasure-demo encode [OBJECTNAME]")
|
||||
}
|
||||
|
||||
// Get from staging area
|
||||
stagingConfig := config
|
||||
stagingConfig.k = 2
|
||||
stagingConfig.m = 1
|
||||
stagingConfig.rootDir = config.stagingDir
|
||||
reader, err := erasureGet(stagingConfig, objectName)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Increase parity to user defined or default 10,6
|
||||
err = erasurePut(config, objectName, reader)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
@ -1,11 +1,10 @@
|
||||
package main
|
||||
package minio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/minio-io/minio/pkgs/storage"
|
||||
es "github.com/minio-io/minio/pkgs/storage/encodedstorage"
|
||||
@ -13,8 +12,7 @@ import (
|
||||
|
||||
func erasureGetList(config inputConfig, objectPath string) (io.Reader, error) {
|
||||
var objectStorage storage.ObjectStorage
|
||||
rootDir := path.Join(config.rootDir, config.storageDriver)
|
||||
objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
|
||||
objectStorage, err := es.NewStorage(config.rootDir, config.k, config.m, config.blockSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -33,8 +31,7 @@ func erasureGetList(config inputConfig, objectPath string) (io.Reader, error) {
|
||||
|
||||
func erasureGet(config inputConfig, objectPath string) (io.Reader, error) {
|
||||
var objectStorage storage.ObjectStorage
|
||||
rootDir := path.Join(config.rootDir, config.storageDriver)
|
||||
objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
|
||||
objectStorage, err := es.NewStorage(config.rootDir, config.k, config.m, config.blockSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -47,12 +44,11 @@ func erasureGet(config inputConfig, objectPath string) (io.Reader, error) {
|
||||
|
||||
func erasurePut(config inputConfig, objectPath string, reader io.Reader) error {
|
||||
var err error
|
||||
rootDir := path.Join(config.rootDir, config.storageDriver)
|
||||
if err := os.MkdirAll(rootDir, 0700); err != nil {
|
||||
if err := os.MkdirAll(config.rootDir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
var objectStorage storage.ObjectStorage
|
||||
if objectStorage, err = es.NewStorage(rootDir, config.k, config.m, config.blockSize); err != nil {
|
||||
if objectStorage, err = es.NewStorage(config.rootDir, config.k, config.m, config.blockSize); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = objectStorage.Put(objectPath, reader); err != nil {
|
30
pkgs/minio/get.go
Normal file
30
pkgs/minio/get.go
Normal file
@ -0,0 +1,30 @@
|
||||
package minio
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
)
|
||||
|
||||
func Get(c *cli.Context) {
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
var objectName string
|
||||
var objectReader io.Reader
|
||||
switch len(c.Args()) {
|
||||
case 1:
|
||||
objectName = c.Args().Get(0)
|
||||
default:
|
||||
log.Fatal("Please specify a valid object name \n # erasure-demo get [OBJECTNAME]")
|
||||
}
|
||||
|
||||
if objectReader, err = erasureGet(config, objectName); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
io.Copy(os.Stdout, objectReader)
|
||||
}
|
1
pkgs/minio/init.go
Normal file
1
pkgs/minio/init.go
Normal file
@ -0,0 +1 @@
|
||||
package minio
|
24
pkgs/minio/list.go
Normal file
24
pkgs/minio/list.go
Normal file
@ -0,0 +1,24 @@
|
||||
package minio
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
)
|
||||
|
||||
func List(c *cli.Context) {
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
config.k = 10
|
||||
config.m = 6
|
||||
reader, err := erasureGetList(config, "")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
io.Copy(os.Stdout, reader)
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package main
|
||||
package minio
|
||||
|
||||
import (
|
||||
"log"
|
||||
@ -8,7 +8,7 @@ import (
|
||||
"github.com/codegangsta/cli"
|
||||
)
|
||||
|
||||
func put(c *cli.Context) {
|
||||
func Put(c *cli.Context) {
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
@ -25,19 +25,19 @@ func put(c *cli.Context) {
|
||||
log.Fatal("Please specify a valid object name \n # erasure-demo put [OBJECTNAME] [FILENAME]")
|
||||
}
|
||||
inputFile, err := os.Open(filePath)
|
||||
defer inputFile.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
switch config.storageDriver {
|
||||
case "erasure":
|
||||
{
|
||||
if err := erasurePut(config, objectName, inputFile); err != nil {
|
||||
|
||||
// Staging parity
|
||||
stagingConfig := config
|
||||
stagingConfig.k = 2
|
||||
stagingConfig.m = 1
|
||||
stagingConfig.rootDir = config.stagingDir
|
||||
|
||||
if err := erasurePut(stagingConfig, objectName, inputFile); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
default:
|
||||
{
|
||||
log.Fatal("Unknown driver")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
1
pkgs/minio/verify.go
Normal file
1
pkgs/minio/verify.go
Normal file
@ -0,0 +1 @@
|
||||
package minio
|
@ -80,7 +80,7 @@ func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
|
||||
func (storage *appendStorage) Get(objectPath string) (io.Reader, error) {
|
||||
header, ok := storage.objects[objectPath]
|
||||
if ok == false {
|
||||
return nil, nil
|
||||
return nil, errors.New("Object not found")
|
||||
}
|
||||
|
||||
offset := header.Offset
|
||||
|
@ -110,7 +110,7 @@ func NewStorage(rootDir string, k, m int, blockSize uint64) (storage.ObjectStora
|
||||
func (eStorage *encodedStorage) Get(objectPath string) (io.Reader, error) {
|
||||
entry, ok := eStorage.objects[objectPath]
|
||||
if ok == false {
|
||||
return nil, nil
|
||||
return nil, errors.New("Object not found")
|
||||
}
|
||||
reader, writer := io.Pipe()
|
||||
go eStorage.readObject(objectPath, entry, writer)
|
||||
|
Loading…
Reference in New Issue
Block a user