mirror of
https://github.com/minio/minio.git
synced 2025-11-07 12:52:58 -05:00
Implement commands
Commands implemented
$ minio put <objectname> <file>
$ minio encode <objectname>
$ minio get <objectname>
$ minio list
TODO :
$ minio init
$ minio verify
...
...
This commit is contained in:
Harshavardhana
@@ -1,112 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/checksum/crc32c"
|
||||
"github.com/minio-io/minio/pkgs/erasure"
|
||||
)
|
||||
|
||||
func decode(c *cli.Context) {
|
||||
// check if minio-encode called without parameters
|
||||
if len(c.Args()) != 1 {
|
||||
cli.ShowCommandHelp(c, "decode")
|
||||
return
|
||||
}
|
||||
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
k := config.k
|
||||
m := config.m
|
||||
|
||||
// check if output file exists, fail if so
|
||||
if _, err := os.Stat(config.output); !os.IsNotExist(err) {
|
||||
log.Fatal("Output file exists")
|
||||
}
|
||||
|
||||
// get list of files
|
||||
var inputFiles []string
|
||||
if _, err := os.Stat(config.input + ".length"); os.IsNotExist(err) {
|
||||
err = nil
|
||||
chunkCount := 0
|
||||
for !os.IsNotExist(err) {
|
||||
_, err = os.Stat(config.input + "." + strconv.Itoa(chunkCount) + ".length")
|
||||
chunkCount += 1
|
||||
}
|
||||
chunkCount = chunkCount - 1
|
||||
inputFiles = make([]string, chunkCount)
|
||||
for i := 0; i < chunkCount; i++ {
|
||||
inputFiles[i] = config.input + "." + strconv.Itoa(i)
|
||||
}
|
||||
} else {
|
||||
inputFiles = []string{config.input}
|
||||
}
|
||||
|
||||
// open file to write
|
||||
outputFile, err := os.OpenFile(config.output, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
defer outputFile.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for _, inputFile := range inputFiles {
|
||||
// get chunks
|
||||
chunks := make([][]byte, k+m)
|
||||
// get checksum
|
||||
cksums := make([][]byte, k+m)
|
||||
for i := 0; i < k+m; i++ {
|
||||
chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
|
||||
cksums[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i) + ".cksum")
|
||||
}
|
||||
|
||||
for i := 0; i < k+m; i++ {
|
||||
crcChunk, err := crc32c.Crc32c(chunks[i])
|
||||
if err != nil {
|
||||
chunks[i] = nil
|
||||
continue
|
||||
}
|
||||
crcChunkStr := strconv.Itoa(int(crcChunk))
|
||||
if string(cksums[i]) != crcChunkStr {
|
||||
chunks[i] = nil
|
||||
}
|
||||
}
|
||||
|
||||
// get length
|
||||
lengthBytes, err := ioutil.ReadFile(inputFile + ".length")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
lengthString := string(lengthBytes)
|
||||
length, err := strconv.Atoi(lengthString)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
|
||||
|
||||
// Get decoder
|
||||
encoder := erasure.NewEncoder(erasureParameters)
|
||||
|
||||
// decode data
|
||||
decodedData, err := encoder.Decode(chunks, length)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// append decoded data
|
||||
length, err = outputFile.Write(decodedData)
|
||||
if err != nil {
|
||||
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/checksum/crc32c"
|
||||
"github.com/minio-io/minio/pkgs/erasure"
|
||||
"github.com/minio-io/minio/pkgs/split"
|
||||
)
|
||||
|
||||
func encode(c *cli.Context) {
|
||||
// check if minio-encode called without parameters
|
||||
if len(c.Args()) != 1 {
|
||||
cli.ShowCommandHelp(c, "encode")
|
||||
return
|
||||
}
|
||||
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// get file
|
||||
inputFile, err := os.Open(config.input)
|
||||
defer inputFile.Close()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// read file
|
||||
input, err := ioutil.ReadAll(inputFile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// set up encoder
|
||||
erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
|
||||
|
||||
// Init new encoder
|
||||
encoder := erasure.NewEncoder(erasureParameters)
|
||||
|
||||
// encode data
|
||||
if config.blockSize == 0 {
|
||||
encodedData, length := encoder.Encode(input)
|
||||
for key, data := range encodedData {
|
||||
crc, err := crc32c.Crc32c(data)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
crcString := strconv.Itoa(int(crc))
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
|
||||
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
|
||||
}
|
||||
} else {
|
||||
chunkCount := 0
|
||||
splitChannel := make(chan split.SplitMessage)
|
||||
inputReader := bytes.NewReader(input)
|
||||
go split.SplitStream(inputReader, config.blockSize, splitChannel)
|
||||
for chunk := range splitChannel {
|
||||
if chunk.Err != nil {
|
||||
log.Fatal(chunk.Err)
|
||||
}
|
||||
encodedData, length := encoder.Encode(chunk.Data)
|
||||
for key, data := range encodedData {
|
||||
crc, err := crc32c.Crc32c(data)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
crcString := strconv.Itoa(int(crc))
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
|
||||
}
|
||||
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
|
||||
chunkCount += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/minio-io/minio/pkgs/storage"
|
||||
es "github.com/minio-io/minio/pkgs/storage/encodedstorage"
|
||||
)
|
||||
|
||||
func erasureGetList(config inputConfig, objectPath string) (io.Reader, error) {
|
||||
var objectStorage storage.ObjectStorage
|
||||
rootDir := path.Join(config.rootDir, config.storageDriver)
|
||||
objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
objectDescList, err := objectStorage.List(objectPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var objectDescListBytes []byte
|
||||
if objectDescListBytes, err = json.Marshal(objectDescList); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
objectDescListBuffer := bytes.NewBuffer(objectDescListBytes)
|
||||
|
||||
return objectDescListBuffer, nil
|
||||
}
|
||||
|
||||
func erasureGet(config inputConfig, objectPath string) (io.Reader, error) {
|
||||
var objectStorage storage.ObjectStorage
|
||||
rootDir := path.Join(config.rootDir, config.storageDriver)
|
||||
objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
object, err := objectStorage.Get(objectPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return object, nil
|
||||
}
|
||||
|
||||
func erasurePut(config inputConfig, objectPath string, reader io.Reader) error {
|
||||
var err error
|
||||
rootDir := path.Join(config.rootDir, config.storageDriver)
|
||||
if err := os.MkdirAll(rootDir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
var objectStorage storage.ObjectStorage
|
||||
if objectStorage, err = es.NewStorage(rootDir, config.k, config.m, config.blockSize); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = objectStorage.Put(objectPath, reader); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
)
|
||||
|
||||
func get(c *cli.Context) {
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
var objectReader io.Reader
|
||||
objectName := c.Args().Get(0)
|
||||
|
||||
switch config.storageDriver {
|
||||
case "erasure":
|
||||
{
|
||||
if len(objectName) == 0 {
|
||||
if objectReader, err = erasureGetList(config, ""); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
if objectReader, err = erasureGet(config, objectName); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
{
|
||||
log.Fatal("Unknown driver")
|
||||
}
|
||||
}
|
||||
if objectReader == nil {
|
||||
log.Fatal("Object not found")
|
||||
}
|
||||
io.Copy(os.Stdout, objectReader)
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/strbyteconv"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := cli.NewApp()
|
||||
app.Name = "erasure-demo"
|
||||
app.Usage = "erasure encode a byte stream"
|
||||
app.Commands = []cli.Command{
|
||||
{
|
||||
Name: "encode",
|
||||
Usage: "erasure encode a byte stream",
|
||||
Action: encode,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "output,o",
|
||||
Value: "",
|
||||
Usage: "Output file",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "decode",
|
||||
Usage: "erasure decode a byte stream",
|
||||
Action: decode,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "output,o",
|
||||
Value: "",
|
||||
Usage: "Output file",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "get",
|
||||
Usage: "get an object",
|
||||
Action: get,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: getMinioDir(),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "driver",
|
||||
Value: "erasure",
|
||||
Usage: "erasure",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "put",
|
||||
Usage: "put an object",
|
||||
Action: put,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: getMinioDir(),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "driver",
|
||||
Value: "erasure",
|
||||
Usage: "erasure",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
app.Run(os.Args)
|
||||
}
|
||||
|
||||
// config representing cli input
|
||||
type inputConfig struct {
|
||||
input string
|
||||
output string
|
||||
k int
|
||||
m int
|
||||
blockSize uint64
|
||||
rootDir string
|
||||
storageDriver string
|
||||
}
|
||||
|
||||
// parses input and returns an inputConfig with parsed input
|
||||
func parseInput(c *cli.Context) (inputConfig, error) {
|
||||
// get input path
|
||||
inputFilePath := c.Args().Get(0)
|
||||
|
||||
// get output path
|
||||
outputFilePath := inputFilePath
|
||||
if c.String("output") != "" {
|
||||
outputFilePath = c.String("output")
|
||||
}
|
||||
|
||||
var k, m int
|
||||
if c.String("protection-level") != "" {
|
||||
protectionLevel := c.String("protection-level")
|
||||
protectionLevelSplit := strings.Split(protectionLevel, ",")
|
||||
if len(protectionLevelSplit) != 2 {
|
||||
return inputConfig{}, errors.New("Malformed input for protection-level")
|
||||
}
|
||||
|
||||
var err error
|
||||
k, err = strconv.Atoi(protectionLevelSplit[0])
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
m, err = strconv.Atoi(protectionLevelSplit[1])
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var blockSize uint64
|
||||
blockSize = 0
|
||||
if c.String("block-size") != "" {
|
||||
if c.String("block-size") != "full" {
|
||||
var err error
|
||||
blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
|
||||
if err != nil {
|
||||
return inputConfig{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var rootDir string
|
||||
if c.String("root") != "" {
|
||||
rootDir = c.String("root")
|
||||
}
|
||||
|
||||
var storageDriver string
|
||||
if c.String("driver") != "" {
|
||||
storageDriver = c.String("driver")
|
||||
}
|
||||
|
||||
config := inputConfig{
|
||||
input: inputFilePath,
|
||||
output: outputFilePath,
|
||||
k: k,
|
||||
m: m,
|
||||
blockSize: blockSize,
|
||||
rootDir: rootDir,
|
||||
storageDriver: storageDriver,
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func getMinioDir() string {
|
||||
user, err := user.Current()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
homePath := user.HomeDir
|
||||
minioPath := path.Join(homePath, ".minio")
|
||||
err = _initMinioDir(minioPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return minioPath
|
||||
}
|
||||
|
||||
func _initMinioDir(dirPath string) error {
|
||||
_, err := os.Lstat(dirPath)
|
||||
if err != nil {
|
||||
log.Printf("%s not found, creating a new-one for the first time",
|
||||
dirPath)
|
||||
err = os.Mkdir(dirPath, 0700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
)
|
||||
|
||||
func put(c *cli.Context) {
|
||||
config, err := parseInput(c)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
var filePath, objectName string
|
||||
switch len(c.Args()) {
|
||||
case 1:
|
||||
objectName = path.Base(c.Args().Get(0))
|
||||
filePath = c.Args().Get(0)
|
||||
case 2:
|
||||
objectName = c.Args().Get(0)
|
||||
filePath = c.Args().Get(1)
|
||||
default:
|
||||
log.Fatal("Please specify a valid object name \n # erasure-demo put [OBJECTNAME] [FILENAME]")
|
||||
}
|
||||
inputFile, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
switch config.storageDriver {
|
||||
case "erasure":
|
||||
{
|
||||
if err := erasurePut(config, objectName, inputFile); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
default:
|
||||
{
|
||||
log.Fatal("Unknown driver")
|
||||
}
|
||||
}
|
||||
}
|
||||
95
cmd/minio/minio.go
Normal file
95
cmd/minio/minio.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/codegangsta/cli"
|
||||
"github.com/minio-io/minio/pkgs/minio"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := cli.NewApp()
|
||||
app.Name = "minio"
|
||||
app.Usage = "minio - object storage"
|
||||
app.Commands = []cli.Command{
|
||||
{
|
||||
Name: "encode",
|
||||
Usage: "erasure encode a byte stream",
|
||||
Action: minio.Encode,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: minio.Getobjectdir(".minio/erasure"),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "staging",
|
||||
Value: minio.Getobjectdir(".minio/staging"),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "get",
|
||||
Usage: "get an object",
|
||||
Action: minio.Get,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: minio.Getobjectdir(".minio/erasure"),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "protection-level",
|
||||
Value: "10,6",
|
||||
Usage: "data,parity",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "put",
|
||||
Usage: "put an object",
|
||||
Action: minio.Put,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "staging",
|
||||
Value: minio.Getobjectdir(".minio/staging"),
|
||||
Usage: "",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "block-size",
|
||||
Value: "1M",
|
||||
Usage: "Size of blocks. Examples: 1K, 1M, full",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "list",
|
||||
Usage: "list objects",
|
||||
Action: minio.List,
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "root",
|
||||
Value: minio.Getobjectdir(".minio/erasure"),
|
||||
Usage: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
app.Run(os.Args)
|
||||
}
|
||||
Reference in New Issue
Block a user