MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Implement commands

Browse Source 
Commands implemented

   $ minio put <objectname> <file>
   $ minio encode <objectname>
   $ minio get <objectname>
   $ minio list

TODO :

   $ minio init
   $ minio verify
    ...
    ...
This commit is contained in:
Harshavardhana 2014-12-14 22:13:37 -08:00
parent e3af681cc1
commit 3ec5c2fb8f
16 changed files with 312 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -44,7 +44,7 @@ cover: build-erasure build-signify build-split build-crc32c build-cpu build-sha1
@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/gateway
install: build-erasure
@godep go install github.com/minio-io/minio/cmd/erasure-demo && echo "Installed erasure-demo into ${GOPATH}/bin"
@godep go install github.com/minio-io/minio/cmd/minio && echo "Installed minio into ${GOPATH}/bin"
save: restore
@godep save ./...

112
cmd/erasure-demo/decode.go
View File

@ -1,112 +0,0 @@
package main
import (
"io/ioutil"
"log"
"os"
"strconv"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/checksum/crc32c"
"github.com/minio-io/minio/pkgs/erasure"
)
func decode(c *cli.Context) {
// check if minio-encode called without parameters
if len(c.Args()) != 1 {
cli.ShowCommandHelp(c, "decode")
return
}
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
k := config.k
m := config.m
// check if output file exists, fail if so
if _, err := os.Stat(config.output); !os.IsNotExist(err) {
log.Fatal("Output file exists")
}
// get list of files
var inputFiles []string
if _, err := os.Stat(config.input + ".length"); os.IsNotExist(err) {
err = nil
chunkCount := 0
for !os.IsNotExist(err) {
_, err = os.Stat(config.input + "." + strconv.Itoa(chunkCount) + ".length")
chunkCount += 1
}
chunkCount = chunkCount - 1
inputFiles = make([]string, chunkCount)
for i := 0; i < chunkCount; i++ {
inputFiles[i] = config.input + "." + strconv.Itoa(i)
}
} else {
inputFiles = []string{config.input}
}
// open file to write
outputFile, err := os.OpenFile(config.output, os.O_CREATE|os.O_WRONLY, 0600)
defer outputFile.Close()
if err != nil {
log.Fatal(err)
}
for _, inputFile := range inputFiles {
// get chunks
chunks := make([][]byte, k+m)
// get checksum
cksums := make([][]byte, k+m)
for i := 0; i < k+m; i++ {
chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
cksums[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i) + ".cksum")
}
for i := 0; i < k+m; i++ {
crcChunk, err := crc32c.Crc32c(chunks[i])
if err != nil {
chunks[i] = nil
continue
}
crcChunkStr := strconv.Itoa(int(crcChunk))
if string(cksums[i]) != crcChunkStr {
chunks[i] = nil
}
}
// get length
lengthBytes, err := ioutil.ReadFile(inputFile + ".length")
if err != nil {
log.Fatal(err)
}
lengthString := string(lengthBytes)
length, err := strconv.Atoi(lengthString)
if err != nil {
log.Fatal(err)
}
// set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
// Get decoder
encoder := erasure.NewEncoder(erasureParameters)
// decode data
decodedData, err := encoder.Decode(chunks, length)
if err != nil {
log.Fatal(err)
}
// append decoded data
length, err = outputFile.Write(decodedData)
if err != nil {
log.Fatal(err)
}
}
}

83
cmd/erasure-demo/encode.go
View File

@ -1,83 +0,0 @@
package main
import (
"bytes"
"io/ioutil"
"log"
"os"
"strconv"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/checksum/crc32c"
"github.com/minio-io/minio/pkgs/erasure"
"github.com/minio-io/minio/pkgs/split"
)
func encode(c *cli.Context) {
// check if minio-encode called without parameters
if len(c.Args()) != 1 {
cli.ShowCommandHelp(c, "encode")
return
}
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
// get file
inputFile, err := os.Open(config.input)
defer inputFile.Close()
if err != nil {
log.Fatal(err)
}
// read file
input, err := ioutil.ReadAll(inputFile)
if err != nil {
log.Fatal(err)
}
// set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
// Init new encoder
encoder := erasure.NewEncoder(erasureParameters)
// encode data
if config.blockSize == 0 {
encodedData, length := encoder.Encode(input)
for key, data := range encodedData {
crc, err := crc32c.Crc32c(data)
if err != nil {
log.Fatal(err)
}
crcString := strconv.Itoa(int(crc))
ioutil.WriteFile(config.output+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
}
} else {
chunkCount := 0
splitChannel := make(chan split.SplitMessage)
inputReader := bytes.NewReader(input)
go split.SplitStream(inputReader, config.blockSize, splitChannel)
for chunk := range splitChannel {
if chunk.Err != nil {
log.Fatal(chunk.Err)
}
encodedData, length := encoder.Encode(chunk.Data)
for key, data := range encodedData {
crc, err := crc32c.Crc32c(data)
if err != nil {
log.Fatal(err)
}
crcString := strconv.Itoa(int(crc))
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
}
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
chunkCount += 1
}
}
}

41
cmd/erasure-demo/get.go
View File

@ -1,41 +0,0 @@
package main
import (
"io"
"log"
"os"
"github.com/codegangsta/cli"
)
func get(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
var objectReader io.Reader
objectName := c.Args().Get(0)
switch config.storageDriver {
case "erasure":
{
if len(objectName) == 0 {
if objectReader, err = erasureGetList(config, ""); err != nil {
log.Fatal(err)
}
} else {
if objectReader, err = erasureGet(config, objectName); err != nil {
log.Fatal(err)
}
}
}
default:
{
log.Fatal("Unknown driver")
}
}
if objectReader == nil {
log.Fatal("Object not found")
}
io.Copy(os.Stdout, objectReader)
}

218
cmd/erasure-demo/main.go
View File

@ -1,218 +0,0 @@
package main
import (
"errors"
"log"
"os"
"os/user"
"path"
"strconv"
"strings"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/strbyteconv"
)
func main() {
app := cli.NewApp()
app.Name = "erasure-demo"
app.Usage = "erasure encode a byte stream"
app.Commands = []cli.Command{
{
Name: "encode",
Usage: "erasure encode a byte stream",
Action: encode,
Flags: []cli.Flag{
cli.StringFlag{
Name: "output,o",
Value: "",
Usage: "Output file",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "decode",
Usage: "erasure decode a byte stream",
Action: decode,
Flags: []cli.Flag{
cli.StringFlag{
Name: "output,o",
Value: "",
Usage: "Output file",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
},
},
{
Name: "get",
Usage: "get an object",
Action: get,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: getMinioDir(),
Usage: "",
},
cli.StringFlag{
Name: "driver",
Value: "erasure",
Usage: "erasure",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "put",
Usage: "put an object",
Action: put,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: getMinioDir(),
Usage: "",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
cli.StringFlag{
Name: "driver",
Value: "erasure",
Usage: "erasure",
},
},
},
}
app.Run(os.Args)
}
// config representing cli input
type inputConfig struct {
input string
output string
k int
m int
blockSize uint64
rootDir string
storageDriver string
}
// parses input and returns an inputConfig with parsed input
func parseInput(c *cli.Context) (inputConfig, error) {
// get input path
inputFilePath := c.Args().Get(0)
// get output path
outputFilePath := inputFilePath
if c.String("output") != "" {
outputFilePath = c.String("output")
}
var k, m int
if c.String("protection-level") != "" {
protectionLevel := c.String("protection-level")
protectionLevelSplit := strings.Split(protectionLevel, ",")
if len(protectionLevelSplit) != 2 {
return inputConfig{}, errors.New("Malformed input for protection-level")
}
var err error
k, err = strconv.Atoi(protectionLevelSplit[0])
if err != nil {
return inputConfig{}, err
}
m, err = strconv.Atoi(protectionLevelSplit[1])
if err != nil {
return inputConfig{}, err
}
}
var blockSize uint64
blockSize = 0
if c.String("block-size") != "" {
if c.String("block-size") != "full" {
var err error
blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
if err != nil {
return inputConfig{}, err
}
}
}
var rootDir string
if c.String("root") != "" {
rootDir = c.String("root")
}
var storageDriver string
if c.String("driver") != "" {
storageDriver = c.String("driver")
}
config := inputConfig{
input: inputFilePath,
output: outputFilePath,
k: k,
m: m,
blockSize: blockSize,
rootDir: rootDir,
storageDriver: storageDriver,
}
return config, nil
}
func getMinioDir() string {
user, err := user.Current()
if err != nil {
log.Fatal(err)
}
homePath := user.HomeDir
minioPath := path.Join(homePath, ".minio")
err = _initMinioDir(minioPath)
if err != nil {
log.Fatal(err)
}
return minioPath
}
func _initMinioDir(dirPath string) error {
_, err := os.Lstat(dirPath)
if err != nil {
log.Printf("%s not found, creating a new-one for the first time",
dirPath)
err = os.Mkdir(dirPath, 0700)
if err != nil {
return err
}
}
return nil
}

95
cmd/minio/minio.go Normal file
View File

@ -0,0 +1,95 @@
package main
import (
"os"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/minio"
)
func main() {
app := cli.NewApp()
app.Name = "minio"
app.Usage = "minio - object storage"
app.Commands = []cli.Command{
{
Name: "encode",
Usage: "erasure encode a byte stream",
Action: minio.Encode,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: minio.Getobjectdir(".minio/erasure"),
Usage: "",
},
cli.StringFlag{
Name: "staging",
Value: minio.Getobjectdir(".minio/staging"),
Usage: "",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "get",
Usage: "get an object",
Action: minio.Get,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: minio.Getobjectdir(".minio/erasure"),
Usage: "",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "put",
Usage: "put an object",
Action: minio.Put,
Flags: []cli.Flag{
cli.StringFlag{
Name: "staging",
Value: minio.Getobjectdir(".minio/staging"),
Usage: "",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "list",
Usage: "list objects",
Action: minio.List,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: minio.Getobjectdir(".minio/erasure"),
Usage: "",
},
},
},
}
app.Run(os.Args)
}

103
pkgs/minio/common.go Normal file
View File

@ -0,0 +1,103 @@
package minio
import (
"errors"
"log"
"os"
"os/user"
"path"
"strconv"
"strings"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/strbyteconv"
)
// config representing cli input
type inputConfig struct {
k int
m int
blockSize uint64
rootDir string
stagingDir string
}
// parses input and returns an inputConfig with parsed input
func parseInput(c *cli.Context) (inputConfig, error) {
var k, m int
if c.String("protection-level") != "" {
protectionLevel := c.String("protection-level")
protectionLevelSplit := strings.Split(protectionLevel, ",")
if len(protectionLevelSplit) != 2 {
return inputConfig{}, errors.New("Malformed input for protection-level")
}
var err error
k, err = strconv.Atoi(protectionLevelSplit[0])
if err != nil {
return inputConfig{}, err
}
m, err = strconv.Atoi(protectionLevelSplit[1])
if err != nil {
return inputConfig{}, err
}
}
var blockSize uint64
blockSize = 0
if c.String("block-size") != "" {
if c.String("block-size") != "full" {
var err error
blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
if err != nil {
return inputConfig{}, err
}
}
}
var rootDir string
if c.String("root") != "" {
rootDir = c.String("root")
}
var stagingDir string
if c.String("staging") != "" {
stagingDir = c.String("staging")
}
config := inputConfig{
k: k,
m: m,
blockSize: blockSize,
rootDir: rootDir,
stagingDir: stagingDir,
}
return config, nil
}
func Getobjectdir(basename string) string {
user, err := user.Current()
if err != nil {
log.Fatal(err)
}
homePath := user.HomeDir
minioPath := path.Join(homePath, basename)
err = _initDir(minioPath)
if err != nil {
log.Fatal(err)
}
return minioPath
}
func _initDir(dirPath string) error {
_, err := os.Lstat(dirPath)
if err != nil {
log.Printf("%s not found, creating a new-one for the first time",
dirPath)
err = os.MkdirAll(dirPath, 0700)
if err != nil {
return err
}
}
return nil
}

37
pkgs/minio/encode.go Normal file
View File

@ -0,0 +1,37 @@
package minio
import (
"log"
"github.com/codegangsta/cli"
)
func Encode(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
var objectName string
switch len(c.Args()) {
case 1:
objectName = c.Args().Get(0)
default:
log.Fatal("Please specify a valid object name \n # erasure-demo encode [OBJECTNAME]")
}
// Get from staging area
stagingConfig := config
stagingConfig.k = 2
stagingConfig.m = 1
stagingConfig.rootDir = config.stagingDir
reader, err := erasureGet(stagingConfig, objectName)
if err != nil {
log.Fatal(err)
}
// Increase parity to user defined or default 10,6
err = erasurePut(config, objectName, reader)
if err != nil {
log.Fatal(err)
}
}

14
cmd/erasure-demo/erasure.go → pkgs/minio/erasure.go
View File

@ -1,11 +1,10 @@
package main
package minio
import (
"bytes"
"encoding/json"
"io"
"os"
"path"
"github.com/minio-io/minio/pkgs/storage"
es "github.com/minio-io/minio/pkgs/storage/encodedstorage"
@ -13,8 +12,7 @@ import (
func erasureGetList(config inputConfig, objectPath string) (io.Reader, error) {
var objectStorage storage.ObjectStorage
rootDir := path.Join(config.rootDir, config.storageDriver)
objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
objectStorage, err := es.NewStorage(config.rootDir, config.k, config.m, config.blockSize)
if err != nil {
return nil, err
}
@ -33,8 +31,7 @@ func erasureGetList(config inputConfig, objectPath string) (io.Reader, error) {
func erasureGet(config inputConfig, objectPath string) (io.Reader, error) {
var objectStorage storage.ObjectStorage
rootDir := path.Join(config.rootDir, config.storageDriver)
objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
objectStorage, err := es.NewStorage(config.rootDir, config.k, config.m, config.blockSize)
if err != nil {
return nil, err
}
@ -47,12 +44,11 @@ func erasureGet(config inputConfig, objectPath string) (io.Reader, error) {
func erasurePut(config inputConfig, objectPath string, reader io.Reader) error {
var err error
rootDir := path.Join(config.rootDir, config.storageDriver)
if err := os.MkdirAll(rootDir, 0700); err != nil {
if err := os.MkdirAll(config.rootDir, 0700); err != nil {
return err
}
var objectStorage storage.ObjectStorage
if objectStorage, err = es.NewStorage(rootDir, config.k, config.m, config.blockSize); err != nil {
if objectStorage, err = es.NewStorage(config.rootDir, config.k, config.m, config.blockSize); err != nil {
return err
}
if err = objectStorage.Put(objectPath, reader); err != nil {

30
pkgs/minio/get.go Normal file
View File

@ -0,0 +1,30 @@
package minio
import (
"io"
"log"
"os"
"github.com/codegangsta/cli"
)
func Get(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
var objectName string
var objectReader io.Reader
switch len(c.Args()) {
case 1:
objectName = c.Args().Get(0)
default:
log.Fatal("Please specify a valid object name \n # erasure-demo get [OBJECTNAME]")
}
if objectReader, err = erasureGet(config, objectName); err != nil {
log.Fatal(err)
}
io.Copy(os.Stdout, objectReader)
}

1
pkgs/minio/init.go Normal file
View File

@ -0,0 +1 @@
package minio

24
pkgs/minio/list.go Normal file
View File

@ -0,0 +1,24 @@
package minio
import (
"io"
"log"
"os"
"github.com/codegangsta/cli"
)
func List(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
config.k = 10
config.m = 6
reader, err := erasureGetList(config, "")
if err != nil {
log.Fatal(err)
}
io.Copy(os.Stdout, reader)
}

24
cmd/erasure-demo/put.go → pkgs/minio/put.go
View File

@ -1,4 +1,4 @@
package main
package minio
import (
"log"
@ -8,7 +8,7 @@ import (
"github.com/codegangsta/cli"
)
func put(c *cli.Context) {
func Put(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
@ -25,19 +25,19 @@ func put(c *cli.Context) {
log.Fatal("Please specify a valid object name \n # erasure-demo put [OBJECTNAME] [FILENAME]")
}
inputFile, err := os.Open(filePath)
defer inputFile.Close()
if err != nil {
log.Fatal(err)
}
switch config.storageDriver {
case "erasure":
{
if err := erasurePut(config, objectName, inputFile); err != nil {
// Staging parity
stagingConfig := config
stagingConfig.k = 2
stagingConfig.m = 1
stagingConfig.rootDir = config.stagingDir
if err := erasurePut(stagingConfig, objectName, inputFile); err != nil {
log.Fatal(err)
}
}
default:
{
log.Fatal("Unknown driver")
}
}
}

1
pkgs/minio/verify.go Normal file
View File

@ -0,0 +1 @@
package minio

2
pkgs/storage/appendstorage/append_storage.go
View File

@ -80,7 +80,7 @@ func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
func (storage *appendStorage) Get(objectPath string) (io.Reader, error) {
header, ok := storage.objects[objectPath]
if ok == false {
return nil, nil
return nil, errors.New("Object not found")
}
offset := header.Offset

2
pkgs/storage/encodedstorage/encoded_storage.go
View File

@ -110,7 +110,7 @@ func NewStorage(rootDir string, k, m int, blockSize uint64) (storage.ObjectStora
func (eStorage *encodedStorage) Get(objectPath string) (io.Reader, error) {
entry, ok := eStorage.objects[objectPath]
if ok == false {
return nil, nil
return nil, errors.New("Object not found")
}
reader, writer := io.Pipe()
go eStorage.readObject(objectPath, entry, writer)