MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 22:25:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into HEAD

Browse Source 
Conflicts:
	pkg/drivers/donut/donut.go
	pkg/storage/donut/bucket.go
	pkg/storage/donut/donut.go
	pkg/storage/donut/donut_test.go
	pkg/storage/donut/donutdriver.go
This commit is contained in:
Frederick F. Kautz IV 2015-03-24 21:04:08 -07:00
commit dd5ba865ff
59 changed files with 1681 additions and 1239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Godeps/Godeps.json generated
View File

@ -5,11 +5,6 @@
"./..."
],
"Deps": [
{
"ImportPath": "github.com/codegangsta/cli",
"Comment": "1.2.0-42-gfbda1ce",
"Rev": "fbda1ce02d5dabcee952040e5f4025753b6c9ce0"
},
{
"ImportPath": "github.com/gorilla/context",
"Rev": "50c25fb3b2b3b3cc724e9b6ac75fb44b3bccd0da"
@ -18,6 +13,11 @@
"ImportPath": "github.com/gorilla/mux",
"Rev": "e444e69cbd2e2e3e0749a2f3c717cec491552bbf"
},
{
"ImportPath": "github.com/minio-io/cli",
"Comment": "1.2.0-96-gfcc23e2",
"Rev": "fcc23e23a705c0d95fce2a446c364ac31a1c73a5"
},
{
"ImportPath": "gopkg.in/check.v1",
"Rev": "64131543e7896d5bcc6bd5a76287eb75ea96c673"

0
Godeps/_workspace/src/github.com/codegangsta/cli/.travis.yml → Godeps/_workspace/src/github.com/minio-io/cli/.travis.yml generated vendored
View File

0
Godeps/_workspace/src/github.com/codegangsta/cli/LICENSE → Godeps/_workspace/src/github.com/minio-io/cli/LICENSE generated vendored
View File

14
Godeps/_workspace/src/github.com/codegangsta/cli/README.md → Godeps/_workspace/src/github.com/minio-io/cli/README.md generated vendored
View File

@ -210,7 +210,7 @@ Subcommands can be defined for a more git-like command line app.
app.Commands = []cli.Command{
{
Name: "add",
ShortName: "a",
Aliases: []string{"a"},
Usage: "add a task to the list",
Action: func(c *cli.Context) {
println("added task: ", c.Args().First())
@ -218,7 +218,7 @@ app.Commands = []cli.Command{
},
{
Name: "complete",
ShortName: "c",
Aliases: []string{"c"},
Usage: "complete a task on the list",
Action: func(c *cli.Context) {
println("completed task: ", c.Args().First())
@ -226,7 +226,7 @@ app.Commands = []cli.Command{
},
{
Name: "template",
ShortName: "r",
Aliases: []string{"r"},
Usage: "options for task templates",
Subcommands: []cli.Command{
{
@ -244,7 +244,7 @@ app.Commands = []cli.Command{
},
},
},
},
},
}
...
```
@ -262,8 +262,8 @@ app := cli.NewApp()
app.EnableBashCompletion = true
app.Commands = []cli.Command{
{
Name: "complete",
ShortName: "c",
Name: "complete",
Aliases: []string{"c"},
Usage: "complete a task on the list",
Action: func(c *cli.Context) {
println("completed task: ", c.Args().First())
@ -293,6 +293,6 @@ setting the `PROG` variable to the name of your program:
## Contribution Guidelines
Feel free to put up a pull request to fix a bug or maybe add a feature. I will give it a code review and make sure that it does not break backwards compatibility. If I or any other collaborators agree that it is in line with the vision of the project, we will work with you to get the code into a mergeable state and merge it into the master branch.
If you are have contributed something significant to the project, I will most likely add you as a collaborator. As a collaborator you are given the ability to merge others pull requests. It is very important that new code does not break existing code, so be careful about what code you do choose to merge. If you have any questions feel free to link @codegangsta to the issue in question and we can review it together.
If you have contributed something significant to the project, I will most likely add you as a collaborator. As a collaborator you are given the ability to merge others pull requests. It is very important that new code does not break existing code, so be careful about what code you do choose to merge. If you have any questions feel free to link @codegangsta to the issue in question and we can review it together.
If you feel like you have contributed to the project but have not yet been added as a collaborator, I probably forgot to add you. Hit @codegangsta up over email and we will get it figured out.

112
Godeps/_workspace/src/github.com/codegangsta/cli/app.go → Godeps/_workspace/src/github.com/minio-io/cli/app.go generated vendored
View File

@ -2,8 +2,12 @@ package cli
import (
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"text/tabwriter"
"text/template"
"time"
)
@ -31,16 +35,23 @@ type App struct {
// An action to execute before any subcommands are run, but after the context is ready
// If a non-nil error is returned, no subcommands are run
Before func(context *Context) error
// An action to execute after any subcommands are run, but after the subcommand has finished
// It is run even if Action() panics
After func(context *Context) error
// The action to execute when no subcommands are specified
Action func(context *Context)
// Execute this function if the proper command cannot be found
CommandNotFound func(context *Context, command string)
// Compilation date
Compiled time.Time
// Author
// List of all authors who contributed
Authors []Author
// Name of Author (Note: Use App.Authors, this is deprecated)
Author string
// Author e-mail
// Email of Author (Note: Use App.Authors, this is deprecated)
Email string
// Writer writer to write output to
Writer io.Writer
}
// Tries to find out when this binary was compiled.
@ -62,15 +73,42 @@ func NewApp() *App {
BashComplete: DefaultAppComplete,
Action: helpCommand.Action,
Compiled: compileTime(),
Writer: os.Stdout,
}
}
// Entry point to the cli app. Parses the arguments slice and routes to the proper flag/args combination
func (a *App) Run(arguments []string) error {
func (a *App) Run(arguments []string) (err error) {
if a.Author != "" || a.Email != "" {
a.Authors = append(a.Authors, Author{Name: a.Author, Email: a.Email})
}
if HelpPrinter == nil {
defer func() {
HelpPrinter = nil
}()
HelpPrinter = func(templ string, data interface{}) {
funcMap := template.FuncMap{
"join": strings.Join,
}
w := tabwriter.NewWriter(a.Writer, 0, 8, 1, '\t', 0)
t := template.Must(template.New("help").Funcs(funcMap).Parse(templ))
err := t.Execute(w, data)
if err != nil {
panic(err)
}
w.Flush()
}
}
// append help to commands
if a.Command(helpCommand.Name) == nil && !a.HideHelp {
a.Commands = append(a.Commands, helpCommand)
a.appendFlag(HelpFlag)
if (HelpFlag != BoolFlag{}) {
a.appendFlag(HelpFlag)
}
}
//append version/help flags
@ -85,21 +123,21 @@ func (a *App) Run(arguments []string) error {
// parse flags
set := flagSet(a.Name, a.Flags)
set.SetOutput(ioutil.Discard)
err := set.Parse(arguments[1:])
err = set.Parse(arguments[1:])
nerr := normalizeFlags(a.Flags, set)
if nerr != nil {
fmt.Println(nerr)
fmt.Fprintln(a.Writer, nerr)
context := NewContext(a, set, set)
ShowAppHelp(context)
fmt.Println("")
fmt.Fprintln(a.Writer)
return nerr
}
context := NewContext(a, set, set)
if err != nil {
fmt.Printf("Incorrect Usage.\n\n")
fmt.Fprintf(a.Writer, "Incorrect Usage.\n\n")
ShowAppHelp(context)
fmt.Println("")
fmt.Fprintln(a.Writer)
return err
}
@ -115,6 +153,15 @@ func (a *App) Run(arguments []string) error {
return nil
}
if a.After != nil {
defer func() {
// err is always nil here.
// There is a check to see if it is non-nil
// just few lines before.
err = a.After(context)
}()
}
if a.Before != nil {
err := a.Before(context)
if err != nil {
@ -139,18 +186,20 @@ func (a *App) Run(arguments []string) error {
// Another entry point to the cli app, takes care of passing arguments and error handling
func (a *App) RunAndExitOnError() {
if err := a.Run(os.Args); err != nil {
os.Stderr.WriteString(fmt.Sprintln(err))
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
// Invokes the subcommand given the context, parses ctx.Args() to generate command-specific flags
func (a *App) RunAsSubcommand(ctx *Context) error {
func (a *App) RunAsSubcommand(ctx *Context) (err error) {
// append help to commands
if len(a.Commands) > 0 {
if a.Command(helpCommand.Name) == nil && !a.HideHelp {
a.Commands = append(a.Commands, helpCommand)
a.appendFlag(HelpFlag)
if (HelpFlag != BoolFlag{}) {
a.appendFlag(HelpFlag)
}
}
}
@ -162,23 +211,23 @@ func (a *App) RunAsSubcommand(ctx *Context) error {
// parse flags
set := flagSet(a.Name, a.Flags)
set.SetOutput(ioutil.Discard)
err := set.Parse(ctx.Args().Tail())
err = set.Parse(ctx.Args().Tail())
nerr := normalizeFlags(a.Flags, set)
context := NewContext(a, set, ctx.globalSet)
if nerr != nil {
fmt.Println(nerr)
fmt.Fprintln(a.Writer, nerr)
if len(a.Commands) > 0 {
ShowSubcommandHelp(context)
} else {
ShowCommandHelp(ctx, context.Args().First())
}
fmt.Println("")
fmt.Fprintln(a.Writer)
return nerr
}
if err != nil {
fmt.Printf("Incorrect Usage.\n\n")
fmt.Fprintf(a.Writer, "Incorrect Usage.\n\n")
ShowSubcommandHelp(context)
return err
}
@ -197,6 +246,15 @@ func (a *App) RunAsSubcommand(ctx *Context) error {
}
}
if a.After != nil {
defer func() {
// err is always nil here.
// There is a check to see if it is non-nil
// just few lines before.
err = a.After(context)
}()
}
if a.Before != nil {
err := a.Before(context)
if err != nil {
@ -214,11 +272,7 @@ func (a *App) RunAsSubcommand(ctx *Context) error {
}
// Run default Action
if len(a.Commands) > 0 {
a.Action(context)
} else {
a.Action(ctx)
}
a.Action(context)
return nil
}
@ -249,3 +303,19 @@ func (a *App) appendFlag(flag Flag) {
a.Flags = append(a.Flags, flag)
}
}
// Author represents someone who has contributed to a cli project.
type Author struct {
Name string // The Authors name
Email string // The Authors email
}
// String makes Author comply to the Stringer interface, to allow an easy print in the templating process
func (a Author) String() string {
e := ""
if a.Email != "" {
e = "<" + a.Email + "> "
}
return fmt.Sprintf("%v %v", a.Name, e)
}

211
Godeps/_workspace/src/github.com/codegangsta/cli/app_test.go → Godeps/_workspace/src/github.com/minio-io/cli/app_test.go generated vendored
View File

@ -1,6 +1,7 @@
package cli_test
import (
"flag"
"fmt"
"os"
"testing"
@ -20,6 +21,9 @@ func ExampleApp() {
app.Action = func(c *cli.Context) {
fmt.Printf("Hello %v\n", c.String("name"))
}
app.Author = "Harrison"
app.Email = "harrison@lolwut.com"
app.Authors = []cli.Author{cli.Author{Name: "Oliver Allen", Email: "oliver@toyshop.com"}}
app.Run(os.Args)
// Output:
// Hello Jeremy
@ -33,13 +37,13 @@ func ExampleAppSubcommand() {
app.Commands = []cli.Command{
{
Name: "hello",
ShortName: "hi",
Aliases: []string{"hi"},
Usage: "use it to see a description",
Description: "This is how we describe hello the function",
Subcommands: []cli.Command{
{
Name: "english",
ShortName: "en",
Aliases: []string{"en"},
Usage: "sends a greeting in english",
Description: "greets someone in english",
Flags: []cli.Flag{
@ -74,7 +78,7 @@ func ExampleAppHelp() {
app.Commands = []cli.Command{
{
Name: "describeit",
ShortName: "d",
Aliases: []string{"d"},
Usage: "use it to see a description",
Description: "This is how we describe describeit the function",
Action: func(c *cli.Context) {
@ -104,7 +108,7 @@ func ExampleAppBashComplete() {
app.Commands = []cli.Command{
{
Name: "describeit",
ShortName: "d",
Aliases: []string{"d"},
Usage: "use it to see a description",
Description: "This is how we describe describeit the function",
Action: func(c *cli.Context) {
@ -158,8 +162,8 @@ var commandAppTests = []struct {
func TestApp_Command(t *testing.T) {
app := cli.NewApp()
fooCommand := cli.Command{Name: "foobar", ShortName: "f"}
batCommand := cli.Command{Name: "batbaz", ShortName: "b"}
fooCommand := cli.Command{Name: "foobar", Aliases: []string{"f"}}
batCommand := cli.Command{Name: "batbaz", Aliases: []string{"b"}}
app.Commands = []cli.Command{
fooCommand,
batCommand,
@ -192,6 +196,76 @@ func TestApp_CommandWithArgBeforeFlags(t *testing.T) {
expect(t, firstArg, "my-arg")
}
func TestApp_RunAsSubcommandParseFlags(t *testing.T) {
var context *cli.Context
a := cli.NewApp()
a.Commands = []cli.Command{
{
Name: "foo",
Action: func(c *cli.Context) {
context = c
},
Flags: []cli.Flag{
cli.StringFlag{
Name: "lang",
Value: "english",
Usage: "language for the greeting",
},
},
Before: func(_ *cli.Context) error { return nil },
},
}
a.Run([]string{"", "foo", "--lang", "spanish", "abcd"})
expect(t, context.Args().Get(0), "abcd")
expect(t, context.String("lang"), "spanish")
}
func TestApp_CommandWithFlagBeforeTerminator(t *testing.T) {
var parsedOption string
var args []string
app := cli.NewApp()
command := cli.Command{
Name: "cmd",
Flags: []cli.Flag{
cli.StringFlag{Name: "option", Value: "", Usage: "some option"},
},
Action: func(c *cli.Context) {
parsedOption = c.String("option")
args = c.Args()
},
}
app.Commands = []cli.Command{command}
app.Run([]string{"", "cmd", "my-arg", "--option", "my-option", "--", "--notARealFlag"})
expect(t, parsedOption, "my-option")
expect(t, args[0], "my-arg")
expect(t, args[1], "--")
expect(t, args[2], "--notARealFlag")
}
func TestApp_CommandWithNoFlagBeforeTerminator(t *testing.T) {
var args []string
app := cli.NewApp()
command := cli.Command{
Name: "cmd",
Action: func(c *cli.Context) {
args = c.Args()
},
}
app.Commands = []cli.Command{command}
app.Run([]string{"", "cmd", "my-arg", "--", "notAFlagAtAll"})
expect(t, args[0], "my-arg")
expect(t, args[1], "--")
expect(t, args[2], "notAFlagAtAll")
}
func TestApp_Float64Flag(t *testing.T) {
var meters float64
@ -265,6 +339,50 @@ func TestApp_ParseSliceFlags(t *testing.T) {
}
}
func TestApp_DefaultStdout(t *testing.T) {
app := cli.NewApp()
if app.Writer != os.Stdout {
t.Error("Default output writer not set.")
}
}
type mockWriter struct {
written []byte
}
func (fw *mockWriter) Write(p []byte) (n int, err error) {
if fw.written == nil {
fw.written = p
} else {
fw.written = append(fw.written, p...)
}
return len(p), nil
}
func (fw *mockWriter) GetWritten() (b []byte) {
return fw.written
}
func TestApp_SetStdout(t *testing.T) {
w := &mockWriter{}
app := cli.NewApp()
app.Name = "test"
app.Writer = w
err := app.Run([]string{"help"})
if err != nil {
t.Fatalf("Run error: %s", err)
}
if len(w.written) == 0 {
t.Error("App did not write output to desired writer.")
}
}
func TestApp_BeforeFunc(t *testing.T) {
beforeRun, subcommandRun := false, false
beforeError := fmt.Errorf("fail")
@ -331,6 +449,87 @@ func TestApp_BeforeFunc(t *testing.T) {
}
func TestApp_AfterFunc(t *testing.T) {
afterRun, subcommandRun := false, false
afterError := fmt.Errorf("fail")
var err error
app := cli.NewApp()
app.After = func(c *cli.Context) error {
afterRun = true
s := c.String("opt")
if s == "fail" {
return afterError
}
return nil
}
app.Commands = []cli.Command{
cli.Command{
Name: "sub",
Action: func(c *cli.Context) {
subcommandRun = true
},
},
}
app.Flags = []cli.Flag{
cli.StringFlag{Name: "opt"},
}
// run with the After() func succeeding
err = app.Run([]string{"command", "--opt", "succeed", "sub"})
if err != nil {
t.Fatalf("Run error: %s", err)
}
if afterRun == false {
t.Errorf("After() not executed when expected")
}
if subcommandRun == false {
t.Errorf("Subcommand not executed when expected")
}
// reset
afterRun, subcommandRun = false, false
// run with the Before() func failing
err = app.Run([]string{"command", "--opt", "fail", "sub"})
// should be the same error produced by the Before func
if err != afterError {
t.Errorf("Run error expected, but not received")
}
if afterRun == false {
t.Errorf("After() not executed when expected")
}
if subcommandRun == false {
t.Errorf("Subcommand not executed when expected")
}
}
func TestAppNoHelpFlag(t *testing.T) {
oldFlag := cli.HelpFlag
defer func() {
cli.HelpFlag = oldFlag
}()
cli.HelpFlag = cli.BoolFlag{}
app := cli.NewApp()
err := app.Run([]string{"test", "-h"})
if err != flag.ErrHelp {
t.Errorf("expected error about missing help flag, but got: %s (%T)", err, err)
}
}
func TestAppHelpPrinter(t *testing.T) {
oldPrinter := cli.HelpPrinter
defer func() {

0
Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/bash_autocomplete → Godeps/_workspace/src/github.com/minio-io/cli/autocomplete/bash_autocomplete generated vendored
View File

0
Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/zsh_autocomplete → Godeps/_workspace/src/github.com/minio-io/cli/autocomplete/zsh_autocomplete generated vendored
View File

0
Godeps/_workspace/src/github.com/codegangsta/cli/cli.go → Godeps/_workspace/src/github.com/minio-io/cli/cli.go generated vendored
View File

28
Godeps/_workspace/src/github.com/codegangsta/cli/cli_test.go → Godeps/_workspace/src/github.com/minio-io/cli/cli_test.go generated vendored
View File

@ -12,17 +12,17 @@ func Example() {
app.Usage = "task list on the command line"
app.Commands = []cli.Command{
{
Name: "add",
ShortName: "a",
Usage: "add a task to the list",
Name: "add",
Aliases: []string{"a"},
Usage: "add a task to the list",
Action: func(c *cli.Context) {
println("added task: ", c.Args().First())
},
},
{
Name: "complete",
ShortName: "c",
Usage: "complete a task on the list",
Name: "complete",
Aliases: []string{"c"},
Usage: "complete a task on the list",
Action: func(c *cli.Context) {
println("completed task: ", c.Args().First())
},
@ -38,13 +38,13 @@ func ExampleSubcommand() {
app.Commands = []cli.Command{
{
Name: "hello",
ShortName: "hi",
Aliases: []string{"hi"},
Usage: "use it to see a description",
Description: "This is how we describe hello the function",
Subcommands: []cli.Command{
{
Name: "english",
ShortName: "en",
Aliases: []string{"en"},
Usage: "sends a greeting in english",
Description: "greets someone in english",
Flags: []cli.Flag{
@ -58,9 +58,9 @@ func ExampleSubcommand() {
println("Hello, ", c.String("name"))
},
}, {
Name: "spanish",
ShortName: "sp",
Usage: "sends a greeting in spanish",
Name: "spanish",
Aliases: []string{"sp"},
Usage: "sends a greeting in spanish",
Flags: []cli.Flag{
cli.StringFlag{
Name: "surname",
@ -72,9 +72,9 @@ func ExampleSubcommand() {
println("Hola, ", c.String("surname"))
},
}, {
Name: "french",
ShortName: "fr",
Usage: "sends a greeting in french",
Name: "french",
Aliases: []string{"fr"},
Usage: "sends a greeting in french",
Flags: []cli.Flag{
cli.StringFlag{
Name: "nickname",

59
Godeps/_workspace/src/github.com/codegangsta/cli/command.go → Godeps/_workspace/src/github.com/minio-io/cli/command.go generated vendored
View File

@ -10,8 +10,10 @@ import (
type Command struct {
// The name of the command
Name string
// short name of the command. Typically one character
// short name of the command. Typically one character (deprecated, use `Aliases`)
ShortName string
// A list of aliases for the command
Aliases []string
// A short description of the usage of this command
Usage string
// A longer explanation of how the command works
@ -21,6 +23,9 @@ type Command struct {
// An action to execute before any sub-subcommands are run, but after the context is ready
// If a non-nil error is returned, no sub-subcommands are run
Before func(context *Context) error
// An action to execute after any subcommands are run, but after the subcommand has finished
// It is run even if Action() panics
After func(context *Context) error
// The function to call when this command is invoked
Action func(context *Context)
// List of child commands
@ -36,11 +41,11 @@ type Command struct {
// Invokes the command given the context, parses ctx.Args() to generate command-specific flags
func (c Command) Run(ctx *Context) error {
if len(c.Subcommands) > 0 || c.Before != nil {
if len(c.Subcommands) > 0 || c.Before != nil || c.After != nil {
return c.startApp(ctx)
}
if !c.HideHelp {
if !c.HideHelp && (HelpFlag != BoolFlag{}) {
// append help to flags
c.Flags = append(
c.Flags,
@ -56,36 +61,48 @@ func (c Command) Run(ctx *Context) error {
set.SetOutput(ioutil.Discard)
firstFlagIndex := -1
terminatorIndex := -1
for index, arg := range ctx.Args() {
if strings.HasPrefix(arg, "-") {
firstFlagIndex = index
if arg == "--" {
terminatorIndex = index
break
} else if strings.HasPrefix(arg, "-") && firstFlagIndex == -1 {
firstFlagIndex = index
}
}
var err error
if firstFlagIndex > -1 && !c.SkipFlagParsing {
args := ctx.Args()
regularArgs := args[1:firstFlagIndex]
flagArgs := args[firstFlagIndex:]
regularArgs := make([]string, len(args[1:firstFlagIndex]))
copy(regularArgs, args[1:firstFlagIndex])
var flagArgs []string
if terminatorIndex > -1 {
flagArgs = args[firstFlagIndex:terminatorIndex]
regularArgs = append(regularArgs, args[terminatorIndex:]...)
} else {
flagArgs = args[firstFlagIndex:]
}
err = set.Parse(append(flagArgs, regularArgs...))
} else {
err = set.Parse(ctx.Args().Tail())
}
if err != nil {
fmt.Printf("Incorrect Usage.\n\n")
fmt.Fprint(ctx.App.Writer, "Incorrect Usage.\n\n")
ShowCommandHelp(ctx, c.Name)
fmt.Println("")
fmt.Fprintln(ctx.App.Writer)
return err
}
nerr := normalizeFlags(c.Flags, set)
if nerr != nil {
fmt.Println(nerr)
fmt.Println("")
fmt.Fprintln(ctx.App.Writer, nerr)
fmt.Fprintln(ctx.App.Writer)
ShowCommandHelp(ctx, c.Name)
fmt.Println("")
fmt.Fprintln(ctx.App.Writer)
return nerr
}
context := NewContext(ctx.App, set, ctx.globalSet)
@ -102,9 +119,24 @@ func (c Command) Run(ctx *Context) error {
return nil
}
func (c Command) Names() []string {
names := []string{c.Name}
if c.ShortName != "" {
names = append(names, c.ShortName)
}
return append(names, c.Aliases...)
}
// Returns true if Command.Name or Command.ShortName matches given name
func (c Command) HasName(name string) bool {
return c.Name == name || c.ShortName == name
for _, n := range c.Names() {
if n == name {
return true
}
}
return false
}
func (c Command) startApp(ctx *Context) error {
@ -134,6 +166,7 @@ func (c Command) startApp(ctx *Context) error {
// set the actions
app.Before = c.Before
app.After = c.After
if c.Action != nil {
app.Action = c.Action
} else {

4
Godeps/_workspace/src/github.com/codegangsta/cli/command_test.go → Godeps/_workspace/src/github.com/minio-io/cli/command_test.go generated vendored
View File

@ -17,7 +17,7 @@ func TestCommandDoNotIgnoreFlags(t *testing.T) {
command := cli.Command{
Name: "test-cmd",
ShortName: "tc",
Aliases: []string{"tc"},
Usage: "this is for testing",
Description: "testing",
Action: func(_ *cli.Context) {},
@ -37,7 +37,7 @@ func TestCommandIgnoreFlags(t *testing.T) {
command := cli.Command{
Name: "test-cmd",
ShortName: "tc",
Aliases: []string{"tc"},
Usage: "this is for testing",
Description: "testing",
Action: func(_ *cli.Context) {},

5
Godeps/_workspace/src/github.com/codegangsta/cli/context.go → Godeps/_workspace/src/github.com/minio-io/cli/context.go generated vendored
View File

@ -106,6 +106,11 @@ func (c *Context) GlobalGeneric(name string) interface{} {
return lookupGeneric(name, c.globalSet)
}
// Returns the number of flags set
func (c *Context) NumFlags() int {
return c.flagSet.NFlag()
}
// Determines if the flag was actually set
func (c *Context) IsSet(name string) bool {
if c.setFlags == nil {

12
Godeps/_workspace/src/github.com/codegangsta/cli/context_test.go → Godeps/_workspace/src/github.com/minio-io/cli/context_test.go generated vendored
View File

@ -97,3 +97,15 @@ func TestContext_GlobalIsSet(t *testing.T) {
expect(t, c.GlobalIsSet("myflagGlobalUnset"), false)
expect(t, c.GlobalIsSet("bogusGlobal"), false)
}
func TestContext_NumFlags(t *testing.T) {
set := flag.NewFlagSet("test", 0)
set.Bool("myflag", false, "doc")
set.String("otherflag", "hello world", "doc")
globalSet := flag.NewFlagSet("test", 0)
globalSet.Bool("myflagGlobal", true, "doc")
c := cli.NewContext(nil, set, globalSet)
set.Parse([]string{"--myflag", "--otherflag=foo"})
globalSet.Parse([]string{"--myflagGlobal"})
expect(t, c.NumFlags(), 2)
}

71
Godeps/_workspace/src/github.com/codegangsta/cli/flag.go → Godeps/_workspace/src/github.com/minio-io/cli/flag.go generated vendored
View File

@ -12,6 +12,7 @@ import (
// This flag enables bash-completion for all commands and subcommands
var BashCompletionFlag = BoolFlag{
Name: "generate-bash-completion",
Hide: true,
}
// This flag prints the version for the application
@ -21,6 +22,8 @@ var VersionFlag = BoolFlag{
}
// This flag prints the help for all commands and subcommands
// Set to the zero value (BoolFlag{}) to disable flag -- keeps subcommand
// unless HideHelp is set to true)
var HelpFlag = BoolFlag{
Name: "help, h",
Usage: "show help",
@ -34,6 +37,7 @@ type Flag interface {
// Apply Flag settings to the given flag set
Apply(*flag.FlagSet)
getName() string
isNotHidden() bool
}
func flagSet(name string, flags []Flag) *flag.FlagSet {
@ -65,12 +69,18 @@ type GenericFlag struct {
Value Generic
Usage string
EnvVar string
Hide bool
}
// String returns the string representation of the generic flag to display the
// help text to the user (uses the String() method of the generic flag to show
// the value)
func (f GenericFlag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s%s %v\t`%v` %s", prefixFor(f.Name), f.Name, f.Value, "-"+f.Name+" option -"+f.Name+" option", f.Usage))
return withEnvHint(f.EnvVar, fmt.Sprintf("%s%s \"%v\"\t%v", prefixFor(f.Name), f.Name, f.Value, f.Usage))
}
// Apply takes the flagset and calls Set on the generic flag with the value
// provided by the user for parsing by the flag
func (f GenericFlag) Apply(set *flag.FlagSet) {
val := f.Value
if f.EnvVar != "" {
@ -92,6 +102,10 @@ func (f GenericFlag) getName() string {
return f.Name
}
func (f GenericFlag) isNotHidden() bool {
return !f.Hide
}
type StringSlice []string
func (f *StringSlice) Set(value string) error {
@ -112,12 +126,13 @@ type StringSliceFlag struct {
Value *StringSlice
Usage string
EnvVar string
Hide bool
}
func (f StringSliceFlag) String() string {
firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
pref := prefixFor(firstName)
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
}
func (f StringSliceFlag) Apply(set *flag.FlagSet) {
@ -145,6 +160,10 @@ func (f StringSliceFlag) getName() string {
return f.Name
}
func (f StringSliceFlag) isNotHidden() bool {
return !f.Hide
}
type IntSlice []int
func (f *IntSlice) Set(value string) error {
@ -171,12 +190,13 @@ type IntSliceFlag struct {
Value *IntSlice
Usage string
EnvVar string
Hide bool
}
func (f IntSliceFlag) String() string {
firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
pref := prefixFor(firstName)
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
}
func (f IntSliceFlag) Apply(set *flag.FlagSet) {
@ -207,10 +227,15 @@ func (f IntSliceFlag) getName() string {
return f.Name
}
func (f IntSliceFlag) isNotHidden() bool {
return !f.Hide
}
type BoolFlag struct {
Name string
Usage string
EnvVar string
Hide bool
}
func (f BoolFlag) String() string {
@ -241,10 +266,15 @@ func (f BoolFlag) getName() string {
return f.Name
}
func (f BoolFlag) isNotHidden() bool {
return !f.Hide
}
type BoolTFlag struct {
Name string
Usage string
EnvVar string
Hide bool
}
func (f BoolTFlag) String() string {
@ -275,11 +305,16 @@ func (f BoolTFlag) getName() string {
return f.Name
}
func (f BoolTFlag) isNotHidden() bool {
return !f.Hide
}
type StringFlag struct {
Name string
Value string
Usage string
EnvVar string
Hide bool
}
func (f StringFlag) String() string {
@ -287,11 +322,10 @@ func (f StringFlag) String() string {
fmtString = "%s %v\t%v"
if len(f.Value) > 0 {
fmtString = "%s '%v'\t%v"
fmtString = "%s \"%v\"\t%v"
} else {
fmtString = "%s %v\t%v"
}
return withEnvHint(f.EnvVar, fmt.Sprintf(fmtString, prefixedNames(f.Name), f.Value, f.Usage))
}
@ -315,15 +349,20 @@ func (f StringFlag) getName() string {
return f.Name
}
func (f StringFlag) isNotHidden() bool {
return !f.Hide
}
type IntFlag struct {
Name string
Value int
Usage string
EnvVar string
Hide bool
}
func (f IntFlag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage))
return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}
func (f IntFlag) Apply(set *flag.FlagSet) {
@ -331,7 +370,7 @@ func (f IntFlag) Apply(set *flag.FlagSet) {
for _, envVar := range strings.Split(f.EnvVar, ",") {
envVar = strings.TrimSpace(envVar)
if envVal := os.Getenv(envVar); envVal != "" {
envValInt, err := strconv.ParseUint(envVal, 10, 64)
envValInt, err := strconv.ParseInt(envVal, 0, 64)
if err == nil {
f.Value = int(envValInt)
break
@ -349,15 +388,20 @@ func (f IntFlag) getName() string {
return f.Name
}
func (f IntFlag) isNotHidden() bool {
return !f.Hide
}
type DurationFlag struct {
Name string
Value time.Duration
Usage string
EnvVar string
Hide bool
}
func (f DurationFlag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage))
return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}
func (f DurationFlag) Apply(set *flag.FlagSet) {
@ -383,15 +427,20 @@ func (f DurationFlag) getName() string {
return f.Name
}
func (f DurationFlag) isNotHidden() bool {
return !f.Hide
}
type Float64Flag struct {
Name string
Value float64
Usage string
EnvVar string
Hide bool
}
func (f Float64Flag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage))
return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}
func (f Float64Flag) Apply(set *flag.FlagSet) {
@ -416,6 +465,10 @@ func (f Float64Flag) getName() string {
return f.Name
}
func (f Float64Flag) isNotHidden() bool {
return !f.Hide
}
func prefixFor(name string) (prefix string) {
if len(name) == 1 {
prefix = "-"

37
Godeps/_workspace/src/github.com/codegangsta/cli/flag_test.go → Godeps/_workspace/src/github.com/minio-io/cli/flag_test.go generated vendored
View File

@ -38,7 +38,7 @@ var stringFlagTests = []struct {
{"help", "", "--help \t"},
{"h", "", "-h \t"},
{"h", "", "-h \t"},
{"test", "Something", "--test 'Something'\t"},
{"test", "Something", "--test \"Something\"\t"},
}
func TestStringFlagHelpOutput(t *testing.T) {
@ -75,22 +75,22 @@ var stringSliceFlagTests = []struct {
s := &cli.StringSlice{}
s.Set("")
return s
}(), "--help '--help option --help option'\t"},
}(), "--help [--help option --help option]\t"},
{"h", func() *cli.StringSlice {
s := &cli.StringSlice{}
s.Set("")
return s
}(), "-h '-h option -h option'\t"},
}(), "-h [-h option -h option]\t"},
{"h", func() *cli.StringSlice {
s := &cli.StringSlice{}
s.Set("")
return s
}(), "-h '-h option -h option'\t"},
}(), "-h [-h option -h option]\t"},
{"test", func() *cli.StringSlice {
s := &cli.StringSlice{}
s.Set("Something")
return s
}(), "--test '--test option --test option'\t"},
}(), "--test [--test option --test option]\t"},
}
func TestStringSliceFlagHelpOutput(t *testing.T) {
@ -122,8 +122,8 @@ var intFlagTests = []struct {
name string
expected string
}{
{"help", "--help '0'\t"},
{"h", "-h '0'\t"},
{"help", "--help \"0\"\t"},
{"h", "-h \"0\"\t"},
}
func TestIntFlagHelpOutput(t *testing.T) {
@ -155,8 +155,8 @@ var durationFlagTests = []struct {
name string
expected string
}{
{"help", "--help '0'\t"},
{"h", "-h '0'\t"},
{"help", "--help \"0\"\t"},
{"h", "-h \"0\"\t"},
}
func TestDurationFlagHelpOutput(t *testing.T) {
@ -189,14 +189,14 @@ var intSliceFlagTests = []struct {
value *cli.IntSlice
expected string
}{
{"help", &cli.IntSlice{}, "--help '--help option --help option'\t"},
{"h", &cli.IntSlice{}, "-h '-h option -h option'\t"},
{"h", &cli.IntSlice{}, "-h '-h option -h option'\t"},
{"help", &cli.IntSlice{}, "--help [--help option --help option]\t"},
{"h", &cli.IntSlice{}, "-h [-h option -h option]\t"},
{"h", &cli.IntSlice{}, "-h [-h option -h option]\t"},
{"test", func() *cli.IntSlice {
i := &cli.IntSlice{}
i.Set("9")
return i
}(), "--test '--test option --test option'\t"},
}(), "--test [--test option --test option]\t"},
}
func TestIntSliceFlagHelpOutput(t *testing.T) {
@ -228,8 +228,8 @@ var float64FlagTests = []struct {
name string
expected string
}{
{"help", "--help '0'\t"},
{"h", "-h '0'\t"},
{"help", "--help \"0\"\t"},
{"h", "-h \"0\"\t"},
}
func TestFloat64FlagHelpOutput(t *testing.T) {
@ -262,15 +262,14 @@ var genericFlagTests = []struct {
value cli.Generic
expected string
}{
{"help", &Parser{}, "--help <nil>\t`-help option -help option` "},
{"h", &Parser{}, "-h <nil>\t`-h option -h option` "},
{"test", &Parser{}, "--test <nil>\t`-test option -test option` "},
{"test", &Parser{"abc", "def"}, "--test \"abc,def\"\ttest flag"},
{"t", &Parser{"abc", "def"}, "-t \"abc,def\"\ttest flag"},
}
func TestGenericFlagHelpOutput(t *testing.T) {
for _, test := range genericFlagTests {
flag := cli.GenericFlag{Name: test.name}
flag := cli.GenericFlag{Name: test.name, Value: test.value, Usage: "test flag"}
output := flag.String()
if output != test.expected {

89
Godeps/_workspace/src/github.com/codegangsta/cli/help.go → Godeps/_workspace/src/github.com/minio-io/cli/help.go generated vendored
View File

@ -1,11 +1,6 @@
package cli
import (
"fmt"
"os"
"text/tabwriter"
"text/template"
)
import "fmt"
// The text template for the Default help topic.
// cli.go uses text/template to render templates. You can
@ -17,14 +12,13 @@ USAGE:
{{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...]
VERSION:
{{.Version}}{{if or .Author .Email}}
{{.Version}}
AUTHOR:{{if .Author}}
{{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
{{.Email}}{{end}}{{end}}
AUTHOR(S):
{{range .Authors}}{{ . }} {{end}}
COMMANDS:
{{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
{{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
{{end}}{{if .Flags}}
GLOBAL OPTIONS:
{{range .Flags}}{{.}}
@ -58,7 +52,7 @@ USAGE:
{{.Name}} command{{if .Flags}} [command options]{{end}} [arguments...]
COMMANDS:
{{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
{{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
{{end}}{{if .Flags}}
OPTIONS:
{{range .Flags}}{{.}}
@ -66,9 +60,9 @@ OPTIONS:
`
var helpCommand = Command{
Name: "help",
ShortName: "h",
Usage: "Shows a list of commands or help for one command",
Name: "help",
Aliases: []string{"h"},
Usage: "Shows a list of commands or help for one command",
Action: func(c *Context) {
args := c.Args()
if args.Present() {
@ -80,9 +74,9 @@ var helpCommand = Command{
}
var helpSubcommand = Command{
Name: "help",
ShortName: "h",
Usage: "Shows a list of commands or help for one command",
Name: "help",
Aliases: []string{"h"},
Usage: "Shows a list of commands or help for one command",
Action: func(c *Context) {
args := c.Args()
if args.Present() {
@ -94,30 +88,61 @@ var helpSubcommand = Command{
}
// Prints help for the App
var HelpPrinter = printHelp
type helpPrinter func(templ string, data interface{})
var HelpPrinter helpPrinter = nil
// Prints version for the App
var VersionPrinter = printVersion
func ShowAppHelp(c *Context) {
HelpPrinter(AppHelpTemplate, c.App)
// Make a copy of c.App context
app := *c.App
app.Flags = make([]Flag, 0)
for _, flag := range c.App.Flags {
if flag.isNotHidden() {
app.Flags = append(app.Flags, flag)
}
}
HelpPrinter(AppHelpTemplate, app)
}
// Prints the list of subcommands as the default app completion method
func DefaultAppComplete(c *Context) {
for _, command := range c.App.Commands {
fmt.Println(command.Name)
if command.ShortName != "" {
fmt.Println(command.ShortName)
for _, name := range command.Names() {
fmt.Fprintln(c.App.Writer, name)
}
}
}
// Prints help for the given command
func ShowCommandHelp(c *Context, command string) {
// show the subcommand help for a command with subcommands
if command == "" {
// Make a copy of c.App context
app := *c.App
app.Flags = make([]Flag, 0)
for _, flag := range c.App.Flags {
if flag.isNotHidden() {
app.Flags = append(app.Flags, flag)
}
}
HelpPrinter(SubcommandHelpTemplate, app)
return
}
for _, c := range c.App.Commands {
if c.HasName(command) {
HelpPrinter(CommandHelpTemplate, c)
// Make a copy of command context
c0 := c
c0.Flags = make([]Flag, 0)
for _, flag := range c.Flags {
if flag.isNotHidden() {
c0.Flags = append(c0.Flags, flag)
}
}
HelpPrinter(CommandHelpTemplate, c0)
return
}
}
@ -125,13 +150,13 @@ func ShowCommandHelp(c *Context, command string) {
if c.App.CommandNotFound != nil {
c.App.CommandNotFound(c, command)
} else {
fmt.Printf("No help topic for '%v'\n", command)
fmt.Fprintf(c.App.Writer, "No help topic for '%v'\n", command)
}
}
// Prints help for the given subcommand
func ShowSubcommandHelp(c *Context) {
HelpPrinter(SubcommandHelpTemplate, c.App)
ShowCommandHelp(c, c.Command.Name)
}
// Prints the version number of the App
@ -140,7 +165,7 @@ func ShowVersion(c *Context) {
}
func printVersion(c *Context) {
fmt.Printf("%v version %v\n", c.App.Name, c.App.Version)
fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version)
}
// Prints the lists of commands within a given context
@ -159,16 +184,6 @@ func ShowCommandCompletions(ctx *Context, command string) {
}
}
func printHelp(templ string, data interface{}) {
w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
t := template.Must(template.New("help").Parse(templ))
err := t.Execute(w, data)
if err != nil {
panic(err)
}
w.Flush()
}
func checkVersion(c *Context) bool {
if c.GlobalBool("version") {
ShowVersion(c)

0
Godeps/_workspace/src/github.com/codegangsta/cli/helpers_test.go → Godeps/_workspace/src/github.com/minio-io/cli/helpers_test.go generated vendored
View File

87
main.go
View File

@ -19,12 +19,47 @@ package main
import (
"os"
"github.com/codegangsta/cli"
"github.com/minio-io/cli"
"github.com/minio-io/minio/pkg/server"
"github.com/minio-io/minio/pkg/utils/log"
)
func getStorageType(input string) server.StorageType {
var flags = []cli.Flag{
cli.StringFlag{
Name: "domain,d",
Value: "",
Usage: "domain used for routing incoming API requests",
},
cli.StringFlag{
Name: "api-address,a",
Value: ":9000",
Usage: "address for incoming API requests",
},
cli.StringFlag{
Name: "web-address,w",
Value: ":9001",
Usage: "address for incoming Management UI requests",
},
cli.StringFlag{
Name: "cert,c",
Hide: true,
Value: "",
Usage: "cert.pem",
},
cli.StringFlag{
Name: "key,k",
Hide: true,
Value: "",
Usage: "key.pem",
},
cli.StringFlag{
Name: "driver-type,t",
Value: "donut",
Usage: "valid entries: file,inmemory,donut",
},
}
func getDriverType(input string) server.DriverType {
switch {
case input == "file":
return server.File
@ -34,15 +69,15 @@ func getStorageType(input string) server.StorageType {
return server.Donut
default:
{
log.Println("Unknown storage type:", input)
log.Println("Choosing default storage type as 'file'..")
log.Println("Unknown driver type:", input)
log.Println("Choosing default driver type as 'file'..")
return server.File
}
}
}
func runCmd(c *cli.Context) {
storageTypeStr := c.String("storage-type")
driverTypeStr := c.String("driver-type")
domain := c.String("domain")
apiaddress := c.String("api-address")
webaddress := c.String("web-address")
@ -52,7 +87,7 @@ func runCmd(c *cli.Context) {
log.Fatal("Both certificate and key must be provided to enable https")
}
tls := (certFile != "" && keyFile != "")
storageType := getStorageType(storageTypeStr)
driverType := getDriverType(driverTypeStr)
var serverConfigs []server.Config
apiServerConfig := server.Config{
Domain: domain,
@ -61,7 +96,7 @@ func runCmd(c *cli.Context) {
CertFile: certFile,
KeyFile: keyFile,
APIType: server.MinioAPI{
StorageType: storageType,
DriverType: driverType,
},
}
webUIServerConfig := server.Config{
@ -82,39 +117,11 @@ func runCmd(c *cli.Context) {
func main() {
app := cli.NewApp()
app.Name = "minio"
app.Usage = ""
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "domain,d",
Value: "",
Usage: "domain used for routing incoming API requests",
},
cli.StringFlag{
Name: "api-address,a",
Value: ":9000",
Usage: "address for incoming API requests",
},
cli.StringFlag{
Name: "web-address,w",
Value: ":9001",
Usage: "address for incoming Management UI requests",
},
cli.StringFlag{
Name: "cert,c",
Value: "",
Usage: "cert.pem",
},
cli.StringFlag{
Name: "key,k",
Value: "",
Usage: "key.pem",
},
cli.StringFlag{
Name: "storage-type,s",
Value: "donut",
Usage: "valid entries: file,inmemory,donut",
},
}
app.Version = "0.1.0"
app.Author = "Minio.io"
app.Usage = "Minimalist Object Storage"
app.EnableBashCompletion = true
app.Flags = flags
app.Action = runCmd
app.Run(os.Args)
}

26
pkg/api/api_bucket_handlers.go
View File

@ -20,7 +20,7 @@ import (
"net/http"
"github.com/gorilla/mux"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/utils/log"
)
@ -45,21 +45,21 @@ func (server *minioAPI) listObjectsHandler(w http.ResponseWriter, req *http.Requ
}
acceptsContentType := getContentType(req)
objects, resources, err := server.storage.ListObjects(bucket, resources)
objects, resources, err := server.driver.ListObjects(bucket, resources)
switch err := err.(type) {
case nil: // success
{
response := generateObjectsListResult(bucket, objects, resources)
w.Write(writeObjectHeadersAndResponse(w, response, acceptsContentType))
}
case mstorage.BucketNotFound:
case drivers.BucketNotFound:
{
error := errorCodeError(NoSuchBucket)
errorResponse := getErrorResponse(error, bucket)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ImplementationError:
case drivers.ImplementationError:
{
// Embed error log on server side
log.Errorln(err)
@ -68,14 +68,14 @@ func (server *minioAPI) listObjectsHandler(w http.ResponseWriter, req *http.Requ
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BucketNameInvalid:
case drivers.BucketNameInvalid:
{
error := errorCodeError(InvalidBucketName)
errorResponse := getErrorResponse(error, bucket)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ObjectNameInvalid:
case drivers.ObjectNameInvalid:
{
error := errorCodeError(NoSuchKey)
errorResponse := getErrorResponse(error, resources.Prefix)
@ -91,14 +91,14 @@ func (server *minioAPI) listObjectsHandler(w http.ResponseWriter, req *http.Requ
// owned by the authenticated sender of the request.
func (server *minioAPI) listBucketsHandler(w http.ResponseWriter, req *http.Request) {
acceptsContentType := getContentType(req)
buckets, err := server.storage.ListBuckets()
buckets, err := server.driver.ListBuckets()
switch err := err.(type) {
case nil:
{
response := generateBucketsListResult(buckets)
w.Write(writeObjectHeadersAndResponse(w, response, acceptsContentType))
}
case mstorage.ImplementationError:
case drivers.ImplementationError:
{
log.Errorln(err)
error := errorCodeError(InternalError)
@ -106,7 +106,7 @@ func (server *minioAPI) listBucketsHandler(w http.ResponseWriter, req *http.Requ
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BackendCorrupted:
case drivers.BackendCorrupted:
{
log.Errorln(err)
error := errorCodeError(InternalError)
@ -123,7 +123,7 @@ func (server *minioAPI) listBucketsHandler(w http.ResponseWriter, req *http.Requ
func (server *minioAPI) putBucketHandler(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
bucket := vars["bucket"]
err := server.storage.CreateBucket(bucket)
err := server.driver.CreateBucket(bucket)
resources := getBucketResources(req.URL.Query())
if resources.Policy == true {
@ -138,21 +138,21 @@ func (server *minioAPI) putBucketHandler(w http.ResponseWriter, req *http.Reques
w.Header().Set("Server", "Minio")
w.Header().Set("Connection", "close")
}
case mstorage.BucketNameInvalid:
case drivers.BucketNameInvalid:
{
error := errorCodeError(InvalidBucketName)
errorResponse := getErrorResponse(error, bucket)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BucketExists:
case drivers.BucketExists:
{
error := errorCodeError(BucketAlreadyExists)
errorResponse := getErrorResponse(error, bucket)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ImplementationError:
case drivers.ImplementationError:
{
// Embed errors log on server side
log.Errorln(err)

38
pkg/api/api_object_handlers.go
View File

@ -20,7 +20,7 @@ import (
"net/http"
"github.com/gorilla/mux"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/utils/log"
)
@ -35,7 +35,7 @@ func (server *minioAPI) getObjectHandler(w http.ResponseWriter, req *http.Reques
bucket = vars["bucket"]
object = vars["object"]
metadata, err := server.storage.GetObjectMetadata(bucket, object, "")
metadata, err := server.driver.GetObjectMetadata(bucket, object, "")
switch err := err.(type) {
case nil: // success
{
@ -52,7 +52,7 @@ func (server *minioAPI) getObjectHandler(w http.ResponseWriter, req *http.Reques
switch httpRange.start == 0 && httpRange.length == 0 {
case true:
writeObjectHeaders(w, metadata)
if _, err := server.storage.GetObject(w, bucket, object); err != nil {
if _, err := server.driver.GetObject(w, bucket, object); err != nil {
log.Errorln(err)
error := errorCodeError(InternalError)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
@ -64,7 +64,7 @@ func (server *minioAPI) getObjectHandler(w http.ResponseWriter, req *http.Reques
metadata.Size = httpRange.length
writeRangeObjectHeaders(w, metadata, httpRange.getContentRange())
w.WriteHeader(http.StatusPartialContent)
_, err := server.storage.GetPartialObject(w, bucket, object, httpRange.start, httpRange.length)
_, err := server.driver.GetPartialObject(w, bucket, object, httpRange.start, httpRange.length)
if err != nil {
log.Errorln(err)
error := errorCodeError(InternalError)
@ -76,28 +76,28 @@ func (server *minioAPI) getObjectHandler(w http.ResponseWriter, req *http.Reques
}
}
case mstorage.ObjectNotFound:
case drivers.ObjectNotFound:
{
error := errorCodeError(NoSuchKey)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ObjectNameInvalid:
case drivers.ObjectNameInvalid:
{
error := errorCodeError(NoSuchKey)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BucketNameInvalid:
case drivers.BucketNameInvalid:
{
error := errorCodeError(InvalidBucketName)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ImplementationError:
case drivers.ImplementationError:
{
// Embed errors log on serve side
log.Errorln(err)
@ -119,25 +119,25 @@ func (server *minioAPI) headObjectHandler(w http.ResponseWriter, req *http.Reque
bucket = vars["bucket"]
object = vars["object"]
metadata, err := server.storage.GetObjectMetadata(bucket, object, "")
metadata, err := server.driver.GetObjectMetadata(bucket, object, "")
switch err := err.(type) {
case nil:
writeObjectHeaders(w, metadata)
case mstorage.ObjectNotFound:
case drivers.ObjectNotFound:
{
error := errorCodeError(NoSuchKey)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ObjectNameInvalid:
case drivers.ObjectNameInvalid:
{
error := errorCodeError(NoSuchKey)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ImplementationError:
case drivers.ImplementationError:
{
// Embed error log on server side
log.Errorln(err)
@ -167,12 +167,12 @@ func (server *minioAPI) putObjectHandler(w http.ResponseWriter, req *http.Reques
// get Content-MD5 sent by client
md5 := req.Header.Get("Content-MD5")
err := server.storage.CreateObject(bucket, object, "", md5, req.Body)
err := server.driver.CreateObject(bucket, object, "", md5, req.Body)
switch err := err.(type) {
case nil:
w.Header().Set("Server", "Minio")
w.Header().Set("Connection", "close")
case mstorage.ImplementationError:
case drivers.ImplementationError:
{
// Embed error log on server side
log.Errorln(err)
@ -181,35 +181,35 @@ func (server *minioAPI) putObjectHandler(w http.ResponseWriter, req *http.Reques
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BucketNotFound:
case drivers.BucketNotFound:
{
error := errorCodeError(NoSuchBucket)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BucketNameInvalid:
case drivers.BucketNameInvalid:
{
error := errorCodeError(InvalidBucketName)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ObjectExists:
case drivers.ObjectExists:
{
error := errorCodeError(NotImplemented)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BadDigest:
case drivers.BadDigest:
{
error := errorCodeError(BadDigest)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.InvalidDigest:
case drivers.InvalidDigest:
{
error := errorCodeError(InvalidDigest)
errorResponse := getErrorResponse(error, "/"+bucket+"/"+object)

26
pkg/api/api_policy_handlers.go
View File

@ -21,7 +21,7 @@ import (
"net/http"
"github.com/gorilla/mux"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/utils/log"
)
@ -34,7 +34,7 @@ func (server *minioAPI) putBucketPolicyHandler(w http.ResponseWriter, req *http.
bucket := vars["bucket"]
acceptsContentType := getContentType(req)
policy, ok := mstorage.Parsepolicy(req.Body)
policy, ok := drivers.Parsepolicy(req.Body)
if ok == false {
error := errorCodeError(InvalidPolicyDocument)
errorResponse := getErrorResponse(error, bucket)
@ -43,7 +43,7 @@ func (server *minioAPI) putBucketPolicyHandler(w http.ResponseWriter, req *http.
return
}
err := server.storage.CreateBucketPolicy(bucket, policy)
err := server.driver.CreateBucketPolicy(bucket, policy)
switch err := err.(type) {
case nil:
{
@ -51,21 +51,21 @@ func (server *minioAPI) putBucketPolicyHandler(w http.ResponseWriter, req *http.
writeCommonHeaders(w, getContentString(acceptsContentType))
w.Header().Set("Connection", "keep-alive")
}
case mstorage.BucketNameInvalid:
case drivers.BucketNameInvalid:
{
error := errorCodeError(InvalidBucketName)
errorResponse := getErrorResponse(error, bucket)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BucketNotFound:
case drivers.BucketNotFound:
{
error := errorCodeError(NoSuchBucket)
errorResponse := getErrorResponse(error, bucket)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BackendCorrupted:
case drivers.BackendCorrupted:
{
log.Errorln(err)
error := errorCodeError(InternalError)
@ -73,7 +73,7 @@ func (server *minioAPI) putBucketPolicyHandler(w http.ResponseWriter, req *http.
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ImplementationError:
case drivers.ImplementationError:
{
log.Errorln(err)
error := errorCodeError(InternalError)
@ -93,7 +93,7 @@ func (server *minioAPI) getBucketPolicyHandler(w http.ResponseWriter, req *http.
bucket := vars["bucket"]
acceptsContentType := getContentType(req)
p, err := server.storage.GetBucketPolicy(bucket)
p, err := server.driver.GetBucketPolicy(bucket)
switch err := err.(type) {
case nil:
{
@ -108,28 +108,28 @@ func (server *minioAPI) getBucketPolicyHandler(w http.ResponseWriter, req *http.
w.Header().Set("Connection", "keep-alive")
w.Write(responsePolicy)
}
case mstorage.BucketNameInvalid:
case drivers.BucketNameInvalid:
{
error := errorCodeError(InvalidBucketName)
errorResponse := getErrorResponse(error, bucket)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BucketNotFound:
case drivers.BucketNotFound:
{
error := errorCodeError(NoSuchBucket)
errorResponse := getErrorResponse(error, bucket)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BucketPolicyNotFound:
case drivers.BucketPolicyNotFound:
{
error := errorCodeError(NoSuchBucketPolicy)
errorResponse := getErrorResponse(error, bucket)
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.BackendCorrupted:
case drivers.BackendCorrupted:
{
log.Errorln(err)
error := errorCodeError(InternalError)
@ -137,7 +137,7 @@ func (server *minioAPI) getBucketPolicyHandler(w http.ResponseWriter, req *http.
w.WriteHeader(error.HTTPStatusCode)
w.Write(writeErrorResponse(w, errorResponse, acceptsContentType))
}
case mstorage.ImplementationError:
case drivers.ImplementationError:
{
log.Errorln(err)
error := errorCodeError(InternalError)

6
pkg/api/api_response.go
View File

@ -19,7 +19,7 @@ package api
import (
"sort"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
)
// Reply date format
@ -33,7 +33,7 @@ const (
//
// output:
// populated struct that can be serialized to match xml and json api spec output
func generateBucketsListResult(buckets []mstorage.BucketMetadata) BucketListResponse {
func generateBucketsListResult(buckets []drivers.BucketMetadata) BucketListResponse {
var listbuckets []*Bucket
var data = BucketListResponse{}
var owner = Owner{}
@ -73,7 +73,7 @@ func (b itemKey) Less(i, j int) bool { return b[i].Key < b[j].Key }
//
// output:
// populated struct that can be serialized to match xml and json api spec output
func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata, bucketResources mstorage.BucketResourcesMetadata) ObjectListResponse {
func generateObjectsListResult(bucket string, objects []drivers.ObjectMetadata, bucketResources drivers.BucketResourcesMetadata) ObjectListResponse {
var contents []*Item
var prefixes []*Prefix
var owner = Owner{}

10
pkg/api/api_router.go
View File

@ -22,13 +22,13 @@ import (
router "github.com/gorilla/mux"
"github.com/minio-io/minio/pkg/api/config"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
)
// private use
type minioAPI struct {
domain string
storage mstorage.Storage
domain string
driver drivers.Driver
}
// Path based routing
@ -72,10 +72,10 @@ func getMux(api minioAPI, mux *router.Router) *router.Router {
}
// HTTPHandler - http wrapper handler
func HTTPHandler(domain string, storage mstorage.Storage) http.Handler {
func HTTPHandler(domain string, driver drivers.Driver) http.Handler {
var mux *router.Router
var api = minioAPI{}
api.storage = storage
api.driver = driver
api.domain = domain
r := router.NewRouter()

108
pkg/api/api_test.go
View File

@ -29,8 +29,8 @@ import (
"time"
"github.com/minio-io/minio/pkg/api"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/storage/memory"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/drivers/memory"
. "gopkg.in/check.v1"
)
@ -42,8 +42,8 @@ type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestNonExistantObject(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
@ -54,14 +54,14 @@ func (s *MySuite) TestNonExistantObject(c *C) {
}
func (s *MySuite) TestEmptyObject(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
buffer := bytes.NewBufferString("")
storage.CreateBucket("bucket")
storage.CreateObject("bucket", "object", "", "", buffer)
driver.CreateBucket("bucket")
driver.CreateObject("bucket", "object", "", "", buffer)
response, err := http.Get(testServer.URL + "/bucket/object")
c.Assert(err, IsNil)
@ -71,7 +71,7 @@ func (s *MySuite) TestEmptyObject(c *C) {
c.Assert(err, IsNil)
c.Assert(true, Equals, bytes.Equal(responseBody, buffer.Bytes()))
metadata, err := storage.GetObjectMetadata("bucket", "object", "")
metadata, err := driver.GetObjectMetadata("bucket", "object", "")
c.Assert(err, IsNil)
verifyHeaders(c, response.Header, metadata.Created, 0, "application/octet-stream", metadata.Md5)
@ -79,14 +79,14 @@ func (s *MySuite) TestEmptyObject(c *C) {
}
func (s *MySuite) TestObject(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
buffer := bytes.NewBufferString("hello world")
storage.CreateBucket("bucket")
storage.CreateObject("bucket", "object", "", "", buffer)
driver.CreateBucket("bucket")
driver.CreateObject("bucket", "object", "", "", buffer)
response, err := http.Get(testServer.URL + "/bucket/object")
c.Assert(err, IsNil)
@ -96,14 +96,14 @@ func (s *MySuite) TestObject(c *C) {
c.Assert(err, IsNil)
c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello world")))
metadata, err := storage.GetObjectMetadata("bucket", "object", "")
metadata, err := driver.GetObjectMetadata("bucket", "object", "")
c.Assert(err, IsNil)
verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", metadata.Md5)
}
func (s *MySuite) TestMultipleObjects(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
@ -111,10 +111,10 @@ func (s *MySuite) TestMultipleObjects(c *C) {
buffer2 := bytes.NewBufferString("hello two")
buffer3 := bytes.NewBufferString("hello three")
storage.CreateBucket("bucket")
storage.CreateObject("bucket", "object1", "", "", buffer1)
storage.CreateObject("bucket", "object2", "", "", buffer2)
storage.CreateObject("bucket", "object3", "", "", buffer3)
driver.CreateBucket("bucket")
driver.CreateObject("bucket", "object1", "", "", buffer1)
driver.CreateObject("bucket", "object2", "", "", buffer2)
driver.CreateObject("bucket", "object3", "", "", buffer3)
// test non-existant object
response, err := http.Get(testServer.URL + "/bucket/object")
@ -129,7 +129,7 @@ func (s *MySuite) TestMultipleObjects(c *C) {
c.Assert(err, IsNil)
// get metadata
metadata, err := storage.GetObjectMetadata("bucket", "object1", "")
metadata, err := driver.GetObjectMetadata("bucket", "object1", "")
c.Assert(err, IsNil)
c.Assert(response.StatusCode, Equals, http.StatusOK)
@ -148,7 +148,7 @@ func (s *MySuite) TestMultipleObjects(c *C) {
c.Assert(err, IsNil)
// get metadata
metadata, err = storage.GetObjectMetadata("bucket", "object2", "")
metadata, err = driver.GetObjectMetadata("bucket", "object2", "")
c.Assert(err, IsNil)
c.Assert(response.StatusCode, Equals, http.StatusOK)
@ -167,7 +167,7 @@ func (s *MySuite) TestMultipleObjects(c *C) {
c.Assert(err, IsNil)
// get metadata
metadata, err = storage.GetObjectMetadata("bucket", "object3", "")
metadata, err = driver.GetObjectMetadata("bucket", "object3", "")
c.Assert(err, IsNil)
c.Assert(response.StatusCode, Equals, http.StatusOK)
@ -182,8 +182,8 @@ func (s *MySuite) TestMultipleObjects(c *C) {
}
func (s *MySuite) TestNotImplemented(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
@ -193,8 +193,8 @@ func (s *MySuite) TestNotImplemented(c *C) {
}
func (s *MySuite) TestHeader(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
@ -203,25 +203,25 @@ func (s *MySuite) TestHeader(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusNotFound)
buffer := bytes.NewBufferString("hello world")
storage.CreateBucket("bucket")
storage.CreateObject("bucket", "object", "", "", buffer)
driver.CreateBucket("bucket")
driver.CreateObject("bucket", "object", "", "", buffer)
response, err = http.Get(testServer.URL + "/bucket/object")
c.Assert(err, IsNil)
c.Assert(response.StatusCode, Equals, http.StatusOK)
metadata, err := storage.GetObjectMetadata("bucket", "object", "")
metadata, err := driver.GetObjectMetadata("bucket", "object", "")
c.Assert(err, IsNil)
verifyHeaders(c, response.Header, metadata.Created, len("hello world"), "application/octet-stream", metadata.Md5)
}
func (s *MySuite) TestPutBucket(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
buckets, err := storage.ListBuckets()
buckets, err := driver.ListBuckets()
c.Assert(len(buckets), Equals, 0)
c.Assert(err, IsNil)
@ -234,23 +234,23 @@ func (s *MySuite) TestPutBucket(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
// check bucket exists
buckets, err = storage.ListBuckets()
buckets, err = driver.ListBuckets()
c.Assert(len(buckets), Equals, 1)
c.Assert(err, IsNil)
c.Assert(buckets[0].Name, Equals, "bucket")
}
func (s *MySuite) TestPutObject(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
resources := mstorage.BucketResourcesMetadata{}
resources := drivers.BucketResourcesMetadata{}
resources.Maxkeys = 1000
resources.Prefix = ""
objects, resources, err := storage.ListObjects("bucket", resources)
objects, resources, err := driver.ListObjects("bucket", resources)
c.Assert(len(objects), Equals, 0)
c.Assert(resources.IsTruncated, Equals, false)
c.Assert(err, Not(IsNil))
@ -278,18 +278,18 @@ func (s *MySuite) TestPutObject(c *C) {
resources.Maxkeys = 1000
resources.Prefix = ""
objects, resources, err = storage.ListObjects("bucket", resources)
objects, resources, err = driver.ListObjects("bucket", resources)
c.Assert(len(objects), Equals, 1)
c.Assert(resources.IsTruncated, Equals, false)
c.Assert(err, IsNil)
var writer bytes.Buffer
storage.GetObject(&writer, "bucket", "two")
driver.GetObject(&writer, "bucket", "two")
c.Assert(bytes.Equal(writer.Bytes(), []byte("hello world")), Equals, true)
metadata, err := storage.GetObjectMetadata("bucket", "two", "")
metadata, err := driver.GetObjectMetadata("bucket", "two", "")
c.Assert(err, IsNil)
lastModified := metadata.Created
@ -298,8 +298,8 @@ func (s *MySuite) TestPutObject(c *C) {
}
func (s *MySuite) TestListBuckets(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
@ -312,7 +312,7 @@ func (s *MySuite) TestListBuckets(c *C) {
c.Assert(err, IsNil)
c.Assert(len(listResponse.Buckets.Bucket), Equals, 0)
storage.CreateBucket("foo")
driver.CreateBucket("foo")
response, err = http.Get(testServer.URL + "/")
defer response.Body.Close()
@ -324,7 +324,7 @@ func (s *MySuite) TestListBuckets(c *C) {
c.Assert(len(listResponse.Buckets.Bucket), Equals, 1)
c.Assert(listResponse.Buckets.Bucket[0].Name, Equals, "foo")
storage.CreateBucket("bar")
driver.CreateBucket("bar")
response, err = http.Get(testServer.URL + "/")
defer response.Body.Close()
@ -377,12 +377,12 @@ func verifyHeaders(c *C, header http.Header, date time.Time, size int, contentTy
}
func (s *MySuite) TestXMLNameNotInBucketListJson(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
err := storage.CreateBucket("foo")
err := driver.CreateBucket("foo")
c.Assert(err, IsNil)
request, err := http.NewRequest("GET", testServer.URL+"/", bytes.NewBufferString(""))
@ -402,12 +402,12 @@ func (s *MySuite) TestXMLNameNotInBucketListJson(c *C) {
}
func (s *MySuite) TestXMLNameNotInObjectListJson(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
err := storage.CreateBucket("foo")
err := driver.CreateBucket("foo")
c.Assert(err, IsNil)
request, err := http.NewRequest("GET", testServer.URL+"/foo", bytes.NewBufferString(""))
@ -427,12 +427,12 @@ func (s *MySuite) TestXMLNameNotInObjectListJson(c *C) {
}
func (s *MySuite) TestContentTypePersists(c *C) {
_, _, storage := memory.Start()
httpHandler := api.HTTPHandler("", storage)
_, _, driver := memory.Start()
httpHandler := api.HTTPHandler("", driver)
testServer := httptest.NewServer(httpHandler)
defer testServer.Close()
err := storage.CreateBucket("bucket")
err := driver.CreateBucket("bucket")
c.Assert(err, IsNil)
client := http.Client{}

6
pkg/api/headers.go
View File

@ -24,7 +24,7 @@ import (
"strconv"
"time"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
)
// No encoder interface exists, so we create one.
@ -58,7 +58,7 @@ func writeErrorResponse(w http.ResponseWriter, response interface{}, acceptsType
}
// Write object header
func writeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMetadata) {
func writeObjectHeaders(w http.ResponseWriter, metadata drivers.ObjectMetadata) {
lastModified := metadata.Created.Format(time.RFC1123)
// common headers
writeCommonHeaders(w, metadata.ContentType)
@ -69,7 +69,7 @@ func writeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMetadata)
}
// Write range object header
func writeRangeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMetadata, ra string) {
func writeRangeObjectHeaders(w http.ResponseWriter, metadata drivers.ObjectMetadata, ra string) {
lastModified := metadata.Created.Format(time.RFC1123)
// common headers
writeCommonHeaders(w, metadata.ContentType)

4
pkg/api/resources.go
View File

@ -20,11 +20,11 @@ import (
"net/url"
"strconv"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
)
// parse bucket url queries
func getBucketResources(values url.Values) (v mstorage.BucketResourcesMetadata) {
func getBucketResources(values url.Values) (v drivers.BucketResourcesMetadata) {
for key, value := range values {
switch true {
case key == "prefix":

176
pkg/storage/storage_api_suite.go → pkg/drivers/api_testsuite.go
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package storage
package drivers
import (
"bytes"
@ -26,7 +26,7 @@ import (
)
// APITestSuite - collection of API tests
func APITestSuite(c *check.C, create func() Storage) {
func APITestSuite(c *check.C, create func() Driver) {
testCreateBucket(c, create)
testMultipleObjectCreation(c, create)
testPaging(c, create)
@ -43,16 +43,16 @@ func APITestSuite(c *check.C, create func() Storage) {
//testContentMd5Set(c, create) TODO
}
func testCreateBucket(c *check.C, create func() Storage) {
storage := create()
err := storage.CreateBucket("bucket")
func testCreateBucket(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
}
func testMultipleObjectCreation(c *check.C, create func() Storage) {
func testMultipleObjectCreation(c *check.C, create func() Driver) {
objects := make(map[string][]byte)
storage := create()
err := storage.CreateBucket("bucket")
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
for i := 0; i < 10; i++ {
randomPerm := rand.Perm(10)
@ -62,7 +62,7 @@ func testMultipleObjectCreation(c *check.C, create func() Storage) {
}
key := "obj" + strconv.Itoa(i)
objects[key] = []byte(randomString)
err := storage.CreateObject("bucket", key, "", "", bytes.NewBufferString(randomString))
err := drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(randomString))
c.Assert(err, check.IsNil)
}
@ -70,11 +70,11 @@ func testMultipleObjectCreation(c *check.C, create func() Storage) {
etags := make(map[string]string)
for key, value := range objects {
var byteBuffer bytes.Buffer
_, err := storage.GetObject(&byteBuffer, "bucket", key)
_, err := drivers.GetObject(&byteBuffer, "bucket", key)
c.Assert(err, check.IsNil)
c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
metadata, err := storage.GetObjectMetadata("bucket", key, "")
metadata, err := drivers.GetObjectMetadata("bucket", key, "")
c.Assert(err, check.IsNil)
c.Assert(metadata.Size, check.Equals, int64(len(value)))
@ -84,20 +84,20 @@ func testMultipleObjectCreation(c *check.C, create func() Storage) {
}
}
func testPaging(c *check.C, create func() Storage) {
storage := create()
storage.CreateBucket("bucket")
func testPaging(c *check.C, create func() Driver) {
drivers := create()
drivers.CreateBucket("bucket")
resources := BucketResourcesMetadata{}
objects, resources, err := storage.ListObjects("bucket", resources)
objects, resources, err := drivers.ListObjects("bucket", resources)
c.Assert(err, check.IsNil)
c.Assert(len(objects), check.Equals, 0)
c.Assert(resources.IsTruncated, check.Equals, false)
// check before paging occurs
for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i)
storage.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
resources.Maxkeys = 5
objects, resources, err = storage.ListObjects("bucket", resources)
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(len(objects), check.Equals, i+1)
c.Assert(resources.IsTruncated, check.Equals, false)
c.Assert(err, check.IsNil)
@ -105,20 +105,20 @@ func testPaging(c *check.C, create func() Storage) {
// check after paging occurs pages work
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
storage.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
resources.Maxkeys = 5
objects, resources, err = storage.ListObjects("bucket", resources)
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(len(objects), check.Equals, 5)
c.Assert(resources.IsTruncated, check.Equals, true)
c.Assert(err, check.IsNil)
}
// check paging with prefix at end returns less objects
{
storage.CreateObject("bucket", "newPrefix", "", "", bytes.NewBufferString("prefix1"))
storage.CreateObject("bucket", "newPrefix2", "", "", bytes.NewBufferString("prefix2"))
drivers.CreateObject("bucket", "newPrefix", "", "", bytes.NewBufferString("prefix1"))
drivers.CreateObject("bucket", "newPrefix2", "", "", bytes.NewBufferString("prefix2"))
resources.Prefix = "new"
resources.Maxkeys = 5
objects, resources, err = storage.ListObjects("bucket", resources)
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(len(objects), check.Equals, 2)
}
@ -126,7 +126,7 @@ func testPaging(c *check.C, create func() Storage) {
{
resources.Prefix = ""
resources.Maxkeys = 1000
objects, resources, err = storage.ListObjects("bucket", resources)
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "newPrefix")
c.Assert(objects[1].Key, check.Equals, "newPrefix2")
c.Assert(objects[2].Key, check.Equals, "obj0")
@ -136,14 +136,14 @@ func testPaging(c *check.C, create func() Storage) {
// check delimited results with delimiter and prefix
{
storage.CreateObject("bucket", "this/is/delimited", "", "", bytes.NewBufferString("prefix1"))
storage.CreateObject("bucket", "this/is/also/a/delimited/file", "", "", bytes.NewBufferString("prefix2"))
drivers.CreateObject("bucket", "this/is/delimited", "", "", bytes.NewBufferString("prefix1"))
drivers.CreateObject("bucket", "this/is/also/a/delimited/file", "", "", bytes.NewBufferString("prefix2"))
var prefixes []string
resources.CommonPrefixes = prefixes // allocate new everytime
resources.Delimiter = "/"
resources.Prefix = "this/is/"
resources.Maxkeys = 10
objects, resources, err = storage.ListObjects("bucket", resources)
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(err, check.IsNil)
c.Assert(len(objects), check.Equals, 1)
c.Assert(resources.CommonPrefixes[0], check.Equals, "also/")
@ -157,7 +157,7 @@ func testPaging(c *check.C, create func() Storage) {
resources.Delimiter = "/"
resources.Prefix = ""
resources.Maxkeys = 1000
objects, resources, err = storage.ListObjects("bucket", resources)
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "newPrefix")
c.Assert(objects[1].Key, check.Equals, "newPrefix2")
c.Assert(objects[2].Key, check.Equals, "obj0")
@ -171,7 +171,7 @@ func testPaging(c *check.C, create func() Storage) {
resources.Prefix = "obj"
resources.Delimiter = ""
resources.Maxkeys = 1000
objects, resources, err = storage.ListObjects("bucket", resources)
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "obj0")
c.Assert(objects[1].Key, check.Equals, "obj1")
c.Assert(objects[2].Key, check.Equals, "obj10")
@ -182,95 +182,95 @@ func testPaging(c *check.C, create func() Storage) {
{
resources.Prefix = "new"
resources.Maxkeys = 5
objects, resources, err = storage.ListObjects("bucket", resources)
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "newPrefix")
c.Assert(objects[1].Key, check.Equals, "newPrefix2")
}
}
func testObjectOverwriteFails(c *check.C, create func() Storage) {
storage := create()
storage.CreateBucket("bucket")
err := storage.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
func testObjectOverwriteFails(c *check.C, create func() Driver) {
drivers := create()
drivers.CreateBucket("bucket")
err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
c.Assert(err, check.IsNil)
err = storage.CreateObject("bucket", "object", "", "", bytes.NewBufferString("three"))
err = drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("three"))
c.Assert(err, check.Not(check.IsNil))
var bytesBuffer bytes.Buffer
length, err := storage.GetObject(&bytesBuffer, "bucket", "object")
length, err := drivers.GetObject(&bytesBuffer, "bucket", "object")
c.Assert(length, check.Equals, int64(len("one")))
c.Assert(err, check.IsNil)
c.Assert(string(bytesBuffer.Bytes()), check.Equals, "one")
}
func testNonExistantBucketOperations(c *check.C, create func() Storage) {
storage := create()
err := storage.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
func testNonExistantBucketOperations(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil))
}
func testBucketRecreateFails(c *check.C, create func() Storage) {
storage := create()
err := storage.CreateBucket("string")
func testBucketRecreateFails(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("string")
c.Assert(err, check.IsNil)
err = storage.CreateBucket("string")
err = drivers.CreateBucket("string")
c.Assert(err, check.Not(check.IsNil))
}
func testPutObjectInSubdir(c *check.C, create func() Storage) {
storage := create()
err := storage.CreateBucket("bucket")
func testPutObjectInSubdir(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
err = storage.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil)
var bytesBuffer bytes.Buffer
length, err := storage.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object")
length, err := drivers.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object")
c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length)
c.Assert(err, check.IsNil)
}
func testListBuckets(c *check.C, create func() Storage) {
storage := create()
func testListBuckets(c *check.C, create func() Driver) {
drivers := create()
// test empty list
buckets, err := storage.ListBuckets()
buckets, err := drivers.ListBuckets()
c.Assert(err, check.IsNil)
c.Assert(len(buckets), check.Equals, 0)
// add one and test exists
err = storage.CreateBucket("bucket1")
err = drivers.CreateBucket("bucket1")
c.Assert(err, check.IsNil)
buckets, err = storage.ListBuckets()
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 1)
c.Assert(err, check.IsNil)
// add two and test exists
err = storage.CreateBucket("bucket2")
err = drivers.CreateBucket("bucket2")
c.Assert(err, check.IsNil)
buckets, err = storage.ListBuckets()
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 2)
c.Assert(err, check.IsNil)
// add three and test exists + prefix
err = storage.CreateBucket("bucket22")
err = drivers.CreateBucket("bucket22")
buckets, err = storage.ListBuckets()
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 3)
c.Assert(err, check.IsNil)
}
func testListBucketsOrder(c *check.C, create func() Storage) {
func testListBucketsOrder(c *check.C, create func() Driver) {
// if implementation contains a map, order of map keys will vary.
// this ensures they return in the same order each time
for i := 0; i < 10; i++ {
storage := create()
drivers := create()
// add one and test exists
storage.CreateBucket("bucket1")
storage.CreateBucket("bucket2")
drivers.CreateBucket("bucket1")
drivers.CreateBucket("bucket2")
buckets, err := storage.ListBuckets()
buckets, err := drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 2)
c.Assert(err, check.IsNil)
c.Assert(buckets[0].Name, check.Equals, "bucket1")
@ -278,22 +278,22 @@ func testListBucketsOrder(c *check.C, create func() Storage) {
}
}
func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Storage) {
storage := create()
func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Driver) {
drivers := create()
resources := BucketResourcesMetadata{Prefix: "", Maxkeys: 1000}
objects, resources, err := storage.ListObjects("bucket", resources)
objects, resources, err := drivers.ListObjects("bucket", resources)
c.Assert(err, check.Not(check.IsNil))
c.Assert(resources.IsTruncated, check.Equals, false)
c.Assert(len(objects), check.Equals, 0)
}
func testNonExistantObjectInBucket(c *check.C, create func() Storage) {
storage := create()
err := storage.CreateBucket("bucket")
func testNonExistantObjectInBucket(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
length, err := storage.GetObject(&byteBuffer, "bucket", "dir1")
length, err := drivers.GetObject(&byteBuffer, "bucket", "dir1")
c.Assert(length, check.Equals, int64(0))
c.Assert(err, check.Not(check.IsNil))
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
@ -309,16 +309,16 @@ func testNonExistantObjectInBucket(c *check.C, create func() Storage) {
}
}
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Storage) {
storage := create()
err := storage.CreateBucket("bucket")
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
err = storage.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
length, err := storage.GetObject(&byteBuffer, "bucket", "dir1")
length, err := drivers.GetObject(&byteBuffer, "bucket", "dir1")
c.Assert(length, check.Equals, int64(0))
switch err := err.(type) {
case ObjectNotFound:
@ -335,7 +335,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Storage) {
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
var byteBuffer2 bytes.Buffer
length, err = storage.GetObject(&byteBuffer, "bucket", "dir1/")
length, err = drivers.GetObject(&byteBuffer, "bucket", "dir1/")
c.Assert(length, check.Equals, int64(0))
switch err := err.(type) {
case ObjectNotFound:
@ -352,40 +352,40 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Storage) {
c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
}
func testDefaultContentType(c *check.C, create func() Storage) {
storage := create()
err := storage.CreateBucket("bucket")
func testDefaultContentType(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
// test empty
err = storage.CreateObject("bucket", "one", "", "", bytes.NewBufferString("one"))
metadata, err := storage.GetObjectMetadata("bucket", "one", "")
err = drivers.CreateObject("bucket", "one", "", "", bytes.NewBufferString("one"))
metadata, err := drivers.GetObjectMetadata("bucket", "one", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
// test custom
storage.CreateObject("bucket", "two", "application/text", "", bytes.NewBufferString("two"))
metadata, err = storage.GetObjectMetadata("bucket", "two", "")
drivers.CreateObject("bucket", "two", "application/text", "", bytes.NewBufferString("two"))
metadata, err = drivers.GetObjectMetadata("bucket", "two", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/text")
// test trim space
storage.CreateObject("bucket", "three", "\tapplication/json ", "", bytes.NewBufferString("three"))
metadata, err = storage.GetObjectMetadata("bucket", "three", "")
drivers.CreateObject("bucket", "three", "\tapplication/json ", "", bytes.NewBufferString("three"))
metadata, err = drivers.GetObjectMetadata("bucket", "three", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/json")
}
/*
func testContentMd5Set(c *check.C, create func() Storage) {
storage := create()
err := storage.CreateBucket("bucket")
func testContentMd5Set(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
// test md5 invalid
err = storage.CreateObject("bucket", "one", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA", bytes.NewBufferString("one"))
err = drivers.CreateObject("bucket", "one", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA", bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil))
err = storage.CreateObject("bucket", "two", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=", bytes.NewBufferString("one"))
err = drivers.CreateObject("bucket", "two", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=", bytes.NewBufferString("one"))
c.Assert(err, check.IsNil)
}
*/

2
pkg/storage/storage_bucket_policy.go → pkg/drivers/bucket_policy.go
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package storage
package drivers
import (
"encoding/json"

2
pkg/storage/storage_bucket_policy_compat.go → pkg/drivers/bucket_policy_compat.go
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package storage
package drivers
// This file implements compatability layer for AWS clients

18
pkg/storage/storage_date.go → pkg/drivers/date.go
View File

@ -1,4 +1,20 @@
package storage
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
import (
"errors"

78
pkg/storage/donutstorage/donutstorage.go → pkg/drivers/donut/donut.go
View File

@ -14,21 +14,22 @@
* limitations under the License.
*/
package donutstorage
package donut
import (
"errors"
"github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/storage/donut"
"io"
"sort"
"strconv"
"strings"
"time"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/storage/donut"
)
// Storage creates a new single disk storage driver using donut without encoding.
type Storage struct {
// donutDriver - creates a new single disk drivers driver using donut
type donutDriver struct {
donut donut.Donut
}
@ -37,15 +38,14 @@ const (
)
// Start a single disk subsystem
func Start(path string) (chan<- string, <-chan error, storage.Storage) {
func Start(path string) (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
s := new(Storage)
s := new(donutDriver)
// TODO donut driver should be passed in as Start param and driven by config
var err error
s.donut, err = donut.NewDonutDriver(path)
s.donut, err = donut.NewDonut(path)
if err != nil {
errorChannel <- err
}
@ -54,13 +54,13 @@ func Start(path string) (chan<- string, <-chan error, storage.Storage) {
return ctrlChannel, errorChannel, s
}
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *Storage) {
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *donutDriver) {
close(errorChannel)
}
// ListBuckets returns a list of buckets
func (donutStorage Storage) ListBuckets() (results []storage.BucketMetadata, err error) {
buckets, err := donutStorage.donut.ListBuckets()
func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error) {
buckets, err := d.donut.ListBuckets()
if err != nil {
return nil, err
}
@ -68,7 +68,7 @@ func (donutStorage Storage) ListBuckets() (results []storage.BucketMetadata, err
if err != nil {
return nil, err
}
result := storage.BucketMetadata{
result := drivers.BucketMetadata{
Name: bucket,
// TODO Add real created date
Created: time.Now(),
@ -79,30 +79,30 @@ func (donutStorage Storage) ListBuckets() (results []storage.BucketMetadata, err
}
// CreateBucket creates a new bucket
func (donutStorage Storage) CreateBucket(bucket string) error {
return donutStorage.donut.CreateBucket(bucket)
func (d donutDriver) CreateBucket(bucket string) error {
return d.donut.CreateBucket(bucket)
}
// GetBucketMetadata retrieves an bucket's metadata
func (donutStorage Storage) GetBucketMetadata(bucket string) (storage.BucketMetadata, error) {
return storage.BucketMetadata{}, errors.New("Not Implemented")
func (d donutDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
return drivers.BucketMetadata{}, errors.New("Not Implemented")
}
// CreateBucketPolicy sets a bucket's access policy
func (donutStorage Storage) CreateBucketPolicy(bucket string, p storage.BucketPolicy) error {
func (d donutDriver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error {
return errors.New("Not Implemented")
}
// GetBucketPolicy returns a bucket's access policy
func (donutStorage Storage) GetBucketPolicy(bucket string) (storage.BucketPolicy, error) {
return storage.BucketPolicy{}, errors.New("Not Implemented")
func (d donutDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
return drivers.BucketPolicy{}, errors.New("Not Implemented")
}
// GetObject retrieves an object and writes it to a writer
func (donutStorage Storage) GetObject(target io.Writer, bucket, key string) (int64, error) {
reader, err := donutStorage.donut.GetObject(bucket, key)
func (d donutDriver) GetObject(target io.Writer, bucket, key string) (int64, error) {
reader, err := d.donut.GetObjectReader(bucket, key)
if err != nil {
return 0, storage.ObjectNotFound{
return 0, drivers.ObjectNotFound{
Bucket: bucket,
Object: key,
}
@ -110,23 +110,23 @@ func (donutStorage Storage) GetObject(target io.Writer, bucket, key string) (int
return io.Copy(target, reader)
}
// GetPartialObject retrieves an object and writes it to a writer
func (donutStorage Storage) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
// GetPartialObject retrieves an object range and writes it to a writer
func (d donutDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
return 0, errors.New("Not Implemented")
}
// GetObjectMetadata retrieves an object's metadata
func (donutStorage Storage) GetObjectMetadata(bucket, key string, prefix string) (storage.ObjectMetadata, error) {
metadata, err := donutStorage.donut.GetObjectMetadata(bucket, key)
func (d donutDriver) GetObjectMetadata(bucket, key string, prefix string) (drivers.ObjectMetadata, error) {
metadata, err := d.donut.GetObjectMetadata(bucket, key)
created, err := time.Parse(time.RFC3339Nano, metadata["sys.created"])
if err != nil {
return storage.ObjectMetadata{}, err
return drivers.ObjectMetadata{}, err
}
size, err := strconv.ParseInt(metadata["sys.size"], 10, 64)
if err != nil {
return storage.ObjectMetadata{}, err
return drivers.ObjectMetadata{}, err
}
objectMetadata := storage.ObjectMetadata{
objectMetadata := drivers.ObjectMetadata{
Bucket: bucket,
Key: key,
@ -138,12 +138,12 @@ func (donutStorage Storage) GetObjectMetadata(bucket, key string, prefix string)
return objectMetadata, nil
}
// ListObjects lists objects
func (donutStorage Storage) ListObjects(bucket string, resources storage.BucketResourcesMetadata) ([]storage.ObjectMetadata, storage.BucketResourcesMetadata, error) {
// ListObjects - returns list of objects
func (d donutDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
// TODO Fix IsPrefixSet && IsDelimiterSet and use them
objects, err := donutStorage.donut.ListObjects(bucket)
objects, err := d.donut.ListObjects(bucket)
if err != nil {
return nil, storage.BucketResourcesMetadata{}, err
return nil, drivers.BucketResourcesMetadata{}, err
}
sort.Strings(objects)
if resources.Prefix != "" {
@ -166,15 +166,15 @@ func (donutStorage Storage) ListObjects(bucket string, resources storage.BucketR
actualObjects = objects
}
var results []storage.ObjectMetadata
var results []drivers.ObjectMetadata
for _, object := range actualObjects {
if len(results) >= resources.Maxkeys {
resources.IsTruncated = true
break
}
metadata, err := donutStorage.GetObjectMetadata(bucket, resources.Prefix+object, "")
metadata, err := d.GetObjectMetadata(bucket, resources.Prefix+object, "")
if err != nil {
return nil, storage.BucketResourcesMetadata{}, err
return nil, drivers.BucketResourcesMetadata{}, err
}
results = append(results, metadata)
}
@ -241,8 +241,8 @@ func uniqueObjects(objects []string) []string {
}
// CreateObject creates a new object
func (donutStorage Storage) CreateObject(bucketKey, objectKey, contentType, expectedMd5sum string, reader io.Reader) error {
writer, err := donutStorage.donut.GetObjectWriter(bucketKey, objectKey)
func (d donutDriver) CreateObject(bucketKey, objectKey, contentType, expectedMd5sum string, reader io.Reader) error {
writer, err := d.donut.GetObjectWriter(bucketKey, objectKey)
if err != nil {
return err
}

8
pkg/storage/donutstorage/donutstorage_test.go → pkg/drivers/donut/donut_test.go
View File

@ -14,14 +14,14 @@
* limitations under the License.
*/
package donutstorage
package donut
import (
"io/ioutil"
"os"
"testing"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
. "gopkg.in/check.v1"
)
@ -35,14 +35,14 @@ var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
// c.Skip("Not Implemented")
var storageList []string
create := func() mstorage.Storage {
create := func() drivers.Driver {
path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
c.Check(err, IsNil)
storageList = append(storageList, path)
_, _, store := Start(path) // TODO Make InMemory driver
return store
}
mstorage.APITestSuite(c, create)
drivers.APITestSuite(c, create)
removeRoots(c, storageList)
}

6
pkg/storage/storage.go → pkg/drivers/driver.go
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package storage
package drivers
import (
"io"
@ -23,8 +23,8 @@ import (
"unicode/utf8"
)
// Storage - generic API interface
type Storage interface {
// Driver - generic API interface for various drivers - donut, file, memory
type Driver interface {
// Bucket Operations
ListBuckets() ([]BucketMetadata, error)
CreateBucket(bucket string) error

2
pkg/storage/storage_errors.go → pkg/drivers/errors.go
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package storage
package drivers
// BackendError - generic disk backend error
type BackendError struct {

12
pkg/storage/file/file.go → pkg/drivers/file/file.go
View File

@ -19,20 +19,22 @@ package file
import (
"os"
"sync"
"github.com/minio-io/minio/pkg/drivers"
)
// Start filesystem channel
func Start(root string) (chan<- string, <-chan error, *Storage) {
func Start(root string) (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
s := Storage{}
s := new(fileDriver)
s.root = root
s.lock = new(sync.Mutex)
go start(ctrlChannel, errorChannel, &s)
return ctrlChannel, errorChannel, &s
go start(ctrlChannel, errorChannel, s)
return ctrlChannel, errorChannel, s
}
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *Storage) {
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *fileDriver) {
err := os.MkdirAll(s.root, 0700)
errorChannel <- err
close(errorChannel)

133
pkg/drivers/file/file_bucket.go Normal file
View File

@ -0,0 +1,133 @@
/*
* Minimalist Object File, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"path"
"sort"
"strings"
"io/ioutil"
"path/filepath"
"github.com/minio-io/minio/pkg/drivers"
)
/// Bucket Operations
// ListBuckets - Get service
func (file *fileDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
files, err := ioutil.ReadDir(file.root)
if err != nil {
return []drivers.BucketMetadata{}, drivers.EmbedError("bucket", "", err)
}
var metadataList []drivers.BucketMetadata
for _, fileName := range files {
// Skip policy files
if strings.HasSuffix(fileName.Name(), "_policy.json") {
continue
}
if !fileName.IsDir() {
return []drivers.BucketMetadata{}, drivers.BackendCorrupted{Path: file.root}
}
metadata := drivers.BucketMetadata{
Name: fileName.Name(),
Created: fileName.ModTime(), // TODO - provide real created time
}
metadataList = append(metadataList, metadata)
}
return metadataList, nil
}
// CreateBucket - PUT Bucket
func (file *fileDriver) CreateBucket(bucket string) error {
file.lock.Lock()
defer file.lock.Unlock()
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(file.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err == nil {
return drivers.BucketExists{
Bucket: bucket,
}
}
// make bucket
err := os.Mkdir(bucketDir, 0700)
if err != nil {
return drivers.EmbedError(bucket, "", err)
}
return nil
}
// ListObjects - GET bucket (list objects)
func (file *fileDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
p := bucketDir{}
p.files = make(map[string]os.FileInfo)
if drivers.IsValidBucket(bucket) == false {
return []drivers.ObjectMetadata{}, resources, drivers.BucketNameInvalid{Bucket: bucket}
}
if resources.Prefix != "" && drivers.IsValidObject(resources.Prefix) == false {
return []drivers.ObjectMetadata{}, resources, drivers.ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix}
}
rootPrefix := path.Join(file.root, bucket)
// check bucket exists
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
return []drivers.ObjectMetadata{}, resources, drivers.BucketNotFound{Bucket: bucket}
}
p.root = rootPrefix
err := filepath.Walk(rootPrefix, p.getAllFiles)
if err != nil {
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
var metadataList []drivers.ObjectMetadata
var metadata drivers.ObjectMetadata
// Populate filtering mode
resources.Mode = drivers.GetMode(resources)
for name, f := range p.files {
if len(metadataList) >= resources.Maxkeys {
resources.IsTruncated = true
goto ret
}
metadata, resources, err = file.filter(bucket, name, f, resources)
if err != nil {
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
if metadata.Bucket != "" {
metadataList = append(metadataList, metadata)
}
}
ret:
sort.Sort(byObjectKey(metadataList))
return metadataList, resources, nil
}

12
pkg/storage/file/file_common.go → pkg/drivers/file/file_common.go
View File

@ -23,17 +23,17 @@ import (
"strings"
"sync"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
)
// Storage - file local variables
type Storage struct {
// fileDriver - file local variables
type fileDriver struct {
root string
lock *sync.Mutex
}
// Metadata - carries metadata about object
type Metadata struct {
// fileMetadata - carries metadata about object
type fileMetadata struct {
Md5sum []byte
ContentType string
}
@ -77,7 +77,7 @@ func delimiter(object, delimiter string) string {
return delimitedStr
}
type byObjectKey []mstorage.ObjectMetadata
type byObjectKey []drivers.ObjectMetadata
// Len
func (b byObjectKey) Len() int { return len(b) }

34
pkg/storage/file/file_filter.go → pkg/drivers/file/file_filter.go
View File

@ -20,13 +20,13 @@ import (
"os"
"strings"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
)
// TODO handle resources.Marker
func (storage *Storage) filter(bucket, name string, file os.FileInfo, resources mstorage.BucketResourcesMetadata) (mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) {
func (file *fileDriver) filter(bucket, name string, f os.FileInfo, resources drivers.BucketResourcesMetadata) (drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
var err error
var metadata mstorage.ObjectMetadata
var metadata drivers.ObjectMetadata
switch true {
// Both delimiter and Prefix is present
@ -37,15 +37,15 @@ func (storage *Storage) filter(bucket, name string, file os.FileInfo, resources
switch true {
case name == resources.Prefix:
// Use resources.Prefix to filter out delimited files
metadata, err = storage.GetObjectMetadata(bucket, name, resources.Prefix)
metadata, err = file.GetObjectMetadata(bucket, name, resources.Prefix)
if err != nil {
return mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err)
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName == file.Name():
case delimitedName == f.Name():
// Use resources.Prefix to filter out delimited files
metadata, err = storage.GetObjectMetadata(bucket, name, resources.Prefix)
metadata, err = file.GetObjectMetadata(bucket, name, resources.Prefix)
if err != nil {
return mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err)
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName != "":
if delimitedName == resources.Delimiter {
@ -61,15 +61,15 @@ func (storage *Storage) filter(bucket, name string, file os.FileInfo, resources
switch true {
case delimitedName == "":
// Do not strip prefix object output
metadata, err = storage.GetObjectMetadata(bucket, name, "")
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err)
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName == file.Name():
case delimitedName == f.Name():
// Do not strip prefix object output
metadata, err = storage.GetObjectMetadata(bucket, name, "")
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err)
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName != "":
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
@ -78,15 +78,15 @@ func (storage *Storage) filter(bucket, name string, file os.FileInfo, resources
case resources.IsPrefixSet():
if strings.HasPrefix(name, resources.Prefix) {
// Do not strip prefix object output
metadata, err = storage.GetObjectMetadata(bucket, name, "")
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err)
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
}
case resources.IsDefault():
metadata, err = storage.GetObjectMetadata(bucket, name, "")
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err)
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
}

142
pkg/storage/file/file_object.go → pkg/drivers/file/file_object.go
View File

@ -28,136 +28,136 @@ import (
"encoding/gob"
"encoding/hex"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
)
/// Object Operations
// GetPartialObject - GET object from range
func (storage *Storage) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
func (file *fileDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
// validate bucket
if mstorage.IsValidBucket(bucket) == false {
return 0, mstorage.BucketNameInvalid{Bucket: bucket}
if drivers.IsValidBucket(bucket) == false {
return 0, drivers.BucketNameInvalid{Bucket: bucket}
}
// validate object
if mstorage.IsValidObject(object) == false {
return 0, mstorage.ObjectNameInvalid{Bucket: bucket, Object: object}
if drivers.IsValidObject(object) == false {
return 0, drivers.ObjectNameInvalid{Bucket: bucket, Object: object}
}
objectPath := path.Join(storage.root, bucket, object)
objectPath := path.Join(file.root, bucket, object)
filestat, err := os.Stat(objectPath)
switch err := err.(type) {
case nil:
{
if filestat.IsDir() {
return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
}
default:
{
if os.IsNotExist(err) {
return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
return 0, mstorage.EmbedError(bucket, object, err)
return 0, drivers.EmbedError(bucket, object, err)
}
}
file, err := os.Open(objectPath)
defer file.Close()
f, err := os.Open(objectPath)
defer f.Close()
if err != nil {
return 0, mstorage.EmbedError(bucket, object, err)
return 0, drivers.EmbedError(bucket, object, err)
}
_, err = file.Seek(start, os.SEEK_SET)
_, err = f.Seek(start, os.SEEK_SET)
if err != nil {
return 0, mstorage.EmbedError(bucket, object, err)
return 0, drivers.EmbedError(bucket, object, err)
}
count, err := io.CopyN(w, file, length)
count, err := io.CopyN(w, f, length)
if err != nil {
return count, mstorage.EmbedError(bucket, object, err)
return count, drivers.EmbedError(bucket, object, err)
}
return count, nil
}
// GetObject - GET object from key
func (storage *Storage) GetObject(w io.Writer, bucket string, object string) (int64, error) {
func (file *fileDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
// validate bucket
if mstorage.IsValidBucket(bucket) == false {
return 0, mstorage.BucketNameInvalid{Bucket: bucket}
if drivers.IsValidBucket(bucket) == false {
return 0, drivers.BucketNameInvalid{Bucket: bucket}
}
// validate object
if mstorage.IsValidObject(object) == false {
return 0, mstorage.ObjectNameInvalid{Bucket: bucket, Object: object}
if drivers.IsValidObject(object) == false {
return 0, drivers.ObjectNameInvalid{Bucket: bucket, Object: object}
}
objectPath := path.Join(storage.root, bucket, object)
objectPath := path.Join(file.root, bucket, object)
filestat, err := os.Stat(objectPath)
switch err := err.(type) {
case nil:
{
if filestat.IsDir() {
return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
}
default:
{
if os.IsNotExist(err) {
return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
return 0, mstorage.EmbedError(bucket, object, err)
return 0, drivers.EmbedError(bucket, object, err)
}
}
file, err := os.Open(objectPath)
defer file.Close()
f, err := os.Open(objectPath)
defer f.Close()
if err != nil {
return 0, mstorage.EmbedError(bucket, object, err)
return 0, drivers.EmbedError(bucket, object, err)
}
count, err := io.Copy(w, file)
count, err := io.Copy(w, f)
if err != nil {
return count, mstorage.EmbedError(bucket, object, err)
return count, drivers.EmbedError(bucket, object, err)
}
return count, nil
}
// GetObjectMetadata - HEAD object
func (storage *Storage) GetObjectMetadata(bucket, object, prefix string) (mstorage.ObjectMetadata, error) {
if mstorage.IsValidBucket(bucket) == false {
return mstorage.ObjectMetadata{}, mstorage.BucketNameInvalid{Bucket: bucket}
func (file *fileDriver) GetObjectMetadata(bucket, object, prefix string) (drivers.ObjectMetadata, error) {
if drivers.IsValidBucket(bucket) == false {
return drivers.ObjectMetadata{}, drivers.BucketNameInvalid{Bucket: bucket}
}
if mstorage.IsValidObject(object) == false {
return mstorage.ObjectMetadata{}, mstorage.ObjectNameInvalid{Bucket: bucket, Object: bucket}
if drivers.IsValidObject(object) == false {
return drivers.ObjectMetadata{}, drivers.ObjectNameInvalid{Bucket: bucket, Object: bucket}
}
// Do not use path.Join() since path.Join strips off any object names with '/', use them as is
// in a static manner so that we can send a proper 'ObjectNotFound' reply back upon os.Stat()
objectPath := storage.root + "/" + bucket + "/" + object
objectPath := file.root + "/" + bucket + "/" + object
stat, err := os.Stat(objectPath)
if os.IsNotExist(err) {
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
_, err = os.Stat(objectPath + "$metadata")
if os.IsNotExist(err) {
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
file, err := os.Open(objectPath + "$metadata")
defer file.Close()
f, err := os.Open(objectPath + "$metadata")
defer f.Close()
if err != nil {
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
return drivers.ObjectMetadata{}, drivers.EmbedError(bucket, object, err)
}
var deserializedMetadata Metadata
decoder := gob.NewDecoder(file)
var deserializedMetadata fileMetadata
decoder := gob.NewDecoder(f)
err = decoder.Decode(&deserializedMetadata)
if err != nil {
return mstorage.ObjectMetadata{}, mstorage.EmbedError(bucket, object, err)
return drivers.ObjectMetadata{}, drivers.EmbedError(bucket, object, err)
}
contentType := "application/octet-stream"
@ -171,7 +171,7 @@ func (storage *Storage) GetObjectMetadata(bucket, object, prefix string) (mstora
etag = hex.EncodeToString(deserializedMetadata.Md5sum)
}
trimmedObject := strings.TrimPrefix(object, prefix)
metadata := mstorage.ObjectMetadata{
metadata := drivers.ObjectMetadata{
Bucket: bucket,
Key: trimmedObject,
Created: stat.ModTime(),
@ -184,24 +184,24 @@ func (storage *Storage) GetObjectMetadata(bucket, object, prefix string) (mstora
}
// CreateObject - PUT object
func (storage *Storage) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
func (file *fileDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
// TODO Commits should stage then move instead of writing directly
storage.lock.Lock()
defer storage.lock.Unlock()
file.lock.Lock()
defer file.lock.Unlock()
// check bucket name valid
if mstorage.IsValidBucket(bucket) == false {
return mstorage.BucketNameInvalid{Bucket: bucket}
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketNameInvalid{Bucket: bucket}
}
// check bucket exists
if _, err := os.Stat(path.Join(storage.root, bucket)); os.IsNotExist(err) {
return mstorage.BucketNotFound{Bucket: bucket}
if _, err := os.Stat(path.Join(file.root, bucket)); os.IsNotExist(err) {
return drivers.BucketNotFound{Bucket: bucket}
}
// verify object path legal
if mstorage.IsValidObject(key) == false {
return mstorage.ObjectNameInvalid{Bucket: bucket, Object: key}
if drivers.IsValidObject(key) == false {
return drivers.ObjectNameInvalid{Bucket: bucket, Object: key}
}
// verify content type
@ -211,54 +211,54 @@ func (storage *Storage) CreateObject(bucket, key, contentType, md5sum string, da
contentType = strings.TrimSpace(contentType)
// get object path
objectPath := path.Join(storage.root, bucket, key)
objectPath := path.Join(file.root, bucket, key)
objectDir := path.Dir(objectPath)
if _, err := os.Stat(objectDir); os.IsNotExist(err) {
err = os.MkdirAll(objectDir, 0700)
if err != nil {
return mstorage.EmbedError(bucket, key, err)
return drivers.EmbedError(bucket, key, err)
}
}
// check if object exists
if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
return mstorage.ObjectExists{
return drivers.ObjectExists{
Bucket: bucket,
Object: key,
}
}
// write object
file, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
defer file.Close()
f, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
defer f.Close()
if err != nil {
return mstorage.EmbedError(bucket, key, err)
return drivers.EmbedError(bucket, key, err)
}
h := md5.New()
mw := io.MultiWriter(file, h)
mw := io.MultiWriter(f, h)
_, err = io.Copy(mw, data)
if err != nil {
return mstorage.EmbedError(bucket, key, err)
return drivers.EmbedError(bucket, key, err)
}
//
file, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
defer file.Close()
f, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
defer f.Close()
if err != nil {
return mstorage.EmbedError(bucket, key, err)
return drivers.EmbedError(bucket, key, err)
}
metadata := &Metadata{
metadata := &fileMetadata{
ContentType: contentType,
Md5sum: h.Sum(nil),
}
// serialize metadata to gob
encoder := gob.NewEncoder(file)
encoder := gob.NewEncoder(f)
err = encoder.Encode(metadata)
if err != nil {
return mstorage.EmbedError(bucket, key, err)
return drivers.EmbedError(bucket, key, err)
}
// Verify data received to be correct, Content-MD5 received
@ -266,10 +266,10 @@ func (storage *Storage) CreateObject(bucket, key, contentType, md5sum string, da
var data []byte
data, err = base64.StdEncoding.DecodeString(md5sum)
if err != nil {
return mstorage.InvalidDigest{Bucket: bucket, Key: key, Md5: md5sum}
return drivers.InvalidDigest{Bucket: bucket, Key: key, Md5: md5sum}
}
if !bytes.Equal(metadata.Md5sum, data) {
return mstorage.BadDigest{Bucket: bucket, Key: key, Md5: md5sum}
return drivers.BadDigest{Bucket: bucket, Key: key, Md5: md5sum}
}
}
return nil

111
pkg/drivers/file/file_policy.go Normal file
View File

@ -0,0 +1,111 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"path"
"encoding/json"
"github.com/minio-io/minio/pkg/drivers"
)
// GetBucketPolicy - GET bucket policy
func (file *fileDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
file.lock.Lock()
defer file.lock.Unlock()
var p drivers.BucketPolicy
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketPolicy{}, drivers.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(file.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err != nil {
return drivers.BucketPolicy{}, drivers.BucketNotFound{Bucket: bucket}
}
// get policy path
bucketPolicy := path.Join(file.root, bucket+"_policy.json")
filestat, err := os.Stat(bucketPolicy)
if os.IsNotExist(err) {
return drivers.BucketPolicy{}, drivers.BucketPolicyNotFound{Bucket: bucket}
}
if filestat.IsDir() {
return drivers.BucketPolicy{}, drivers.BackendCorrupted{Path: bucketPolicy}
}
f, err := os.OpenFile(bucketPolicy, os.O_RDONLY, 0666)
defer f.Close()
if err != nil {
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err)
}
encoder := json.NewDecoder(f)
err = encoder.Decode(&p)
if err != nil {
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err)
}
return p, nil
}
// CreateBucketPolicy - PUT bucket policy
func (file *fileDriver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error {
file.lock.Lock()
defer file.lock.Unlock()
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(file.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err != nil {
return drivers.BucketNotFound{
Bucket: bucket,
}
}
// get policy path
bucketPolicy := path.Join(file.root, bucket+"_policy.json")
filestat, ret := os.Stat(bucketPolicy)
if !os.IsNotExist(ret) {
if filestat.IsDir() {
return drivers.BackendCorrupted{Path: bucketPolicy}
}
}
f, err := os.OpenFile(bucketPolicy, os.O_WRONLY|os.O_CREATE, 0600)
defer f.Close()
if err != nil {
return drivers.EmbedError(bucket, "", err)
}
encoder := json.NewEncoder(f)
err = encoder.Encode(p)
if err != nil {
return drivers.EmbedError(bucket, "", err)
}
return nil
}

6
pkg/storage/file/file_test.go → pkg/drivers/file/file_test.go
View File

@ -21,7 +21,7 @@ import (
"os"
"testing"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
. "gopkg.in/check.v1"
)
@ -34,14 +34,14 @@ var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
var storageList []string
create := func() mstorage.Storage {
create := func() drivers.Driver {
path, err := ioutil.TempDir(os.TempDir(), "minio-file-")
c.Check(err, IsNil)
storageList = append(storageList, path)
_, _, store := Start(path)
return store
}
mstorage.APITestSuite(c, create)
drivers.APITestSuite(c, create)
removeRoots(c, storageList)
}

111
pkg/storage/memory/memory.go → pkg/drivers/memory/memory.go
View File

@ -1,5 +1,5 @@
/*
* Minio Object Storage, (C) 2015 Minio, Inc.
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -19,7 +19,6 @@ package memory
import (
"bufio"
"bytes"
"fmt"
"io"
"sort"
"strings"
@ -28,37 +27,40 @@ import (
"crypto/md5"
"encoding/hex"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
)
// Storage - local variables
type Storage struct {
// memoryDriver - local variables
type memoryDriver struct {
bucketdata map[string]storedBucket
objectdata map[string]storedObject
lock *sync.RWMutex
}
type storedBucket struct {
metadata mstorage.BucketMetadata
metadata drivers.BucketMetadata
// owner string // TODO
// id string // TODO
}
type storedObject struct {
metadata mstorage.ObjectMetadata
metadata drivers.ObjectMetadata
data []byte
}
// Start memory object server
func Start() (chan<- string, <-chan error, *Storage) {
func Start() (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
memory := new(memoryDriver)
memory.bucketdata = make(map[string]storedBucket)
memory.objectdata = make(map[string]storedObject)
memory.lock = new(sync.RWMutex)
go start(ctrlChannel, errorChannel)
return ctrlChannel, errorChannel, &Storage{
bucketdata: make(map[string]storedBucket),
objectdata: make(map[string]storedObject),
lock: new(sync.RWMutex),
}
return ctrlChannel, errorChannel, memory
}
func start(ctrlChannel <-chan string, errorChannel chan<- error) {
@ -66,43 +68,43 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error) {
}
// GetObject - GET object from memory buffer
func (storage *Storage) GetObject(w io.Writer, bucket string, object string) (int64, error) {
func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
// get object
key := object
if val, ok := storage.objectdata[key]; ok {
if val, ok := memory.objectdata[key]; ok {
objectBuffer := bytes.NewBuffer(val.data)
written, err := io.Copy(w, objectBuffer)
return written, err
}
return 0, mstorage.ObjectNotFound{Bucket: bucket, Object: object}
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
// GetPartialObject - GET object from memory buffer range
func (storage *Storage) GetPartialObject(w io.Writer, bucket, object string, start, end int64) (int64, error) {
return 0, mstorage.APINotImplemented{API: "GetPartialObject"}
func (memory memoryDriver) GetPartialObject(w io.Writer, bucket, object string, start, end int64) (int64, error) {
return 0, drivers.APINotImplemented{API: "GetPartialObject"}
}
// CreateBucketPolicy - Not implemented
func (storage *Storage) CreateBucketPolicy(bucket string, policy mstorage.BucketPolicy) error {
return mstorage.APINotImplemented{API: "PutBucketPolicy"}
func (memory memoryDriver) CreateBucketPolicy(bucket string, policy drivers.BucketPolicy) error {
return drivers.APINotImplemented{API: "PutBucketPolicy"}
}
// GetBucketPolicy - Not implemented
func (storage *Storage) GetBucketPolicy(bucket string) (mstorage.BucketPolicy, error) {
return mstorage.BucketPolicy{}, mstorage.APINotImplemented{API: "GetBucketPolicy"}
func (memory memoryDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
return drivers.BucketPolicy{}, drivers.APINotImplemented{API: "GetBucketPolicy"}
}
// CreateObject - PUT object to memory buffer
func (storage *Storage) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
storage.lock.Lock()
defer storage.lock.Unlock()
func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
memory.lock.Lock()
defer memory.lock.Unlock()
if _, ok := storage.bucketdata[bucket]; ok == false {
return mstorage.BucketNotFound{Bucket: bucket}
if _, ok := memory.bucketdata[bucket]; ok == false {
return drivers.BucketNotFound{Bucket: bucket}
}
if _, ok := storage.objectdata[key]; ok == true {
return mstorage.ObjectExists{Bucket: bucket, Object: key}
if _, ok := memory.objectdata[key]; ok == true {
return drivers.ObjectExists{Bucket: bucket, Object: key}
}
if contentType == "" {
@ -117,7 +119,7 @@ func (storage *Storage) CreateObject(bucket, key, contentType, md5sum string, da
size := bytesBuffer.Len()
md5SumBytes := md5.Sum(bytesBuffer.Bytes())
md5Sum := hex.EncodeToString(md5SumBytes[:])
newObject.metadata = mstorage.ObjectMetadata{
newObject.metadata = drivers.ObjectMetadata{
Bucket: bucket,
Key: key,
@ -128,27 +130,27 @@ func (storage *Storage) CreateObject(bucket, key, contentType, md5sum string, da
}
newObject.data = bytesBuffer.Bytes()
}
storage.objectdata[key] = newObject
memory.objectdata[key] = newObject
return nil
}
// CreateBucket - create bucket in memory
func (storage *Storage) CreateBucket(bucketName string) error {
storage.lock.Lock()
defer storage.lock.Unlock()
if !mstorage.IsValidBucket(bucketName) {
return mstorage.BucketNameInvalid{Bucket: bucketName}
func (memory memoryDriver) CreateBucket(bucketName string) error {
memory.lock.Lock()
defer memory.lock.Unlock()
if !drivers.IsValidBucket(bucketName) {
return drivers.BucketNameInvalid{Bucket: bucketName}
}
if _, ok := storage.bucketdata[bucketName]; ok == true {
return mstorage.BucketExists{Bucket: bucketName}
if _, ok := memory.bucketdata[bucketName]; ok == true {
return drivers.BucketExists{Bucket: bucketName}
}
var newBucket = storedBucket{}
newBucket.metadata = mstorage.BucketMetadata{}
newBucket.metadata = drivers.BucketMetadata{}
newBucket.metadata.Name = bucketName
newBucket.metadata.Created = time.Now()
storage.bucketdata[bucketName] = newBucket
memory.bucketdata[bucketName] = newBucket
return nil
}
@ -172,13 +174,13 @@ func appendUniq(slice []string, i string) []string {
}
// ListObjects - list objects from memory
func (storage *Storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) {
if _, ok := storage.bucketdata[bucket]; ok == false {
return []mstorage.ObjectMetadata{}, mstorage.BucketResourcesMetadata{IsTruncated: false}, mstorage.BucketNotFound{Bucket: bucket}
func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
if _, ok := memory.bucketdata[bucket]; ok == false {
return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket}
}
var results []mstorage.ObjectMetadata
var results []drivers.ObjectMetadata
var keys []string
for key := range storage.objectdata {
for key := range memory.objectdata {
switch true {
// Prefix absent, delimit object key based on delimiter
case resources.Delimiter != "" && resources.Prefix == "":
@ -193,7 +195,6 @@ func (storage *Storage) ListObjects(bucket string, resources mstorage.BucketReso
case resources.Delimiter != "" && resources.Prefix != "" && strings.HasPrefix(key, resources.Prefix):
trimmedName := strings.TrimPrefix(key, resources.Prefix)
delimitedName := delimiter(trimmedName, resources.Delimiter)
fmt.Println(trimmedName, delimitedName, key, resources.Prefix)
switch true {
case key == resources.Prefix:
keys = appendUniq(keys, key)
@ -219,9 +220,9 @@ func (storage *Storage) ListObjects(bucket string, resources mstorage.BucketReso
sort.Strings(keys)
for _, key := range keys {
if len(results) == resources.Maxkeys {
return results, mstorage.BucketResourcesMetadata{IsTruncated: true}, nil
return results, drivers.BucketResourcesMetadata{IsTruncated: true}, nil
}
object := storage.objectdata[key]
object := memory.objectdata[key]
if bucket == object.metadata.Bucket {
results = append(results, object.metadata)
}
@ -230,7 +231,7 @@ func (storage *Storage) ListObjects(bucket string, resources mstorage.BucketReso
}
// ByBucketName is a type for sorting bucket metadata by bucket name
type ByBucketName []mstorage.BucketMetadata
type ByBucketName []drivers.BucketMetadata
// Len of bucket name
func (b ByBucketName) Len() int { return len(b) }
@ -242,9 +243,9 @@ func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
// ListBuckets - List buckets from memory
func (storage *Storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
var results []mstorage.BucketMetadata
for _, bucket := range storage.bucketdata {
func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
var results []drivers.BucketMetadata
for _, bucket := range memory.bucketdata {
results = append(results, bucket.metadata)
}
sort.Sort(ByBucketName(results))
@ -252,9 +253,9 @@ func (storage *Storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
}
// GetObjectMetadata - get object metadata from memory
func (storage *Storage) GetObjectMetadata(bucket, key, prefix string) (mstorage.ObjectMetadata, error) {
if object, ok := storage.objectdata[key]; ok == true {
func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (drivers.ObjectMetadata, error) {
if object, ok := memory.objectdata[key]; ok == true {
return object.metadata, nil
}
return mstorage.ObjectMetadata{}, mstorage.ObjectNotFound{Bucket: bucket, Object: key}
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key}
}

7
pkg/storage/memory/memory_test.go → pkg/drivers/memory/memory_test.go
View File

@ -19,7 +19,7 @@ package memory
import (
"testing"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/drivers"
. "gopkg.in/check.v1"
)
@ -31,10 +31,9 @@ type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
create := func() mstorage.Storage {
create := func() drivers.Driver {
_, _, store := Start()
return store
}
mstorage.APITestSuite(c, create)
drivers.APITestSuite(c, create)
}

2
pkg/encoding/erasure/ctypes.go
View File

@ -24,7 +24,7 @@ import (
)
// Integer to Int conversion
func int2cInt(src_err_list []int) *C.int32_t {
func int2CInt(src_err_list []int) *C.int32_t {
var sizeErrInt = int(unsafe.Sizeof(src_err_list[0]))
switch sizeInt {
case sizeErrInt:

52
pkg/encoding/erasure/erasure_decode.go
View File

@ -32,47 +32,47 @@ import (
//
// Decoded data is exactly similar in length and content as the original data.
func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
var decode_matrix *C.uint8_t
var decode_tbls *C.uint8_t
var decode_index *C.uint32_t
var decodeMatrix *C.uint8_t
var decodeTbls *C.uint8_t
var decodeIndex *C.uint32_t
var source, target **C.uint8_t
k := e.params.K
m := e.params.M
k := int(e.params.K)
m := int(e.params.M)
n := k + m
if len(chunks) != int(n) {
if len(chunks) != n {
return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
}
chunk_size := GetEncodedChunkLen(length, uint8(k))
chunkLen := GetEncodedBlockLen(length, uint8(k))
error_index := make([]int, n+1)
var err_count int = 0
errorIndex := make([]int, n+1)
var errCount int = 0
for i := range chunks {
// Check of chunks are really null
if chunks[i] == nil || len(chunks[i]) == 0 {
error_index[err_count] = i
err_count++
errorIndex[errCount] = i
errCount++
}
}
error_index[err_count] = -1
err_count++
errorIndex[errCount] = -1
errCount++
// Too many missing chunks, cannot be more than parity `m`
if err_count-1 > int(n-k) {
if errCount-1 > int(n-k) {
return nil, errors.New("too many erasures requested, can't decode")
}
error_index_ptr := int2cInt(error_index[:err_count])
errorIndex_ptr := int2CInt(errorIndex[:errCount])
for i := range chunks {
if chunks[i] == nil || len(chunks[i]) == 0 {
chunks[i] = make([]byte, chunk_size)
chunks[i] = make([]byte, chunkLen)
}
}
C.minio_init_decoder(error_index_ptr, C.int(k), C.int(n), C.int(err_count-1),
e.encode_matrix, &decode_matrix, &decode_tbls, &decode_index)
C.minio_init_decoder(errorIndex_ptr, C.int(k), C.int(n), C.int(errCount-1),
e.encodeMatrix, &decodeMatrix, &decodeTbls, &decodeIndex)
pointers := make([]*byte, n)
for i := range chunks {
@ -81,24 +81,24 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
data := (**C.uint8_t)(unsafe.Pointer(&pointers[0]))
ret := C.minio_get_source_target(C.int(err_count-1), C.int(k), C.int(m), error_index_ptr,
decode_index, data, &source, &target)
ret := C.minio_get_source_target(C.int(errCount-1), C.int(k), C.int(m), errorIndex_ptr,
decodeIndex, data, &source, &target)
if int(ret) == -1 {
return nil, errors.New("Decoding source target failed")
}
C.ec_encode_data(C.int(chunk_size), C.int(k), C.int(err_count-1), decode_tbls,
C.ec_encode_data(C.int(chunkLen), C.int(k), C.int(errCount-1), decodeTbls,
source, target)
recovered_output := make([]byte, 0, chunk_size*int(k))
recoveredOutput := make([]byte, 0, chunkLen*int(k))
for i := 0; i < int(k); i++ {
recovered_output = append(recovered_output, chunks[i]...)
recoveredOutput = append(recoveredOutput, chunks[i]...)
}
// TODO cache this if necessary
e.decode_matrix = decode_matrix
e.decode_tbls = decode_tbls
e.decodeMatrix = decodeMatrix
e.decodeTbls = decodeTbls
return recovered_output[:length], nil
return recoveredOutput[:length], nil
}

128
pkg/encoding/erasure/erasure_encode.go
View File

@ -26,11 +26,12 @@ import (
"unsafe"
)
type Technique int
type Technique uint8
const (
Vandermonde Technique = iota
Cauchy
None
)
const (
@ -52,10 +53,10 @@ type EncoderParams struct {
// Encoder is an object used to encode and decode data.
type Encoder struct {
params *EncoderParams
encode_matrix,
encode_tbls,
decode_matrix,
decode_tbls *C.uint8_t
encodeMatrix,
encodeTbls,
decodeMatrix,
decodeTbls *C.uint8_t
}
// ParseEncoderParams creates an EncoderParams object.
@ -94,30 +95,30 @@ func ParseEncoderParams(k, m uint8, technique Technique) (*EncoderParams, error)
// NewEncoder creates an encoder object with a given set of parameters.
func NewEncoder(ep *EncoderParams) *Encoder {
var encode_matrix *C.uint8_t
var encode_tbls *C.uint8_t
var k = C.int(ep.K)
var m = C.int(ep.M)
k := C.int(ep.K)
m := C.int(ep.M)
var encodeMatrix *C.uint8_t
var encodeTbls *C.uint8_t
C.minio_init_encoder(C.int(ep.Technique), k, m, &encode_matrix,
&encode_tbls)
C.minio_init_encoder(C.int(ep.Technique), k, m, &encodeMatrix,
&encodeTbls)
return &Encoder{
params: ep,
encode_matrix: encode_matrix,
encode_tbls: encode_tbls,
decode_matrix: nil,
decode_tbls: nil,
params: ep,
encodeMatrix: encodeMatrix,
encodeTbls: encodeTbls,
decodeMatrix: nil,
decodeTbls: nil,
}
}
func GetEncodedLen(inputLen int, k, m uint8) (outputLen int) {
outputLen = GetEncodedChunkLen(inputLen, k) * int(k+m)
func GetEncodedBlocksLen(inputLen int, k, m uint8) (outputLen int) {
outputLen = GetEncodedBlockLen(inputLen, k) * int(k+m)
return outputLen
}
func GetEncodedChunkLen(inputLen int, k uint8) (outputChunkLen int) {
func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) {
alignment := int(k) * SIMDAlign
remainder := inputLen % alignment
@ -125,55 +126,66 @@ func GetEncodedChunkLen(inputLen int, k uint8) (outputChunkLen int) {
if remainder != 0 {
paddedInputLen = inputLen + (alignment - remainder)
}
outputChunkLen = paddedInputLen / int(k)
return outputChunkLen
encodedOutputLen = paddedInputLen / int(k)
return encodedOutputLen
}
// Encode encodes a block of data. The input is the original data. The output
// is a 2 tuple containing (k + m) chunks of erasure encoded data and the
// length of the original object.
func (e *Encoder) Encode(input []byte) ([][]byte, error) {
inputLen := len(input)
k := C.int(e.params.K)
m := C.int(e.params.M)
n := k + m
// Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
// Output is [k+m][]blocks of data and parity slices.
func (e *Encoder) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
k := int(e.params.K) // "k" data blocks
m := int(e.params.M) // "m" parity blocks
n := k + m // "n" total encoded blocks
chunkLen := GetEncodedChunkLen(inputLen, e.params.K)
encodedDataLen := chunkLen * int(k)
paddedDataLen := int(encodedDataLen) - inputLen
// Length of a single encoded chunk.
// Total number of encoded chunks = "k" data + "m" parity blocks
encodedBlockLen := GetEncodedBlockLen(len(inputData), uint8(k))
if paddedDataLen > 0 {
s := make([]byte, paddedDataLen)
// Length of total number of "k" data chunks
encodedDataBlocksLen := encodedBlockLen * k
// Length of extra padding required for the data blocks.
encodedDataBlocksPadLen := encodedDataBlocksLen - len(inputData)
// Extend inputData buffer to accommodate coded data blocks if necesssary
if encodedDataBlocksPadLen > 0 {
padding := make([]byte, encodedDataBlocksPadLen)
// Expand with new padded blocks to the byte array
input = append(input, s...)
inputData = append(inputData, padding...)
}
encodedParityLen := chunkLen * int(e.params.M)
c := make([]byte, encodedParityLen)
input = append(input, c...)
// encodedOutLen := encodedDataLen + encodedParityLen
// Allocate chunks
chunks := make([][]byte, k+m)
pointers := make([]*byte, k+m)
var i int
// Add data blocks to chunks
for i = 0; i < int(k); i++ {
chunks[i] = input[i*chunkLen : (i+1)*chunkLen]
pointers[i] = &chunks[i][0]
// Extend inputData buffer to accommodate coded parity blocks
if true { // create a temporary scope to trigger garbage collect
encodedParityBlocksLen := encodedBlockLen * m
parityBlocks := make([]byte, encodedParityBlocksLen)
inputData = append(inputData, parityBlocks...)
}
for i = int(k); i < int(n); i++ {
chunks[i] = make([]byte, chunkLen)
pointers[i] = &chunks[i][0]
// Allocate memory to the "encoded blocks" return buffer
encodedBlocks = make([][]byte, n) // Return buffer
// Nessary to bridge Go to the C world. C requires 2D arry of pointers to
// byte array. "encodedBlocks" is a 2D slice.
pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.
// Copy data block slices to encoded block buffer
for i := 0; i < k; i++ {
encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
pointersToEncodedBlock[i] = &encodedBlocks[i][0]
}
data := (**C.uint8_t)(unsafe.Pointer(&pointers[:k][0]))
coding := (**C.uint8_t)(unsafe.Pointer(&pointers[k:][0]))
// Copy erasure block slices to encoded block buffer
for i := k; i < n; i++ {
encodedBlocks[i] = make([]byte, encodedBlockLen)
pointersToEncodedBlock[i] = &encodedBlocks[i][0]
}
C.ec_encode_data(C.int(chunkLen), k, m, e.encode_tbls, data,
coding)
return chunks, nil
// Erasure code the data into K data blocks and M parity
// blocks. Only the parity blocks are filled. Data blocks remain
// intact.
C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
(**C.uint8_t)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
(**C.uint8_t)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks
return encodedBlocks, nil
}

58
pkg/server/server.go
View File

@ -24,11 +24,11 @@ import (
"github.com/minio-io/minio/pkg/api"
"github.com/minio-io/minio/pkg/api/web"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/drivers/donut"
"github.com/minio-io/minio/pkg/drivers/file"
"github.com/minio-io/minio/pkg/drivers/memory"
"github.com/minio-io/minio/pkg/server/httpserver"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/storage/donutstorage"
"github.com/minio-io/minio/pkg/storage/file"
"github.com/minio-io/minio/pkg/storage/memory"
)
// Config - http server parameters
@ -41,9 +41,9 @@ type Config struct {
APIType interface{}
}
// MinioAPI - storage type donut, file, memory
// MinioAPI - driver type donut, file, memory
type MinioAPI struct {
StorageType StorageType
DriverType DriverType
}
// Web - web related
@ -51,12 +51,12 @@ type Web struct {
Websocket bool // TODO
}
// StorageType - different storage types supported by minio
type StorageType int
// DriverType - different driver types supported by minio
type DriverType int
// Storage types
// Driver types
const (
Memory = iota
Memory DriverType = iota
File
Donut
)
@ -71,7 +71,7 @@ func getHTTPChannels(configs []Config) (ctrlChans []chan<- string, statusChans [
case MinioAPI:
{
// configure web server
var storage mstorage.Storage
var driver drivers.Driver
var httpConfig = httpserver.Config{}
httpConfig.Address = config.Address
httpConfig.Websocket = false
@ -84,9 +84,9 @@ func getHTTPChannels(configs []Config) (ctrlChans []chan<- string, statusChans [
httpConfig.KeyFile = config.KeyFile
}
ctrlChans, statusChans, storage = getStorageChannels(k.StorageType)
// start minio api in a web server, pass storage driver into it
ctrlChan, statusChan, _ = httpserver.Start(api.HTTPHandler(config.Domain, storage), httpConfig)
ctrlChans, statusChans, driver = getDriverChannels(k.DriverType)
// start minio api in a web server, pass driver driver into it
ctrlChan, statusChan, _ = httpserver.Start(api.HTTPHandler(config.Domain, driver), httpConfig)
ctrlChans = append(ctrlChans, ctrlChan)
statusChans = append(statusChans, statusChan)
@ -113,50 +113,50 @@ func getHTTPChannels(configs []Config) (ctrlChans []chan<- string, statusChans [
return
}
func getStorageChannels(storageType StorageType) (ctrlChans []chan<- string, statusChans []<-chan error, storage mstorage.Storage) {
func getDriverChannels(driverType DriverType) (ctrlChans []chan<- string, statusChans []<-chan error, driver drivers.Driver) {
// a pair of control channels, we use these primarily to add to the lists above
var ctrlChan chan<- string
var statusChan <-chan error
// instantiate storage
// instantiate driver
// preconditions:
// - storage type specified
// - any configuration for storage is populated
// - driver type specified
// - any configuration for driver is populated
// postconditions:
// - storage driver is initialized
// - ctrlChans has channel to communicate to storage
// - statusChans has channel for messages coming from storage
// - driver driver is initialized
// - ctrlChans has channel to communicate to driver
// - statusChans has channel for messages coming from driver
switch {
case storageType == Memory:
case driverType == Memory:
{
ctrlChan, statusChan, storage = memory.Start()
ctrlChan, statusChan, driver = memory.Start()
ctrlChans = append(ctrlChans, ctrlChan)
statusChans = append(statusChans, statusChan)
}
case storageType == File:
case driverType == File:
{
u, err := user.Current()
if err != nil {
return nil, nil, nil
}
root := path.Join(u.HomeDir, "minio-storage", "file")
ctrlChan, statusChan, storage = file.Start(root)
ctrlChan, statusChan, driver = file.Start(root)
ctrlChans = append(ctrlChans, ctrlChan)
statusChans = append(statusChans, statusChan)
}
case storageType == Donut:
case driverType == Donut:
{
u, err := user.Current()
if err != nil {
return nil, nil, nil
}
root := path.Join(u.HomeDir, "minio-storage", "donut")
ctrlChan, statusChan, storage = donutstorage.Start(root)
root := path.Join(u.HomeDir, "minio-driver", "donut")
ctrlChan, statusChan, driver = donut.Start(root)
ctrlChans = append(ctrlChans, ctrlChan)
statusChans = append(statusChans, statusChan)
}
default: // should never happen
log.Fatal("No storage driver found")
log.Fatal("No driver found")
}
return
}

10
pkg/storage/donut/bucketdriver.go → pkg/storage/donut/bucket.go
View File

@ -6,11 +6,13 @@ import (
"strings"
)
type bucketDriver struct {
nodes []string
type donutBucket struct {
nodes []string
objects map[string][]byte
}
func (b bucketDriver) GetNodes() ([]string, error) {
// GetNodes - get list of associated nodes for a given bucket
func (b donutBucket) GetNodes() ([]string, error) {
var nodes []string
for _, node := range b.nodes {
nodes = append(nodes, node)
@ -18,7 +20,7 @@ func (b bucketDriver) GetNodes() ([]string, error) {
return nodes, nil
}
func (b bucketDriver) AddNode(nodeID, bucketID string) error {
func (b donutBucket) AddNode(nodeID, bucketID string) error {
tokens := strings.Split(bucketID, ":")
if len(tokens) != 3 {
return errors.New("Bucket ID malformed: " + bucketID)

202
pkg/storage/donut/donut.go
View File

@ -1,51 +1,181 @@
package donut
import (
"errors"
"io"
"sort"
"strconv"
"strings"
)
// INTERFACES
// Donut interface
type Donut interface {
CreateBucket(bucket string) error
GetObject(bucket, object string) (io.ReadCloser, error)
GetObjectMetadata(bucket, object string) (map[string]string, error)
GetObjectWriter(bucket, object string) (ObjectWriter, error)
ListBuckets() ([]string, error)
ListObjects(bucket string) ([]string, error)
type donut struct {
buckets map[string]Bucket
nodes map[string]Node
}
// Bucket interface
type Bucket interface {
GetNodes() ([]string, error)
AddNode(nodeID, bucketID string) error
// NewDonut - instantiate new donut driver
func NewDonut(root string) (Donut, error) {
nodes := make(map[string]Node)
nodes["localhost"] = &localDirectoryNode{root: root}
driver := &donut{
buckets: make(map[string]Bucket),
nodes: nodes,
}
for nodeID, node := range nodes {
bucketIDs, err := node.GetBuckets()
if err != nil {
return nil, err
}
for _, bucketID := range bucketIDs {
tokens := strings.Split(bucketID, ":")
if _, ok := driver.buckets[tokens[0]]; ok {
// found bucket, skip creating
} else {
bucket := donutBucket{
nodes: make([]string, 16),
}
// TODO catch errors
driver.buckets[tokens[0]] = bucket
}
if err = driver.buckets[tokens[0]].AddNode(nodeID, bucketID); err != nil {
return nil, err
}
}
}
return driver, nil
}
// Node interface
type Node interface {
CreateBucket(bucket string) error
GetBuckets() ([]string, error)
GetDonutMetadata(bucket, object string) (map[string]string, error)
GetMetadata(bucket, object string) (map[string]string, error)
GetReader(bucket, object string) (io.ReadCloser, error)
GetWriter(bucket, object string) (Writer, error)
ListObjects(bucket string) ([]string, error)
// CreateBucket - create a new bucket
func (d donut) CreateBucket(bucketName string) error {
if _, ok := d.buckets[bucketName]; ok == false {
bucketName = strings.TrimSpace(bucketName)
if bucketName == "" {
return errors.New("Cannot create bucket with no name")
}
// assign nodes
// TODO assign other nodes
nodes := make([]string, 16)
for i := 0; i < 16; i++ {
nodes[i] = "localhost"
if node, ok := d.nodes["localhost"]; ok {
node.CreateBucket(bucketName + ":0:" + strconv.Itoa(i))
}
}
bucket := donutBucket{
nodes: nodes,
}
d.buckets[bucketName] = bucket
return nil
}
return errors.New("Bucket exists")
}
// ObjectWriter interface
type ObjectWriter interface {
Close() error
CloseWithError(error) error
GetMetadata() (map[string]string, error)
SetMetadata(map[string]string) error
Write([]byte) (int, error)
// ListBuckets - list all buckets
func (d donut) ListBuckets() ([]string, error) {
var buckets []string
for bucket := range d.buckets {
buckets = append(buckets, bucket)
}
sort.Strings(buckets)
return buckets, nil
}
// Writer interface
type Writer interface {
ObjectWriter
GetDonutMetadata() (map[string]string, error)
SetDonutMetadata(map[string]string) error
// GetObjectWriter - get a new writer interface for a new object
func (d donut) GetObjectWriter(bucketName, objectName string) (ObjectWriter, error) {
if bucket, ok := d.buckets[bucketName]; ok == true {
writers := make([]Writer, 16)
nodes, err := bucket.GetNodes()
if err != nil {
return nil, err
}
for i, nodeID := range nodes {
if node, ok := d.nodes[nodeID]; ok == true {
writer, err := node.GetWriter(bucketName+":0:"+strconv.Itoa(i), objectName)
if err != nil {
for _, writerToClose := range writers {
if writerToClose != nil {
writerToClose.CloseWithError(err)
}
}
return nil, err
}
writers[i] = writer
}
}
return newErasureWriter(writers), nil
}
return nil, errors.New("Bucket not found")
}
// GetObjectReader - get a new reader interface for a new object
func (d donut) GetObjectReader(bucketName, objectName string) (io.ReadCloser, error) {
r, w := io.Pipe()
if bucket, ok := d.buckets[bucketName]; ok == true {
readers := make([]io.ReadCloser, 16)
nodes, err := bucket.GetNodes()
if err != nil {
return nil, err
}
var metadata map[string]string
for i, nodeID := range nodes {
if node, ok := d.nodes[nodeID]; ok == true {
bucketID := bucketName + ":0:" + strconv.Itoa(i)
reader, err := node.GetReader(bucketID, objectName)
if err != nil {
return nil, err
}
readers[i] = reader
if metadata == nil {
metadata, err = node.GetDonutMetadata(bucketID, objectName)
if err != nil {
return nil, err
}
}
}
}
go erasureReader(readers, metadata, w)
return r, nil
}
return nil, errors.New("Bucket not found")
}
// GetObjectMetadata returns metadata for a given object in a bucket
func (d donut) GetObjectMetadata(bucketName, object string) (map[string]string, error) {
if bucket, ok := d.buckets[bucketName]; ok {
nodes, err := bucket.GetNodes()
if err != nil {
return nil, err
}
if node, ok := d.nodes[nodes[0]]; ok {
bucketID := bucketName + ":0:0"
metadata, err := node.GetMetadata(bucketID, object)
if err != nil {
return nil, err
}
donutMetadata, err := node.GetDonutMetadata(bucketID, object)
if err != nil {
return nil, err
}
metadata["sys.created"] = donutMetadata["created"]
metadata["sys.md5"] = donutMetadata["md5"]
metadata["sys.size"] = donutMetadata["size"]
return metadata, nil
}
return nil, errors.New("Cannot connect to node: " + nodes[0])
}
return nil, errors.New("Bucket not found")
}
// ListObjects - list all the available objects in a bucket
func (d donut) ListObjects(bucketName string) ([]string, error) {
if bucket, ok := d.buckets[bucketName]; ok {
nodes, err := bucket.GetNodes()
if err != nil {
return nil, err
}
if node, ok := d.nodes[nodes[0]]; ok {
return node.ListObjects(bucketName + ":0:0")
}
}
return nil, errors.New("Bucket not found")
}

32
pkg/storage/donut/donutdriver_test.go → pkg/storage/donut/donut_test.go
View File

@ -1,14 +1,14 @@
package donut
import (
"testing"
"bytes"
. "gopkg.in/check.v1"
"io"
"io/ioutil"
"os"
"testing"
"time"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) { TestingT(t) }
@ -21,7 +21,7 @@ func (s *MySuite) TestEmptyBucket(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
// check buckets are empty
@ -34,7 +34,7 @@ func (s *MySuite) TestBucketWithoutNameFails(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
// fail to create new bucket without a name
err = donut.CreateBucket("")
@ -48,7 +48,7 @@ func (s *MySuite) TestCreateBucketAndList(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
// create bucket
err = donut.CreateBucket("foo")
@ -64,7 +64,7 @@ func (s *MySuite) TestCreateBucketWithSameNameFails(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
err = donut.CreateBucket("foo")
c.Assert(err, IsNil)
@ -77,7 +77,7 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
// add a second bucket
err = donut.CreateBucket("foo")
@ -102,7 +102,7 @@ func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
writer, err := donut.GetObjectWriter("foo", "obj")
@ -114,7 +114,7 @@ func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
writer, err := donut.GetObjectWriter("foo", "")
@ -130,7 +130,7 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
err = donut.CreateBucket("foo")
@ -161,7 +161,7 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
c.Assert(err, IsNil)
reader, err := donut.GetObject("foo", "obj")
reader, err := donut.GetObjectReader("foo", "obj")
c.Assert(err, IsNil)
var actualData bytes.Buffer
@ -183,7 +183,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
c.Assert(donut.CreateBucket("foo"), IsNil)
@ -199,7 +199,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
// c.Skip("not complete")
reader, err := donut.GetObject("foo", "obj1")
reader, err := donut.GetObjectReader("foo", "obj1")
c.Assert(err, IsNil)
var readerBuffer1 bytes.Buffer
_, err = io.Copy(&readerBuffer1, reader)
@ -207,7 +207,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
// c.Skip("Not Implemented")
c.Assert(readerBuffer1.Bytes(), DeepEquals, []byte("one"))
reader, err = donut.GetObject("foo", "obj2")
reader, err = donut.GetObjectReader("foo", "obj2")
c.Assert(err, IsNil)
var readerBuffer2 bytes.Buffer
_, err = io.Copy(&readerBuffer2, reader)
@ -224,7 +224,7 @@ func (s *MySuite) TestSysPrefixShouldFail(c *C) {
root, err := ioutil.TempDir(os.TempDir(), "donut-")
c.Assert(err, IsNil)
defer os.RemoveAll(root)
donut, err := NewDonutDriver(root)
donut, err := NewDonut(root)
c.Assert(err, IsNil)
c.Assert(donut.CreateBucket("foo"), IsNil)

176
pkg/storage/donut/donutdriver.go
View File

@ -1,176 +0,0 @@
package donut
import (
"errors"
"io"
"sort"
"strconv"
"strings"
)
type donutDriver struct {
buckets map[string]Bucket
nodes map[string]Node
}
// NewDonutDriver - instantiate new donut driver
func NewDonutDriver(root string) (Donut, error) {
nodes := make(map[string]Node)
nodes["localhost"] = &localDirectoryNode{root: root}
driver := &donutDriver{
buckets: make(map[string]Bucket),
nodes: nodes,
}
for nodeID, node := range nodes {
bucketIDs, err := node.GetBuckets()
if err != nil {
return nil, err
}
for _, bucketID := range bucketIDs {
tokens := strings.Split(bucketID, ":")
if _, ok := driver.buckets[tokens[0]]; ok {
// found bucket, skip creating
} else {
bucket := bucketDriver{
nodes: make([]string, 16),
}
// TODO catch errors
driver.buckets[tokens[0]] = bucket
}
if err = driver.buckets[tokens[0]].AddNode(nodeID, bucketID); err != nil {
return nil, err
}
}
}
return driver, nil
}
func (driver donutDriver) CreateBucket(bucketName string) error {
if _, ok := driver.buckets[bucketName]; ok == false {
bucketName = strings.TrimSpace(bucketName)
if bucketName == "" {
return errors.New("Cannot create bucket with no name")
}
// assign nodes
// TODO assign other nodes
nodes := make([]string, 16)
for i := 0; i < 16; i++ {
nodes[i] = "localhost"
if node, ok := driver.nodes["localhost"]; ok {
node.CreateBucket(bucketName + ":0:" + strconv.Itoa(i))
}
}
bucket := bucketDriver{
nodes: nodes,
}
driver.buckets[bucketName] = bucket
return nil
}
return errors.New("Bucket exists")
}
func (driver donutDriver) ListBuckets() ([]string, error) {
var buckets []string
for bucket := range driver.buckets {
buckets = append(buckets, bucket)
}
sort.Strings(buckets)
return buckets, nil
}
func (driver donutDriver) GetObjectWriter(bucketName, objectName string) (ObjectWriter, error) {
if bucket, ok := driver.buckets[bucketName]; ok == true {
writers := make([]Writer, 16)
nodes, err := bucket.GetNodes()
if err != nil {
return nil, err
}
for i, nodeID := range nodes {
if node, ok := driver.nodes[nodeID]; ok == true {
writer, err := node.GetWriter(bucketName+":0:"+strconv.Itoa(i), objectName)
if err != nil {
for _, writerToClose := range writers {
if writerToClose != nil {
writerToClose.CloseWithError(err)
}
}
return nil, err
}
writers[i] = writer
}
}
return newErasureWriter(writers), nil
}
return nil, errors.New("Bucket not found")
}
func (driver donutDriver) GetObject(bucketName, objectName string) (io.ReadCloser, error) {
r, w := io.Pipe()
if bucket, ok := driver.buckets[bucketName]; ok == true {
readers := make([]io.ReadCloser, 16)
nodes, err := bucket.GetNodes()
if err != nil {
return nil, err
}
var metadata map[string]string
for i, nodeID := range nodes {
if node, ok := driver.nodes[nodeID]; ok == true {
bucketID := bucketName + ":0:" + strconv.Itoa(i)
reader, err := node.GetReader(bucketID, objectName)
if err != nil {
return nil, err
}
readers[i] = reader
if metadata == nil {
metadata, err = node.GetDonutMetadata(bucketID, objectName)
if err != nil {
return nil, err
}
}
}
}
go erasureReader(readers, metadata, w)
return r, nil
}
return nil, errors.New("Bucket not found")
}
// GetObjectMetadata returns metadata for a given object in a bucket
func (driver donutDriver) GetObjectMetadata(bucketName, object string) (map[string]string, error) {
if bucket, ok := driver.buckets[bucketName]; ok {
nodes, err := bucket.GetNodes()
if err != nil {
return nil, err
}
if node, ok := driver.nodes[nodes[0]]; ok {
bucketID := bucketName + ":0:0"
metadata, err := node.GetMetadata(bucketID, object)
if err != nil {
return nil, err
}
donutMetadata, err := node.GetDonutMetadata(bucketID, object)
if err != nil {
return nil, err
}
metadata["sys.created"] = donutMetadata["created"]
metadata["sys.md5"] = donutMetadata["md5"]
metadata["sys.size"] = donutMetadata["size"]
return metadata, nil
}
return nil, errors.New("Cannot connect to node: " + nodes[0])
}
return nil, errors.New("Bucket not found")
}
func (driver donutDriver) ListObjects(bucketName string) ([]string, error) {
if bucket, ok := driver.buckets[bucketName]; ok {
nodes, err := bucket.GetNodes()
if err != nil {
return nil, err
}
if node, ok := driver.nodes[nodes[0]]; ok {
return node.ListObjects(bucketName + ":0:0")
}
}
return nil, errors.New("Bucket not found")
}

81
pkg/storage/donut/erasure.go
View File

@ -2,29 +2,70 @@ package donut
import (
"bytes"
"errors"
"io"
"strconv"
"strings"
"time"
"crypto/md5"
"encoding/hex"
"errors"
"github.com/minio-io/minio/pkg/encoding/erasure"
"github.com/minio-io/minio/pkg/utils/split"
"strings"
)
// getErasureTechnique - convert technique string into Technique type
func getErasureTechnique(technique string) (erasure.Technique, error) {
switch true {
case technique == "Cauchy":
return erasure.Cauchy, nil
case technique == "Vandermonde":
return erasure.Cauchy, nil
default:
return erasure.None, errors.New("Invalid erasure technique")
}
}
// erasureReader - returns aligned streaming reads over a PipeWriter
func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, writer *io.PipeWriter) {
// TODO handle errors
totalChunks, _ := strconv.Atoi(donutMetadata["chunkCount"])
totalLeft, _ := strconv.Atoi(donutMetadata["size"])
blockSize, _ := strconv.Atoi(donutMetadata["blockSize"])
k, _ := strconv.Atoi(donutMetadata["erasureK"])
m, _ := strconv.Atoi(donutMetadata["erasureM"])
expectedMd5sum, _ := hex.DecodeString(donutMetadata["md5"])
totalChunks, err := strconv.Atoi(donutMetadata["chunkCount"])
if err != nil {
writer.CloseWithError(err)
return
}
totalLeft, err := strconv.Atoi(donutMetadata["size"])
if err != nil {
writer.CloseWithError(err)
return
}
blockSize, err := strconv.Atoi(donutMetadata["blockSize"])
if err != nil {
writer.CloseWithError(err)
return
}
k, err := strconv.Atoi(donutMetadata["erasureK"])
if err != nil {
writer.CloseWithError(err)
return
}
m, err := strconv.Atoi(donutMetadata["erasureM"])
if err != nil {
writer.CloseWithError(err)
return
}
expectedMd5sum, err := hex.DecodeString(donutMetadata["md5"])
if err != nil {
writer.CloseWithError(err)
return
}
technique, err := getErasureTechnique(donutMetadata["erasureTechnique"])
if err != nil {
writer.CloseWithError(err)
return
}
summer := md5.New()
// TODO select technique properly
params, _ := erasure.ParseEncoderParams(uint8(k), uint8(m), erasure.Cauchy)
params, _ := erasure.ParseEncoderParams(uint8(k), uint8(m), technique)
encoder := erasure.NewEncoder(params)
for _, reader := range readers {
defer reader.Close()
@ -34,13 +75,17 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
if blockSize < totalLeft {
curBlockSize = blockSize
}
curChunkSize := erasure.GetEncodedChunkLen(curBlockSize, uint8(k))
curChunkSize := erasure.GetEncodedBlockLen(curBlockSize, uint8(k))
encodedBytes := make([][]byte, 16)
for i, reader := range readers {
var bytesBuffer bytes.Buffer
// TODO watch for errors
io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
_, err := io.CopyN(&bytesBuffer, reader, int64(curChunkSize))
if err != nil {
writer.CloseWithError(err)
return
}
encodedBytes[i] = bytesBuffer.Bytes()
}
decodedData, err := encoder.Decode(encodedBytes, curBlockSize)
@ -49,7 +94,11 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
return
}
summer.Write(decodedData)
io.Copy(writer, bytes.NewBuffer(decodedData))
_, err = io.Copy(writer, bytes.NewBuffer(decodedData))
if err != nil {
writer.CloseWithError(err)
return
}
totalLeft = totalLeft - blockSize
}
actualMd5sum := summer.Sum(nil)
@ -58,6 +107,7 @@ func erasureReader(readers []io.ReadCloser, donutMetadata map[string]string, wri
return
}
writer.Close()
return
}
// erasure writer
@ -70,6 +120,7 @@ type erasureWriter struct {
isClosed <-chan bool
}
// newErasureWriter - get a new writer
func newErasureWriter(writers []Writer) ObjectWriter {
r, w := io.Pipe()
isClosed := make(chan bool)

51
pkg/storage/donut/interfaces.go Normal file
View File

@ -0,0 +1,51 @@
package donut
import (
"io"
)
// Collection of Donut specification interfaces
// Donut interface
type Donut interface {
CreateBucket(bucket string) error
GetObjectReader(bucket, object string) (io.ReadCloser, error)
GetObjectWriter(bucket, object string) (ObjectWriter, error)
GetObjectMetadata(bucket, object string) (map[string]string, error)
ListBuckets() ([]string, error)
ListObjects(bucket string) ([]string, error)
}
// Bucket interface
type Bucket interface {
GetNodes() ([]string, error)
AddNode(nodeID, bucketID string) error
}
// Node interface
type Node interface {
CreateBucket(bucket string) error
GetBuckets() ([]string, error)
GetDonutMetadata(bucket, object string) (map[string]string, error)
GetMetadata(bucket, object string) (map[string]string, error)
GetReader(bucket, object string) (io.ReadCloser, error)
GetWriter(bucket, object string) (Writer, error)
ListObjects(bucket string) ([]string, error)
}
// ObjectWriter interface
type ObjectWriter interface {
Close() error
CloseWithError(error) error
GetMetadata() (map[string]string, error)
SetMetadata(map[string]string) error
Write([]byte) (int, error)
}
// Writer interface
type Writer interface {
ObjectWriter
GetDonutMetadata() (map[string]string, error)
SetDonutMetadata(map[string]string) error
}

2
pkg/storage/donut/local.go → pkg/storage/donut/node_local.go
View File

@ -41,7 +41,7 @@ func (d localDirectoryNode) GetWriter(bucket, object string) (Writer, error) {
if err != nil {
return nil, err
}
return newDonutFileWriter(objectPath)
return newDonutObjectWriter(objectPath)
}
func (d localDirectoryNode) GetReader(bucket, object string) (io.ReadCloser, error) {

20
pkg/storage/donut/donutwriter.go → pkg/storage/donut/object_writer.go
View File

@ -7,12 +7,12 @@ import (
"path"
)
func newDonutFileWriter(objectDir string) (Writer, error) {
func newDonutObjectWriter(objectDir string) (Writer, error) {
dataFile, err := os.OpenFile(path.Join(objectDir, "data"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return nil, err
}
return donutFileWriter{
return donutObjectWriter{
root: objectDir,
file: dataFile,
metadata: make(map[string]string),
@ -20,7 +20,7 @@ func newDonutFileWriter(objectDir string) (Writer, error) {
}, nil
}
type donutFileWriter struct {
type donutObjectWriter struct {
root string
file *os.File
metadata map[string]string
@ -28,11 +28,11 @@ type donutFileWriter struct {
err error
}
func (d donutFileWriter) Write(data []byte) (int, error) {
func (d donutObjectWriter) Write(data []byte) (int, error) {
return d.file.Write(data)
}
func (d donutFileWriter) Close() error {
func (d donutObjectWriter) Close() error {
if d.err != nil {
return d.err
}
@ -44,14 +44,14 @@ func (d donutFileWriter) Close() error {
return d.file.Close()
}
func (d donutFileWriter) CloseWithError(err error) error {
func (d donutObjectWriter) CloseWithError(err error) error {
if d.err != nil {
d.err = err
}
return d.Close()
}
func (d donutFileWriter) SetMetadata(metadata map[string]string) error {
func (d donutObjectWriter) SetMetadata(metadata map[string]string) error {
for k := range d.metadata {
delete(d.metadata, k)
}
@ -61,7 +61,7 @@ func (d donutFileWriter) SetMetadata(metadata map[string]string) error {
return nil
}
func (d donutFileWriter) GetMetadata() (map[string]string, error) {
func (d donutObjectWriter) GetMetadata() (map[string]string, error) {
metadata := make(map[string]string)
for k, v := range d.metadata {
metadata[k] = v
@ -69,7 +69,7 @@ func (d donutFileWriter) GetMetadata() (map[string]string, error) {
return metadata, nil
}
func (d donutFileWriter) SetDonutMetadata(metadata map[string]string) error {
func (d donutObjectWriter) SetDonutMetadata(metadata map[string]string) error {
for k := range d.donutMetadata {
delete(d.donutMetadata, k)
}
@ -79,7 +79,7 @@ func (d donutFileWriter) SetDonutMetadata(metadata map[string]string) error {
return nil
}
func (d donutFileWriter) GetDonutMetadata() (map[string]string, error) {
func (d donutObjectWriter) GetDonutMetadata() (map[string]string, error) {
donutMetadata := make(map[string]string)
for k, v := range d.donutMetadata {
donutMetadata[k] = v

39
pkg/storage/donut/objectwriter.go
View File

@ -1,39 +0,0 @@
package donut
import (
"errors"
)
type objectWriter struct {
metadata map[string]string
}
func (obj objectWriter) Write(data []byte) (length int, err error) {
return 11, nil
}
func (obj objectWriter) Close() error {
return nil
}
func (obj objectWriter) CloseWithError(err error) error {
return errors.New("Not Implemented")
}
func (obj objectWriter) SetMetadata(metadata map[string]string) error {
for k := range obj.metadata {
delete(obj.metadata, k)
}
for k, v := range metadata {
obj.metadata[k] = v
}
return nil
}
func (obj objectWriter) GetMetadata() (map[string]string, error) {
ret := make(map[string]string)
for k, v := range obj.metadata {
ret[k] = v
}
return ret, nil
}

133
pkg/storage/file/file_bucket.go
View File

@ -1,133 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"path"
"sort"
"strings"
"io/ioutil"
"path/filepath"
mstorage "github.com/minio-io/minio/pkg/storage"
)
/// Bucket Operations
// ListBuckets - Get service
func (storage *Storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
files, err := ioutil.ReadDir(storage.root)
if err != nil {
return []mstorage.BucketMetadata{}, mstorage.EmbedError("bucket", "", err)
}
var metadataList []mstorage.BucketMetadata
for _, file := range files {
// Skip policy files
if strings.HasSuffix(file.Name(), "_policy.json") {
continue
}
if !file.IsDir() {
return []mstorage.BucketMetadata{}, mstorage.BackendCorrupted{Path: storage.root}
}
metadata := mstorage.BucketMetadata{
Name: file.Name(),
Created: file.ModTime(), // TODO - provide real created time
}
metadataList = append(metadataList, metadata)
}
return metadataList, nil
}
// CreateBucket - PUT Bucket
func (storage *Storage) CreateBucket(bucket string) error {
storage.lock.Lock()
defer storage.lock.Unlock()
// verify bucket path legal
if mstorage.IsValidBucket(bucket) == false {
return mstorage.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(storage.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err == nil {
return mstorage.BucketExists{
Bucket: bucket,
}
}
// make bucket
err := os.Mkdir(bucketDir, 0700)
if err != nil {
return mstorage.EmbedError(bucket, "", err)
}
return nil
}
// ListObjects - GET bucket (list objects)
func (storage *Storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) {
p := bucketDir{}
p.files = make(map[string]os.FileInfo)
if mstorage.IsValidBucket(bucket) == false {
return []mstorage.ObjectMetadata{}, resources, mstorage.BucketNameInvalid{Bucket: bucket}
}
if resources.Prefix != "" && mstorage.IsValidObject(resources.Prefix) == false {
return []mstorage.ObjectMetadata{}, resources, mstorage.ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix}
}
rootPrefix := path.Join(storage.root, bucket)
// check bucket exists
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
return []mstorage.ObjectMetadata{}, resources, mstorage.BucketNotFound{Bucket: bucket}
}
p.root = rootPrefix
err := filepath.Walk(rootPrefix, p.getAllFiles)
if err != nil {
return []mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err)
}
var metadataList []mstorage.ObjectMetadata
var metadata mstorage.ObjectMetadata
// Populate filtering mode
resources.Mode = mstorage.GetMode(resources)
for name, file := range p.files {
if len(metadataList) >= resources.Maxkeys {
resources.IsTruncated = true
goto ret
}
metadata, resources, err = storage.filter(bucket, name, file, resources)
if err != nil {
return []mstorage.ObjectMetadata{}, resources, mstorage.EmbedError(bucket, "", err)
}
if metadata.Bucket != "" {
metadataList = append(metadataList, metadata)
}
}
ret:
sort.Sort(byObjectKey(metadataList))
return metadataList, resources, nil
}

111
pkg/storage/file/file_policy.go
View File

@ -1,111 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"path"
"encoding/json"
mstorage "github.com/minio-io/minio/pkg/storage"
)
// GetBucketPolicy - GET bucket policy
func (storage *Storage) GetBucketPolicy(bucket string) (mstorage.BucketPolicy, error) {
storage.lock.Lock()
defer storage.lock.Unlock()
var p mstorage.BucketPolicy
// verify bucket path legal
if mstorage.IsValidBucket(bucket) == false {
return mstorage.BucketPolicy{}, mstorage.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(storage.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err != nil {
return mstorage.BucketPolicy{}, mstorage.BucketNotFound{Bucket: bucket}
}
// get policy path
bucketPolicy := path.Join(storage.root, bucket+"_policy.json")
filestat, err := os.Stat(bucketPolicy)
if os.IsNotExist(err) {
return mstorage.BucketPolicy{}, mstorage.BucketPolicyNotFound{Bucket: bucket}
}
if filestat.IsDir() {
return mstorage.BucketPolicy{}, mstorage.BackendCorrupted{Path: bucketPolicy}
}
file, err := os.OpenFile(bucketPolicy, os.O_RDONLY, 0666)
defer file.Close()
if err != nil {
return mstorage.BucketPolicy{}, mstorage.EmbedError(bucket, "", err)
}
encoder := json.NewDecoder(file)
err = encoder.Decode(&p)
if err != nil {
return mstorage.BucketPolicy{}, mstorage.EmbedError(bucket, "", err)
}
return p, nil
}
// CreateBucketPolicy - PUT bucket policy
func (storage *Storage) CreateBucketPolicy(bucket string, p mstorage.BucketPolicy) error {
storage.lock.Lock()
defer storage.lock.Unlock()
// verify bucket path legal
if mstorage.IsValidBucket(bucket) == false {
return mstorage.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(storage.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err != nil {
return mstorage.BucketNotFound{
Bucket: bucket,
}
}
// get policy path
bucketPolicy := path.Join(storage.root, bucket+"_policy.json")
filestat, ret := os.Stat(bucketPolicy)
if !os.IsNotExist(ret) {
if filestat.IsDir() {
return mstorage.BackendCorrupted{Path: bucketPolicy}
}
}
file, err := os.OpenFile(bucketPolicy, os.O_WRONLY|os.O_CREATE, 0600)
defer file.Close()
if err != nil {
return mstorage.EmbedError(bucket, "", err)
}
encoder := json.NewEncoder(file)
err = encoder.Encode(p)
if err != nil {
return mstorage.EmbedError(bucket, "", err)
}
return nil
}