MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-06 20:33:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

Updating to use objectdriver

Browse Source
This commit is contained in:
Frederick F. Kautz IV
2015-04-06 18:00:18 -07:00
parent 1ebc692ea2
commit ea961ef16d
35 changed files with 272 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/api/api_bucket_handlers.go
View File

@@ -21,8 +21,8 @@ import (
"github.com/gorilla/mux"
"github.com/minio-io/iodine"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/utils/log"
"github.com/minio-io/objectdriver"
)
// GET Bucket (List Objects)

2
pkg/api/api_object_handlers.go
View File

@@ -21,8 +21,8 @@ import (
"github.com/gorilla/mux"
"github.com/minio-io/iodine"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/utils/log"
"github.com/minio-io/objectdriver"
)
// GET Object

2
pkg/api/api_policy_handlers.go
View File

@@ -22,8 +22,8 @@ import (
"github.com/gorilla/mux"
"github.com/minio-io/iodine"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/utils/log"
"github.com/minio-io/objectdriver"
)
// PUT Bucket policy

2
pkg/api/api_response.go
View File

@@ -19,7 +19,7 @@ package api
import (
"sort"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/objectdriver"
)
// Reply date format

2
pkg/api/api_router.go
View File

@@ -23,7 +23,7 @@ import (
router "github.com/gorilla/mux"
"github.com/minio-io/iodine"
"github.com/minio-io/minio/pkg/api/config"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/objectdriver"
)
// private use

12
pkg/api/api_test.go
View File

@@ -32,13 +32,13 @@ import (
"net/http"
"net/http/httptest"
"github.com/fkautz/testify/mock"
"github.com/minio-io/minio/pkg/api"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/drivers/donut"
"github.com/minio-io/minio/pkg/drivers/file"
"github.com/minio-io/minio/pkg/drivers/memory"
"github.com/minio-io/minio/pkg/drivers/mocks"
"github.com/stretchr/testify/mock"
"github.com/minio-io/objectdriver"
"github.com/minio-io/objectdriver/donut"
"github.com/minio-io/objectdriver/file"
"github.com/minio-io/objectdriver/memory"
"github.com/minio-io/objectdriver/mocks"
. "github.com/minio-io/check"
)

2
pkg/api/headers.go
View File

@@ -24,7 +24,7 @@ import (
"strconv"
"time"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/objectdriver"
)
// No encoder interface exists, so we create one.

2
pkg/api/resources.go
View File

@@ -20,7 +20,7 @@ import (
"net/url"
"strconv"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/objectdriver"
)
// parse bucket url queries

392
pkg/drivers/api_testsuite.go
View File

@@ -1,392 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
import (
"bytes"
"math/rand"
"strconv"
"time"
"github.com/minio-io/check"
)
// APITestSuite - collection of API tests
func APITestSuite(c *check.C, create func() Driver) {
testCreateBucket(c, create)
testMultipleObjectCreation(c, create)
testPaging(c, create)
testObjectOverwriteFails(c, create)
testNonExistantBucketOperations(c, create)
testBucketRecreateFails(c, create)
testPutObjectInSubdir(c, create)
testListBuckets(c, create)
testListBucketsOrder(c, create)
testListObjectsTestsForNonExistantBucket(c, create)
testNonExistantObjectInBucket(c, create)
testGetDirectoryReturnsObjectNotFound(c, create)
testDefaultContentType(c, create)
//testContentMd5Set(c, create) TODO
}
func testCreateBucket(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
}
func testMultipleObjectCreation(c *check.C, create func() Driver) {
objects := make(map[string][]byte)
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
for i := 0; i < 10; i++ {
randomPerm := rand.Perm(10)
randomString := ""
for _, num := range randomPerm {
randomString = randomString + strconv.Itoa(num)
}
key := "obj" + strconv.Itoa(i)
objects[key] = []byte(randomString)
err := drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(randomString))
c.Assert(err, check.IsNil)
}
// ensure no duplicate etags
etags := make(map[string]string)
for key, value := range objects {
var byteBuffer bytes.Buffer
_, err := drivers.GetObject(&byteBuffer, "bucket", key)
c.Assert(err, check.IsNil)
c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
metadata, err := drivers.GetObjectMetadata("bucket", key, "")
c.Assert(err, check.IsNil)
c.Assert(metadata.Size, check.Equals, int64(len(value)))
_, ok := etags[metadata.Md5]
c.Assert(ok, check.Equals, false)
etags[metadata.Md5] = metadata.Md5
}
}
func testPaging(c *check.C, create func() Driver) {
drivers := create()
drivers.CreateBucket("bucket")
resources := BucketResourcesMetadata{}
objects, resources, err := drivers.ListObjects("bucket", resources)
c.Assert(err, check.IsNil)
c.Assert(len(objects), check.Equals, 0)
c.Assert(resources.IsTruncated, check.Equals, false)
// check before paging occurs
for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i)
drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
resources.Maxkeys = 5
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(len(objects), check.Equals, i+1)
c.Assert(resources.IsTruncated, check.Equals, false)
c.Assert(err, check.IsNil)
}
// check after paging occurs pages work
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
resources.Maxkeys = 5
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(len(objects), check.Equals, 5)
c.Assert(resources.IsTruncated, check.Equals, true)
c.Assert(err, check.IsNil)
}
// check paging with prefix at end returns less objects
{
drivers.CreateObject("bucket", "newPrefix", "", "", bytes.NewBufferString("prefix1"))
drivers.CreateObject("bucket", "newPrefix2", "", "", bytes.NewBufferString("prefix2"))
resources.Prefix = "new"
resources.Maxkeys = 5
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(len(objects), check.Equals, 2)
}
// check ordering of pages
{
resources.Prefix = ""
resources.Maxkeys = 1000
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "newPrefix")
c.Assert(objects[1].Key, check.Equals, "newPrefix2")
c.Assert(objects[2].Key, check.Equals, "obj0")
c.Assert(objects[3].Key, check.Equals, "obj1")
c.Assert(objects[4].Key, check.Equals, "obj10")
}
// check delimited results with delimiter and prefix
{
drivers.CreateObject("bucket", "this/is/delimited", "", "", bytes.NewBufferString("prefix1"))
drivers.CreateObject("bucket", "this/is/also/a/delimited/file", "", "", bytes.NewBufferString("prefix2"))
var prefixes []string
resources.CommonPrefixes = prefixes // allocate new everytime
resources.Delimiter = "/"
resources.Prefix = "this/is/"
resources.Maxkeys = 10
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(err, check.IsNil)
c.Assert(len(objects), check.Equals, 1)
c.Assert(resources.CommonPrefixes[0], check.Equals, "also/")
}
time.Sleep(time.Second)
// check delimited results with delimiter without prefix
{
var prefixes []string
resources.CommonPrefixes = prefixes // allocate new everytime
resources.Delimiter = "/"
resources.Prefix = ""
resources.Maxkeys = 1000
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "newPrefix")
c.Assert(objects[1].Key, check.Equals, "newPrefix2")
c.Assert(objects[2].Key, check.Equals, "obj0")
c.Assert(objects[3].Key, check.Equals, "obj1")
c.Assert(objects[4].Key, check.Equals, "obj10")
c.Assert(resources.CommonPrefixes[0], check.Equals, "this/")
}
// check ordering of results with prefix
{
resources.Prefix = "obj"
resources.Delimiter = ""
resources.Maxkeys = 1000
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "obj0")
c.Assert(objects[1].Key, check.Equals, "obj1")
c.Assert(objects[2].Key, check.Equals, "obj10")
c.Assert(objects[3].Key, check.Equals, "obj2")
c.Assert(objects[4].Key, check.Equals, "obj3")
}
// check ordering of results with prefix and no paging
{
resources.Prefix = "new"
resources.Maxkeys = 5
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "newPrefix")
c.Assert(objects[1].Key, check.Equals, "newPrefix2")
}
}
func testObjectOverwriteFails(c *check.C, create func() Driver) {
drivers := create()
drivers.CreateBucket("bucket")
err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
c.Assert(err, check.IsNil)
err = drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("three"))
c.Assert(err, check.Not(check.IsNil))
var bytesBuffer bytes.Buffer
length, err := drivers.GetObject(&bytesBuffer, "bucket", "object")
c.Assert(length, check.Equals, int64(len("one")))
c.Assert(err, check.IsNil)
c.Assert(string(bytesBuffer.Bytes()), check.Equals, "one")
}
func testNonExistantBucketOperations(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil))
}
func testBucketRecreateFails(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("string")
c.Assert(err, check.IsNil)
err = drivers.CreateBucket("string")
c.Assert(err, check.Not(check.IsNil))
}
func testPutObjectInSubdir(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil)
var bytesBuffer bytes.Buffer
length, err := drivers.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object")
c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length)
c.Assert(err, check.IsNil)
}
func testListBuckets(c *check.C, create func() Driver) {
drivers := create()
// test empty list
buckets, err := drivers.ListBuckets()
c.Assert(err, check.IsNil)
c.Assert(len(buckets), check.Equals, 0)
// add one and test exists
err = drivers.CreateBucket("bucket1")
c.Assert(err, check.IsNil)
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 1)
c.Assert(err, check.IsNil)
// add two and test exists
err = drivers.CreateBucket("bucket2")
c.Assert(err, check.IsNil)
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 2)
c.Assert(err, check.IsNil)
// add three and test exists + prefix
err = drivers.CreateBucket("bucket22")
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 3)
c.Assert(err, check.IsNil)
}
func testListBucketsOrder(c *check.C, create func() Driver) {
// if implementation contains a map, order of map keys will vary.
// this ensures they return in the same order each time
for i := 0; i < 10; i++ {
drivers := create()
// add one and test exists
drivers.CreateBucket("bucket1")
drivers.CreateBucket("bucket2")
buckets, err := drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 2)
c.Assert(err, check.IsNil)
c.Assert(buckets[0].Name, check.Equals, "bucket1")
c.Assert(buckets[1].Name, check.Equals, "bucket2")
}
}
func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Driver) {
drivers := create()
resources := BucketResourcesMetadata{Prefix: "", Maxkeys: 1000}
objects, resources, err := drivers.ListObjects("bucket", resources)
c.Assert(err, check.Not(check.IsNil))
c.Assert(resources.IsTruncated, check.Equals, false)
c.Assert(len(objects), check.Equals, 0)
}
func testNonExistantObjectInBucket(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
length, err := drivers.GetObject(&byteBuffer, "bucket", "dir1")
c.Assert(length, check.Equals, int64(0))
c.Assert(err, check.Not(check.IsNil))
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
switch err := err.(type) {
case ObjectNotFound:
{
c.Assert(err, check.ErrorMatches, "Object not Found: bucket#dir1")
}
default:
{
c.Assert(err, check.Equals, "fails")
}
}
}
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
length, err := drivers.GetObject(&byteBuffer, "bucket", "dir1")
c.Assert(length, check.Equals, int64(0))
switch err := err.(type) {
case ObjectNotFound:
{
c.Assert(err.Bucket, check.Equals, "bucket")
c.Assert(err.Object, check.Equals, "dir1")
}
default:
{
// force a failure with a line number
c.Assert(err, check.Equals, "ObjectNotFound")
}
}
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
var byteBuffer2 bytes.Buffer
length, err = drivers.GetObject(&byteBuffer, "bucket", "dir1/")
c.Assert(length, check.Equals, int64(0))
switch err := err.(type) {
case ObjectNotFound:
{
c.Assert(err.Bucket, check.Equals, "bucket")
c.Assert(err.Object, check.Equals, "dir1/")
}
default:
{
// force a failure with a line number
c.Assert(err, check.Equals, "ObjectNotFound")
}
}
c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
}
func testDefaultContentType(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
// test empty
err = drivers.CreateObject("bucket", "one", "", "", bytes.NewBufferString("one"))
metadata, err := drivers.GetObjectMetadata("bucket", "one", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
// test custom
drivers.CreateObject("bucket", "two", "application/text", "", bytes.NewBufferString("two"))
metadata, err = drivers.GetObjectMetadata("bucket", "two", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/text")
// test trim space
drivers.CreateObject("bucket", "three", "\tapplication/json ", "", bytes.NewBufferString("three"))
metadata, err = drivers.GetObjectMetadata("bucket", "three", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/json")
}
/*
func testContentMd5Set(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
// test md5 invalid
err = drivers.CreateObject("bucket", "one", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA", bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil))
err = drivers.CreateObject("bucket", "two", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=", bytes.NewBufferString("one"))
c.Assert(err, check.IsNil)
}
*/

199
pkg/drivers/bucket_policy.go
View File

@@ -1,199 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
import (
"encoding/json"
"io"
"strings"
)
// User - canonical
type User struct {
AWS string
}
// Statement - minio policy statement
type Statement struct {
Sid string
Effect string
Principal User
Action []string
Resource []string
// add Condition struct/var TODO - fix it in future if necessary
}
// BucketPolicy - minio policy collection
type BucketPolicy struct {
Version string // date in 0000-00-00 format
Statement []Statement
}
// Resource delimiter
const (
MinioResource = "minio:::"
)
// TODO support canonical user
// Principal delimiter
const (
MinioPrincipal = "minio::"
)
// Action map
var SupportedActionMap = map[string]bool{
"*": true,
"minio:GetObject": true,
"minio:ListBucket": true,
"minio:PutObject": true,
"minio:CreateBucket": true,
"minio:GetBucketPolicy": true,
"minio:DeleteBucketPolicy": true,
"minio:ListAllMyBuckets": true,
"minio:PutBucketPolicy": true,
}
// Effect map
var SupportedEffectMap = map[string]bool{
"Allow": true,
"Deny": true,
}
func isValidAction(action []string) bool {
for _, a := range action {
if !SupportedActionMap[a] {
return false
}
}
return true
}
func isValidEffect(effect string) bool {
if SupportedEffectMap[effect] {
return true
}
return false
}
func isValidResource(resources []string) bool {
var ok bool
for _, resource := range resources {
switch true {
case strings.HasPrefix(resource, AwsResource):
bucket := strings.SplitAfter(resource, AwsResource)[1]
ok = true
if len(bucket) == 0 {
ok = false
}
case strings.HasPrefix(resource, MinioResource):
bucket := strings.SplitAfter(resource, MinioResource)[1]
ok = true
if len(bucket) == 0 {
ok = false
}
default:
ok = false
}
}
return ok
}
func isValidPrincipal(principal string) bool {
var ok bool
if principal == "*" {
return true
}
switch true {
case strings.HasPrefix(principal, AwsPrincipal):
username := strings.SplitAfter(principal, AwsPrincipal)[1]
ok = true
if len(username) == 0 {
ok = false
}
case strings.HasPrefix(principal, MinioPrincipal):
username := strings.SplitAfter(principal, MinioPrincipal)[1]
ok = true
if len(username) == 0 {
ok = false
}
default:
ok = false
}
return ok
}
func parseStatement(statement Statement) bool {
if len(statement.Sid) == 0 {
return false
}
if len(statement.Effect) == 0 {
return false
}
if !isValidEffect(statement.Effect) {
return false
}
if len(statement.Principal.AWS) == 0 {
return false
}
if !isValidPrincipal(statement.Principal.AWS) {
return false
}
if len(statement.Action) == 0 {
return false
}
if !isValidAction(statement.Action) && !isValidActionS3(statement.Action) {
return false
}
if len(statement.Resource) == 0 {
return false
}
if !isValidResource(statement.Resource) {
return false
}
return true
}
// Parsepolicy - validate request body is proper JSON and in accordance with policy standards
func Parsepolicy(data io.Reader) (BucketPolicy, bool) {
var policy BucketPolicy
decoder := json.NewDecoder(data)
err := decoder.Decode(&policy)
if err != nil {
goto error
}
if len(policy.Version) == 0 {
goto error
}
_, err = parseDate(policy.Version)
if err != nil {
goto error
}
if len(policy.Statement) == 0 {
goto error
}
for _, statement := range policy.Statement {
if !parseStatement(statement) {
goto error
}
}
return policy, true
error:
return BucketPolicy{}, false
}

52
pkg/drivers/bucket_policy_compat.go
View File

@@ -1,52 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
// This file implements compatability layer for AWS clients
// Resource delimiter
const (
AwsResource = "arn:aws:s3:::"
)
// TODO support canonical user
// Principal delimiter
const (
AwsPrincipal = "arn:aws:iam::"
)
// Action map
var SupportedActionMapCompat = map[string]bool{
"*": true,
"s3:GetObject": true,
"s3:ListBucket": true,
"s3:PutObject": true,
"s3:CreateBucket": true,
"s3:GetBucketPolicy": true,
"s3:DeleteBucketPolicy": true,
"s3:ListAllMyBuckets": true,
"s3:PutBucketPolicy": true,
}
func isValidActionS3(action []string) bool {
for _, a := range action {
if !SupportedActionMapCompat[a] {
return false
}
}
return true
}

78
pkg/drivers/date.go
View File

@@ -1,78 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
import (
"errors"
"fmt"
"strconv"
"strings"
)
// Date - [0000-00-00]
type Date struct {
Year int16
Month byte
Day byte
}
// String output in yyyy-mm-dd format
func (d Date) String() string {
return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day)
}
// IsZero true if date is 0000-00-00
func (d Date) IsZero() bool {
return d.Day == 0 && d.Month == 0 && d.Year == 0
}
// Convert string date in format YYYY-MM-DD to Date.
// Leading and trailing spaces are ignored. If format is invalid returns zero.
func parseDate(str string) (d Date, err error) {
str = strings.TrimSpace(str)
if str == "0000-00-00" {
return
}
var (
y, m, n int
)
if len(str) != 10 || str[4] != '-' || str[7] != '-' {
err = errors.New("Invalid 0000-00-000 style DATE string: " + str)
return
}
if y, err = strconv.Atoi(str[0:4]); err != nil {
return
}
if m, err = strconv.Atoi(str[5:7]); err != nil {
return
}
if m < 1 || m > 12 {
err = errors.New("Invalid 0000-00-000 style DATE string: " + str)
return
}
if n, err = strconv.Atoi(str[8:10]); err != nil {
return
}
if n < 1 || n > 31 {
err = errors.New("Invalid 0000-00-000 style DATE string: " + str)
return
}
d.Year = int16(y)
d.Month = byte(m)
d.Day = byte(n)
return
}

376
pkg/drivers/donut/donut.go
View File

@@ -1,376 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package donut
import (
"errors"
"io"
"os"
"path"
"sort"
"strconv"
"strings"
"time"
"io/ioutil"
"github.com/minio-io/donut"
"github.com/minio-io/iodine"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/utils/log"
)
// donutDriver - creates a new single disk drivers driver using donut
type donutDriver struct {
donut donut.Donut
}
const (
blockSize = 10 * 1024 * 1024
)
// This is a dummy nodeDiskMap which is going to be deprecated soon
// once the Management API is standardized, this map is useful for now
// to show multi disk API correctness behavior.
//
// This should be obtained from donut configuration file
func createNodeDiskMap(p string) map[string][]string {
nodes := make(map[string][]string)
nodes["localhost"] = make([]string, 16)
for i := 0; i < len(nodes["localhost"]); i++ {
diskPath := path.Join(p, strconv.Itoa(i))
if _, err := os.Stat(diskPath); err != nil {
if os.IsNotExist(err) {
os.MkdirAll(diskPath, 0700)
}
}
nodes["localhost"][i] = diskPath
}
return nodes
}
// Start a single disk subsystem
func Start(path string) (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
errParams := map[string]string{"path": path}
// Soon to be user configurable, when Management API
// is finished we remove "default" to something
// which is passed down from configuration
donut, err := donut.NewDonut("default", createNodeDiskMap(path))
if err != nil {
err = iodine.New(err, errParams)
log.Error.Println(err)
}
s := new(donutDriver)
s.donut = donut
go start(ctrlChannel, errorChannel, s)
return ctrlChannel, errorChannel, s
}
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *donutDriver) {
close(errorChannel)
}
// byBucketName is a type for sorting bucket metadata by bucket name
type byBucketName []drivers.BucketMetadata
func (b byBucketName) Len() int { return len(b) }
func (b byBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
// ListBuckets returns a list of buckets
func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error) {
buckets, err := d.donut.ListBuckets()
if err != nil {
return nil, err
}
for name := range buckets {
result := drivers.BucketMetadata{
Name: name,
// TODO Add real created date
Created: time.Now(),
}
results = append(results, result)
}
sort.Sort(byBucketName(results))
return results, nil
}
// CreateBucket creates a new bucket
func (d donutDriver) CreateBucket(bucketName string) error {
if drivers.IsValidBucket(bucketName) && !strings.Contains(bucketName, ".") {
return d.donut.MakeBucket(bucketName)
}
return errors.New("Invalid bucket")
}
// GetBucketMetadata retrieves an bucket's metadata
func (d donutDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
return drivers.BucketMetadata{}, errors.New("Not Implemented")
}
// CreateBucketPolicy sets a bucket's access policy
func (d donutDriver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error {
return errors.New("Not Implemented")
}
// GetBucketPolicy returns a bucket's access policy
func (d donutDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
return drivers.BucketPolicy{}, errors.New("Not Implemented")
}
// GetObject retrieves an object and writes it to a writer
func (d donutDriver) GetObject(target io.Writer, bucketName, objectName string) (int64, error) {
errParams := map[string]string{
"bucketName": bucketName,
"objectName": objectName,
}
if bucketName == "" || strings.TrimSpace(bucketName) == "" {
return 0, iodine.New(errors.New("invalid argument"), errParams)
}
if objectName == "" || strings.TrimSpace(objectName) == "" {
return 0, iodine.New(errors.New("invalid argument"), errParams)
}
buckets, err := d.donut.ListBuckets()
if err != nil {
return 0, iodine.New(err, nil)
}
if _, ok := buckets[bucketName]; !ok {
return 0, drivers.BucketNotFound{Bucket: bucketName}
}
reader, size, err := buckets[bucketName].GetObject(objectName)
if err != nil {
return 0, drivers.ObjectNotFound{
Bucket: bucketName,
Object: objectName,
}
}
n, err := io.CopyN(target, reader, size)
return n, iodine.New(err, errParams)
}
// GetPartialObject retrieves an object range and writes it to a writer
func (d donutDriver) GetPartialObject(w io.Writer, bucketName, objectName string, start, length int64) (int64, error) {
// TODO more efficient get partial object with proper donut support
errParams := map[string]string{
"bucketName": bucketName,
"objectName": objectName,
"start": strconv.FormatInt(start, 10),
"length": strconv.FormatInt(length, 10),
}
if bucketName == "" || strings.TrimSpace(bucketName) == "" {
return 0, iodine.New(errors.New("invalid argument"), errParams)
}
if objectName == "" || strings.TrimSpace(objectName) == "" {
return 0, iodine.New(errors.New("invalid argument"), errParams)
}
if start < 0 {
return 0, iodine.New(errors.New("invalid argument"), errParams)
}
buckets, err := d.donut.ListBuckets()
if err != nil {
return 0, iodine.New(err, nil)
}
if _, ok := buckets[bucketName]; !ok {
return 0, drivers.BucketNotFound{Bucket: bucketName}
}
reader, size, err := buckets[bucketName].GetObject(objectName)
defer reader.Close()
if err != nil {
return 0, drivers.ObjectNotFound{
Bucket: bucketName,
Object: objectName,
}
}
if start > size || (start+length-1) > size {
return 0, iodine.New(errors.New("invalid range"), errParams)
}
_, err = io.CopyN(ioutil.Discard, reader, start)
if err != nil {
return 0, iodine.New(err, errParams)
}
n, err := io.CopyN(w, reader, length)
return n, iodine.New(err, errParams)
}
// GetObjectMetadata retrieves an object's metadata
func (d donutDriver) GetObjectMetadata(bucketName, objectName, prefixName string) (drivers.ObjectMetadata, error) {
errParams := map[string]string{
"bucketName": bucketName,
"objectName": objectName,
"prefixName": prefixName,
}
buckets, err := d.donut.ListBuckets()
if err != nil {
return drivers.ObjectMetadata{}, iodine.New(err, errParams)
}
if _, ok := buckets[bucketName]; !ok {
return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucketName}
}
objectList, err := buckets[bucketName].ListObjects()
if err != nil {
return drivers.ObjectMetadata{}, iodine.New(err, errParams)
}
object, ok := objectList[objectName]
if !ok {
// return ObjectNotFound quickly on an error, API needs this to handle invalid requests
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{
Bucket: bucketName,
Object: objectName,
}
}
donutObjectMetadata, err := object.GetDonutObjectMetadata()
if err != nil {
// return ObjectNotFound quickly on an error, API needs this to handle invalid requests
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{
Bucket: bucketName,
Object: objectName,
}
}
objectMetadata, err := object.GetObjectMetadata()
if err != nil {
// return ObjectNotFound quickly on an error, API needs this to handle invalid requests
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{
Bucket: bucketName,
Object: objectName,
}
}
created, err := time.Parse(time.RFC3339Nano, donutObjectMetadata["created"])
if err != nil {
return drivers.ObjectMetadata{}, iodine.New(err, nil)
}
size, err := strconv.ParseInt(donutObjectMetadata["size"], 10, 64)
if err != nil {
return drivers.ObjectMetadata{}, iodine.New(err, nil)
}
driversObjectMetadata := drivers.ObjectMetadata{
Bucket: bucketName,
Key: objectName,
ContentType: objectMetadata["contentType"],
Created: created,
Md5: donutObjectMetadata["md5"],
Size: size,
}
return driversObjectMetadata, nil
}
type byObjectKey []drivers.ObjectMetadata
func (b byObjectKey) Len() int { return len(b) }
func (b byObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key }
// ListObjects - returns list of objects
func (d donutDriver) ListObjects(bucketName string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
errParams := map[string]string{
"bucketName": bucketName,
}
buckets, err := d.donut.ListBuckets()
if err != nil {
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams)
}
if _, ok := buckets[bucketName]; !ok {
return nil, drivers.BucketResourcesMetadata{}, drivers.BucketNotFound{Bucket: bucketName}
}
objectList, err := buckets[bucketName].ListObjects()
if err != nil {
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams)
}
var objects []string
for key := range objectList {
objects = append(objects, key)
}
sort.Strings(objects)
if resources.Maxkeys <= 0 || resources.Maxkeys > 1000 {
resources.Maxkeys = 1000
}
// Populate filtering mode
resources.Mode = drivers.GetMode(resources)
// filter objects based on resources.Prefix and resources.Delimiter
actualObjects, commonPrefixes := d.filter(objects, resources)
resources.CommonPrefixes = commonPrefixes
var results []drivers.ObjectMetadata
for _, objectName := range actualObjects {
if len(results) >= resources.Maxkeys {
resources.IsTruncated = true
break
}
if _, ok := objectList[objectName]; !ok {
return nil, drivers.BucketResourcesMetadata{}, iodine.New(errors.New("object corrupted"), errParams)
}
objectMetadata, err := objectList[objectName].GetDonutObjectMetadata()
if err != nil {
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, errParams)
}
t, err := time.Parse(time.RFC3339Nano, objectMetadata["created"])
if err != nil {
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil)
}
size, err := strconv.ParseInt(objectMetadata["size"], 10, 64)
if err != nil {
return nil, drivers.BucketResourcesMetadata{}, iodine.New(err, nil)
}
metadata := drivers.ObjectMetadata{
Key: objectName,
Created: t,
Size: size,
}
results = append(results, metadata)
}
sort.Sort(byObjectKey(results))
return results, resources, nil
}
// CreateObject creates a new object
func (d donutDriver) CreateObject(bucketName, objectName, contentType, expectedMd5sum string, reader io.Reader) error {
errParams := map[string]string{
"bucketName": bucketName,
"objectName": objectName,
"contentType": contentType,
}
if bucketName == "" || strings.TrimSpace(bucketName) == "" {
return iodine.New(errors.New("invalid argument"), errParams)
}
if objectName == "" || strings.TrimSpace(objectName) == "" {
return iodine.New(errors.New("invalid argument"), errParams)
}
buckets, err := d.donut.ListBuckets()
if err != nil {
return iodine.New(err, errParams)
}
if _, ok := buckets[bucketName]; !ok {
return drivers.BucketNotFound{Bucket: bucketName}
}
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
err = buckets[bucketName].PutObject(objectName, contentType, reader)
if err != nil {
return iodine.New(err, errParams)
}
// handle expectedMd5sum
return nil
}

68
pkg/drivers/donut/donut_filter.go
View File

@@ -1,68 +0,0 @@
package donut
import (
"bufio"
"bytes"
"strings"
"github.com/minio-io/minio/pkg/drivers"
)
func delimiter(object, delimiter string) string {
readBuffer := bytes.NewBufferString(object)
reader := bufio.NewReader(readBuffer)
stringReader := strings.NewReader(delimiter)
delimited, _ := stringReader.ReadByte()
delimitedStr, _ := reader.ReadString(delimited)
return delimitedStr
}
func appendUniq(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}
func (d donutDriver) filter(objects []string, resources drivers.BucketResourcesMetadata) ([]string, []string) {
var actualObjects []string
var commonPrefixes []string
for _, name := range objects {
switch true {
// Both delimiter and Prefix is present
case resources.IsDelimiterPrefixSet():
if strings.HasPrefix(name, resources.Prefix) {
trimmedName := strings.TrimPrefix(name, resources.Prefix)
delimitedName := delimiter(trimmedName, resources.Delimiter)
if delimitedName != "" {
if delimitedName == resources.Delimiter {
commonPrefixes = appendUniq(commonPrefixes, resources.Prefix+delimitedName)
} else {
commonPrefixes = appendUniq(commonPrefixes, delimitedName)
}
if trimmedName == delimitedName {
actualObjects = appendUniq(actualObjects, name)
}
}
}
// Delimiter present and Prefix is absent
case resources.IsDelimiterSet():
delimitedName := delimiter(name, resources.Delimiter)
switch true {
case delimitedName == name:
actualObjects = appendUniq(actualObjects, name)
case delimitedName != "":
commonPrefixes = appendUniq(commonPrefixes, delimitedName)
}
case resources.IsPrefixSet():
if strings.HasPrefix(name, resources.Prefix) {
actualObjects = appendUniq(actualObjects, name)
}
case resources.IsDefault():
return objects, nil
}
}
return actualObjects, commonPrefixes
}

54
pkg/drivers/donut/donut_test.go
View File

@@ -1,54 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package donut
import (
"io/ioutil"
"os"
"testing"
"github.com/minio-io/minio/pkg/drivers"
. "github.com/minio-io/check"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
// c.Skip("Not Implemented")
var storageList []string
create := func() drivers.Driver {
path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
c.Check(err, IsNil)
storageList = append(storageList, path)
_, _, store := Start(path) // TODO Make InMemory driver
return store
}
drivers.APITestSuite(c, create)
removeRoots(c, storageList)
}
func removeRoots(c *C, roots []string) {
for _, root := range roots {
err := os.RemoveAll(root)
c.Check(err, IsNil)
}
}

150
pkg/drivers/driver.go
View File

@@ -1,150 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
import (
"io"
"regexp"
"time"
"unicode/utf8"
)
// Driver - generic API interface for various drivers - donut, file, memory
type Driver interface {
// Bucket Operations
ListBuckets() ([]BucketMetadata, error)
CreateBucket(bucket string) error
CreateBucketPolicy(bucket string, p BucketPolicy) error
GetBucketPolicy(bucket string) (BucketPolicy, error)
// Object Operations
GetObject(w io.Writer, bucket, object string) (int64, error)
GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error)
GetObjectMetadata(bucket string, object string, prefix string) (ObjectMetadata, error)
ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, error)
CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error
}
// BucketMetadata - name and create date
type BucketMetadata struct {
Name string
Created time.Time
}
// ObjectMetadata - object key and its relevant metadata
type ObjectMetadata struct {
Bucket string
Key string
ContentType string
Created time.Time
Md5 string
Size int64
}
// FilterMode type
type FilterMode int
// FilterMode list
const (
DelimiterPrefixMode FilterMode = iota
DelimiterMode
PrefixMode
DefaultMode
)
// BucketResourcesMetadata - various types of bucket resources
type BucketResourcesMetadata struct {
Prefix string
Marker string
Maxkeys int
Delimiter string
IsTruncated bool
CommonPrefixes []string
Mode FilterMode
Policy bool
// TODO
Logging string
Notification string
}
// GetMode - Populate filter mode
func GetMode(resources BucketResourcesMetadata) FilterMode {
var f FilterMode
switch true {
case resources.Delimiter != "" && resources.Prefix != "":
f = DelimiterPrefixMode
case resources.Delimiter != "" && resources.Prefix == "":
f = DelimiterMode
case resources.Delimiter == "" && resources.Prefix != "":
f = PrefixMode
case resources.Delimiter == "" && resources.Prefix == "":
f = DefaultMode
}
return f
}
// IsDelimiterPrefixSet Delimiter and Prefix set
func (b BucketResourcesMetadata) IsDelimiterPrefixSet() bool {
return b.Mode == DelimiterPrefixMode
}
// IsDelimiterSet Delimiter set
func (b BucketResourcesMetadata) IsDelimiterSet() bool {
return b.Mode == DelimiterMode
}
// IsPrefixSet Prefix set
func (b BucketResourcesMetadata) IsPrefixSet() bool {
return b.Mode == PrefixMode
}
// IsDefault No query values
func (b BucketResourcesMetadata) IsDefault() bool {
return b.Mode == DefaultMode
}
// IsValidBucket - verify bucket name in accordance with
// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
func IsValidBucket(bucket string) bool {
if len(bucket) < 3 || len(bucket) > 63 {
return false
}
if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
return false
}
if match, _ := regexp.MatchString("\\.\\.", bucket); match == true {
return false
}
// We don't support buckets with '.' in them
match, _ := regexp.MatchString("^[a-zA-Z][a-zA-Z0-9\\-]+[a-zA-Z0-9]$", bucket)
return match
}
// IsValidObject - verify object name in accordance with
// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
func IsValidObject(object string) bool {
if len(object) > 1024 || len(object) == 0 {
return false
}
if !utf8.ValidString(object) {
return false
}
return true
}

163
pkg/drivers/errors.go
View File

@@ -1,163 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
// BackendError - generic disk backend error
type BackendError struct {
Path string
}
// BackendCorrupted - path has corrupted data
type BackendCorrupted BackendError
// APINotImplemented - generic API not implemented error
type APINotImplemented struct {
API string
}
// GenericBucketError - generic bucket error
type GenericBucketError struct {
Bucket string
}
// GenericObjectError - generic object error
type GenericObjectError struct {
Bucket string
Object string
}
// ImplementationError - generic implementation error
type ImplementationError struct {
Bucket string
Object string
Err error
}
// DigestError - Generic Md5 error
type DigestError struct {
Bucket string
Key string
Md5 string
}
/// Bucket related errors
// BucketPolicyNotFound - missing bucket policy
type BucketPolicyNotFound GenericBucketError
// BucketNameInvalid - bucketname provided is invalid
type BucketNameInvalid GenericBucketError
// BucketExists - bucket already exists
type BucketExists GenericBucketError
// BucketNotFound - requested bucket not found
type BucketNotFound GenericBucketError
/// Object related errors
// ObjectNotFound - requested object not found
type ObjectNotFound GenericObjectError
// ObjectExists - object already exists
type ObjectExists GenericObjectError
// ObjectNameInvalid - object name provided is invalid
type ObjectNameInvalid GenericObjectError
// BadDigest - md5 mismatch from data received
type BadDigest DigestError
// InvalidDigest - md5 in request header invalid
type InvalidDigest DigestError
// Return string an error formatted as the given text
func (e ImplementationError) Error() string {
error := ""
if e.Bucket != "" {
error = error + "Bucket: " + e.Bucket + " "
}
if e.Object != "" {
error = error + "Object: " + e.Object + " "
}
error = error + "Error: " + e.Err.Error()
return error
}
// EmbedError - wrapper function for error object
func EmbedError(bucket, object string, err error) ImplementationError {
return ImplementationError{
Bucket: bucket,
Object: object,
Err: err,
}
}
// Return string an error formatted as the given text
func (e BucketPolicyNotFound) Error() string {
return "Bucket policy not found for: " + e.Bucket
}
// Return string an error formatted as the given text
func (e ObjectNotFound) Error() string {
return "Object not Found: " + e.Bucket + "#" + e.Object
}
// Return string an error formatted as the given text
func (e APINotImplemented) Error() string {
return "Api not implemented: " + e.API
}
// Return string an error formatted as the given text
func (e ObjectExists) Error() string {
return "Object exists: " + e.Bucket + "#" + e.Object
}
// Return string an error formatted as the given text
func (e BucketNameInvalid) Error() string {
return "Bucket name invalid: " + e.Bucket
}
// Return string an error formatted as the given text
func (e BucketExists) Error() string {
return "Bucket exists: " + e.Bucket
}
// Return string an error formatted as the given text
func (e BucketNotFound) Error() string {
return "Bucket not Found: " + e.Bucket
}
// Return string an error formatted as the given text
func (e ObjectNameInvalid) Error() string {
return "Object name invalid: " + e.Bucket + "#" + e.Object
}
// Return string an error formatted as the given text
func (e BackendCorrupted) Error() string {
return "Backend corrupted: " + e.Path
}
// Return string an error formatted as the given text
func (e BadDigest) Error() string {
return "Md5 provided " + e.Md5 + " mismatches for: " + e.Bucket + "#" + e.Key
}
// Return string an error formatted as the given text
func (e InvalidDigest) Error() string {
return "Md5 provided " + e.Md5 + " is invalid"
}

41
pkg/drivers/file/file.go
View File

@@ -1,41 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"sync"
"github.com/minio-io/minio/pkg/drivers"
)
// Start filesystem channel
func Start(root string) (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
s := new(fileDriver)
s.root = root
s.lock = new(sync.Mutex)
go start(ctrlChannel, errorChannel, s)
return ctrlChannel, errorChannel, s
}
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *fileDriver) {
err := os.MkdirAll(s.root, 0700)
errorChannel <- err
close(errorChannel)
}

133
pkg/drivers/file/file_bucket.go
View File

@@ -1,133 +0,0 @@
/*
* Minimalist Object File, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"path"
"sort"
"strings"
"io/ioutil"
"path/filepath"
"github.com/minio-io/minio/pkg/drivers"
)
/// Bucket Operations
// ListBuckets - Get service
func (file *fileDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
files, err := ioutil.ReadDir(file.root)
if err != nil {
return []drivers.BucketMetadata{}, drivers.EmbedError("bucket", "", err)
}
var metadataList []drivers.BucketMetadata
for _, fileName := range files {
// Skip policy files
if strings.HasSuffix(fileName.Name(), "_policy.json") {
continue
}
if !fileName.IsDir() {
return []drivers.BucketMetadata{}, drivers.BackendCorrupted{Path: file.root}
}
metadata := drivers.BucketMetadata{
Name: fileName.Name(),
Created: fileName.ModTime(), // TODO - provide real created time
}
metadataList = append(metadataList, metadata)
}
return metadataList, nil
}
// CreateBucket - PUT Bucket
func (file *fileDriver) CreateBucket(bucket string) error {
file.lock.Lock()
defer file.lock.Unlock()
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(file.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err == nil {
return drivers.BucketExists{
Bucket: bucket,
}
}
// make bucket
err := os.Mkdir(bucketDir, 0700)
if err != nil {
return drivers.EmbedError(bucket, "", err)
}
return nil
}
// ListObjects - GET bucket (list objects)
func (file *fileDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
p := bucketDir{}
p.files = make(map[string]os.FileInfo)
if drivers.IsValidBucket(bucket) == false {
return []drivers.ObjectMetadata{}, resources, drivers.BucketNameInvalid{Bucket: bucket}
}
if resources.Prefix != "" && drivers.IsValidObject(resources.Prefix) == false {
return []drivers.ObjectMetadata{}, resources, drivers.ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix}
}
rootPrefix := path.Join(file.root, bucket)
// check bucket exists
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
return []drivers.ObjectMetadata{}, resources, drivers.BucketNotFound{Bucket: bucket}
}
p.root = rootPrefix
err := filepath.Walk(rootPrefix, p.getAllFiles)
if err != nil {
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
var metadataList []drivers.ObjectMetadata
var metadata drivers.ObjectMetadata
// Populate filtering mode
resources.Mode = drivers.GetMode(resources)
for name, f := range p.files {
if len(metadataList) >= resources.Maxkeys {
resources.IsTruncated = true
goto ret
}
metadata, resources, err = file.filter(bucket, name, f, resources)
if err != nil {
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
if metadata.Bucket != "" {
metadataList = append(metadataList, metadata)
}
}
ret:
sort.Sort(byObjectKey(metadataList))
return metadataList, resources, nil
}

89
pkg/drivers/file/file_common.go
View File

@@ -1,89 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"bufio"
"bytes"
"os"
"strings"
"sync"
"github.com/minio-io/minio/pkg/drivers"
)
// fileDriver - file local variables
type fileDriver struct {
root string
lock *sync.Mutex
}
// fileMetadata - carries metadata about object
type fileMetadata struct {
Md5sum []byte
ContentType string
}
func appendUniq(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}
type bucketDir struct {
files map[string]os.FileInfo
root string
}
func (p *bucketDir) getAllFiles(object string, fl os.FileInfo, err error) error {
if err != nil {
return err
}
if fl.Mode().IsRegular() {
if strings.HasSuffix(object, "$metadata") {
return nil
}
_p := strings.Split(object, p.root+"/")
if len(_p) > 1 {
p.files[_p[1]] = fl
}
}
return nil
}
func delimiter(object, delimiter string) string {
readBuffer := bytes.NewBufferString(object)
reader := bufio.NewReader(readBuffer)
stringReader := strings.NewReader(delimiter)
delimited, _ := stringReader.ReadByte()
delimitedStr, _ := reader.ReadString(delimited)
return delimitedStr
}
type byObjectKey []drivers.ObjectMetadata
// Len
func (b byObjectKey) Len() int { return len(b) }
// Swap
func (b byObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Less
func (b byObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key }

104
pkg/drivers/file/file_filter.go
View File

@@ -1,104 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"strings"
"github.com/minio-io/minio/pkg/drivers"
)
func (file *fileDriver) filterDelimiterPrefix(bucket, name, fname, delimitedName string, resources drivers.BucketResourcesMetadata) (drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
var err error
var metadata drivers.ObjectMetadata
switch true {
case name == resources.Prefix:
// Use resources.Prefix to filter out delimited files
metadata, err = file.GetObjectMetadata(bucket, name, resources.Prefix)
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName == fname:
// Use resources.Prefix to filter out delimited files
metadata, err = file.GetObjectMetadata(bucket, name, resources.Prefix)
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName != "":
if delimitedName == resources.Delimiter {
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, resources.Prefix+delimitedName)
} else {
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
}
return metadata, resources, nil
}
// TODO handle resources.Marker
func (file *fileDriver) filter(bucket, name string, f os.FileInfo, resources drivers.BucketResourcesMetadata) (drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
var err error
var metadata drivers.ObjectMetadata
switch true {
// Both delimiter and Prefix is present
case resources.IsDelimiterPrefixSet():
if strings.HasPrefix(name, resources.Prefix) {
trimmedName := strings.TrimPrefix(name, resources.Prefix)
delimitedName := delimiter(trimmedName, resources.Delimiter)
metadata, resources, err = file.filterDelimiterPrefix(bucket, name, f.Name(), delimitedName, resources)
if err != nil {
return drivers.ObjectMetadata{}, resources, err
}
}
// Delimiter present and Prefix is absent
case resources.IsDelimiterSet():
delimitedName := delimiter(name, resources.Delimiter)
switch true {
case delimitedName == "":
// Do not strip prefix object output
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName == f.Name():
// Do not strip prefix object output
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName != "":
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
// Delimiter is absent and only Prefix is present
case resources.IsPrefixSet():
if strings.HasPrefix(name, resources.Prefix) {
// Do not strip prefix object output
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
}
case resources.IsDefault():
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
}
return metadata, resources, nil
}

283
pkg/drivers/file/file_object.go
View File

@@ -1,283 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"bytes"
"io"
"os"
"path"
"strings"
"crypto/md5"
"encoding/base64"
"encoding/gob"
"encoding/hex"
"github.com/minio-io/minio/pkg/drivers"
)
/// Object Operations
// GetPartialObject - GET object from range
func (file *fileDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
// validate bucket
if drivers.IsValidBucket(bucket) == false {
return 0, drivers.BucketNameInvalid{Bucket: bucket}
}
// validate object
if drivers.IsValidObject(object) == false {
return 0, drivers.ObjectNameInvalid{Bucket: bucket, Object: object}
}
objectPath := path.Join(file.root, bucket, object)
filestat, err := os.Stat(objectPath)
switch err := err.(type) {
case nil:
{
if filestat.IsDir() {
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
}
default:
{
if os.IsNotExist(err) {
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
return 0, drivers.EmbedError(bucket, object, err)
}
}
f, err := os.Open(objectPath)
defer f.Close()
if err != nil {
return 0, drivers.EmbedError(bucket, object, err)
}
_, err = f.Seek(start, os.SEEK_SET)
if err != nil {
return 0, drivers.EmbedError(bucket, object, err)
}
count, err := io.CopyN(w, f, length)
if err != nil {
return count, drivers.EmbedError(bucket, object, err)
}
return count, nil
}
// GetObject - GET object from key
func (file *fileDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
// validate bucket
if drivers.IsValidBucket(bucket) == false {
return 0, drivers.BucketNameInvalid{Bucket: bucket}
}
// check bucket exists
if _, err := os.Stat(path.Join(file.root, bucket)); os.IsNotExist(err) {
return 0, drivers.BucketNotFound{Bucket: bucket}
}
// validate object
if drivers.IsValidObject(object) == false {
return 0, drivers.ObjectNameInvalid{Bucket: bucket, Object: object}
}
objectPath := path.Join(file.root, bucket, object)
filestat, err := os.Stat(objectPath)
switch err := err.(type) {
case nil:
{
if filestat.IsDir() {
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
}
default:
{
if os.IsNotExist(err) {
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
return 0, drivers.EmbedError(bucket, object, err)
}
}
f, err := os.Open(objectPath)
defer f.Close()
if err != nil {
return 0, drivers.EmbedError(bucket, object, err)
}
count, err := io.Copy(w, f)
if err != nil {
return count, drivers.EmbedError(bucket, object, err)
}
return count, nil
}
// GetObjectMetadata - HEAD object
func (file *fileDriver) GetObjectMetadata(bucket, object, prefix string) (drivers.ObjectMetadata, error) {
if drivers.IsValidBucket(bucket) == false {
return drivers.ObjectMetadata{}, drivers.BucketNameInvalid{Bucket: bucket}
}
if drivers.IsValidObject(object) == false {
return drivers.ObjectMetadata{}, drivers.ObjectNameInvalid{Bucket: bucket, Object: bucket}
}
// check bucket exists
if _, err := os.Stat(path.Join(file.root, bucket)); os.IsNotExist(err) {
return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket}
}
// Do not use path.Join() since path.Join strips off any object names with '/', use them as is
// in a static manner so that we can send a proper 'ObjectNotFound' reply back upon os.Stat()
objectPath := file.root + "/" + bucket + "/" + object
stat, err := os.Stat(objectPath)
if os.IsNotExist(err) {
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
_, err = os.Stat(objectPath + "$metadata")
if os.IsNotExist(err) {
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
f, err := os.Open(objectPath + "$metadata")
defer f.Close()
if err != nil {
return drivers.ObjectMetadata{}, drivers.EmbedError(bucket, object, err)
}
var deserializedMetadata fileMetadata
decoder := gob.NewDecoder(f)
err = decoder.Decode(&deserializedMetadata)
if err != nil {
return drivers.ObjectMetadata{}, drivers.EmbedError(bucket, object, err)
}
contentType := "application/octet-stream"
if deserializedMetadata.ContentType != "" {
contentType = deserializedMetadata.ContentType
}
contentType = strings.TrimSpace(contentType)
etag := bucket + "#" + path.Base(object)
if len(deserializedMetadata.Md5sum) != 0 {
etag = hex.EncodeToString(deserializedMetadata.Md5sum)
}
trimmedObject := strings.TrimPrefix(object, prefix)
metadata := drivers.ObjectMetadata{
Bucket: bucket,
Key: trimmedObject,
Created: stat.ModTime(),
Size: stat.Size(),
Md5: etag,
ContentType: contentType,
}
return metadata, nil
}
// CreateObject - PUT object
func (file *fileDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
// TODO Commits should stage then move instead of writing directly
file.lock.Lock()
defer file.lock.Unlock()
// check bucket name valid
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketNameInvalid{Bucket: bucket}
}
// check bucket exists
if _, err := os.Stat(path.Join(file.root, bucket)); os.IsNotExist(err) {
return drivers.BucketNotFound{Bucket: bucket}
}
// verify object path legal
if drivers.IsValidObject(key) == false {
return drivers.ObjectNameInvalid{Bucket: bucket, Object: key}
}
// verify content type
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
// get object path
objectPath := path.Join(file.root, bucket, key)
objectDir := path.Dir(objectPath)
if _, err := os.Stat(objectDir); os.IsNotExist(err) {
err = os.MkdirAll(objectDir, 0700)
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
}
// check if object exists
if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
return drivers.ObjectExists{
Bucket: bucket,
Object: key,
}
}
// write object
f, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
defer f.Close()
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
h := md5.New()
mw := io.MultiWriter(f, h)
_, err = io.Copy(mw, data)
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
//
f, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
defer f.Close()
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
metadata := &fileMetadata{
ContentType: contentType,
Md5sum: h.Sum(nil),
}
// serialize metadata to gob
encoder := gob.NewEncoder(f)
err = encoder.Encode(metadata)
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
// Verify data received to be correct, Content-MD5 received
if md5sum != "" {
var data []byte
data, err = base64.StdEncoding.DecodeString(md5sum)
if err != nil {
return drivers.InvalidDigest{Bucket: bucket, Key: key, Md5: md5sum}
}
if !bytes.Equal(metadata.Md5sum, data) {
return drivers.BadDigest{Bucket: bucket, Key: key, Md5: md5sum}
}
}
return nil
}

112
pkg/drivers/file/file_policy.go
View File

@@ -1,112 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"path"
"encoding/json"
"github.com/minio-io/minio/pkg/drivers"
)
// GetBucketPolicy - GET bucket policy
func (file *fileDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
file.lock.Lock()
defer file.lock.Unlock()
var p drivers.BucketPolicy
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketPolicy{}, drivers.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(file.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err != nil {
return drivers.BucketPolicy{}, drivers.BucketNotFound{Bucket: bucket}
}
// get policy path
bucketPolicy := path.Join(file.root, bucket+"_policy.json")
filestat, err := os.Stat(bucketPolicy)
if os.IsNotExist(err) {
return drivers.BucketPolicy{}, drivers.BucketPolicyNotFound{Bucket: bucket}
}
if filestat.IsDir() {
return drivers.BucketPolicy{}, drivers.BackendCorrupted{Path: bucketPolicy}
}
f, err := os.OpenFile(bucketPolicy, os.O_RDONLY, 0666)
defer f.Close()
if err != nil {
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err)
}
encoder := json.NewDecoder(f)
err = encoder.Decode(&p)
if err != nil {
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err)
}
return p, nil
}
// CreateBucketPolicy - PUT bucket policy
func (file *fileDriver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error {
file.lock.Lock()
defer file.lock.Unlock()
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(file.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err != nil {
return drivers.BucketNotFound{
Bucket: bucket,
}
}
// get policy path
bucketPolicy := path.Join(file.root, bucket+"_policy.json")
filestat, ret := os.Stat(bucketPolicy)
if !os.IsNotExist(ret) {
if filestat.IsDir() {
return drivers.BackendCorrupted{Path: bucketPolicy}
}
}
f, err := os.OpenFile(bucketPolicy, os.O_WRONLY|os.O_CREATE, 0600)
defer f.Close()
if err != nil {
return drivers.EmbedError(bucket, "", err)
}
encoder := json.NewEncoder(f)
err = encoder.Encode(p)
if err != nil {
return drivers.EmbedError(bucket, "", err)
}
return nil
}

53
pkg/drivers/file/file_test.go
View File

@@ -1,53 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"io/ioutil"
"os"
"testing"
"github.com/minio-io/minio/pkg/drivers"
. "github.com/minio-io/check"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
var storageList []string
create := func() drivers.Driver {
path, err := ioutil.TempDir(os.TempDir(), "minio-file-")
c.Check(err, IsNil)
storageList = append(storageList, path)
_, _, store := Start(path)
return store
}
drivers.APITestSuite(c, create)
removeRoots(c, storageList)
}
func removeRoots(c *C, roots []string) {
for _, root := range roots {
err := os.RemoveAll(root)
c.Check(err, IsNil)
}
}

283
pkg/drivers/memory/memory.go
View File

@@ -1,283 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package memory
import (
"bufio"
"bytes"
"io"
"sort"
"strings"
"sync"
"time"
"crypto/md5"
"encoding/hex"
"io/ioutil"
"github.com/minio-io/minio/pkg/drivers"
)
// memoryDriver - local variables
type memoryDriver struct {
bucketdata map[string]storedBucket
objectdata map[string]storedObject
lock *sync.RWMutex
}
type storedBucket struct {
metadata drivers.BucketMetadata
// owner string // TODO
// id string // TODO
}
type storedObject struct {
metadata drivers.ObjectMetadata
data []byte
}
// Start memory object server
func Start() (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
memory := new(memoryDriver)
memory.bucketdata = make(map[string]storedBucket)
memory.objectdata = make(map[string]storedObject)
memory.lock = new(sync.RWMutex)
go start(ctrlChannel, errorChannel)
return ctrlChannel, errorChannel, memory
}
func start(ctrlChannel <-chan string, errorChannel chan<- error) {
close(errorChannel)
}
// GetObject - GET object from memory buffer
func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
if _, ok := memory.bucketdata[bucket]; ok == false {
return 0, drivers.BucketNotFound{Bucket: bucket}
}
// get object
key := object
if val, ok := memory.objectdata[key]; ok {
objectBuffer := bytes.NewBuffer(val.data)
written, err := io.Copy(w, objectBuffer)
return written, err
}
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
// GetPartialObject - GET object from memory buffer range
func (memory memoryDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
var sourceBuffer bytes.Buffer
if _, err := memory.GetObject(&sourceBuffer, bucket, object); err != nil {
return 0, err
}
if _, err := io.CopyN(ioutil.Discard, &sourceBuffer, start); err != nil {
return 0, err
}
return io.CopyN(w, &sourceBuffer, length)
}
// CreateBucketPolicy - Not implemented
func (memory memoryDriver) CreateBucketPolicy(bucket string, policy drivers.BucketPolicy) error {
return drivers.APINotImplemented{API: "PutBucketPolicy"}
}
// GetBucketPolicy - Not implemented
func (memory memoryDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
return drivers.BucketPolicy{}, drivers.APINotImplemented{API: "GetBucketPolicy"}
}
// CreateObject - PUT object to memory buffer
func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
memory.lock.Lock()
defer memory.lock.Unlock()
if _, ok := memory.bucketdata[bucket]; ok == false {
return drivers.BucketNotFound{Bucket: bucket}
}
if _, ok := memory.objectdata[key]; ok == true {
return drivers.ObjectExists{Bucket: bucket, Object: key}
}
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
var bytesBuffer bytes.Buffer
var newObject = storedObject{}
if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
size := bytesBuffer.Len()
md5SumBytes := md5.Sum(bytesBuffer.Bytes())
md5Sum := hex.EncodeToString(md5SumBytes[:])
newObject.metadata = drivers.ObjectMetadata{
Bucket: bucket,
Key: key,
ContentType: contentType,
Created: time.Now(),
Md5: md5Sum,
Size: int64(size),
}
newObject.data = bytesBuffer.Bytes()
}
memory.objectdata[key] = newObject
return nil
}
// CreateBucket - create bucket in memory
func (memory memoryDriver) CreateBucket(bucketName string) error {
memory.lock.Lock()
defer memory.lock.Unlock()
if !drivers.IsValidBucket(bucketName) {
return drivers.BucketNameInvalid{Bucket: bucketName}
}
if _, ok := memory.bucketdata[bucketName]; ok == true {
return drivers.BucketExists{Bucket: bucketName}
}
var newBucket = storedBucket{}
newBucket.metadata = drivers.BucketMetadata{}
newBucket.metadata.Name = bucketName
newBucket.metadata.Created = time.Now()
memory.bucketdata[bucketName] = newBucket
return nil
}
func delimiter(object, delimiter string) string {
readBuffer := bytes.NewBufferString(object)
reader := bufio.NewReader(readBuffer)
stringReader := strings.NewReader(delimiter)
delimited, _ := stringReader.ReadByte()
delimitedStr, _ := reader.ReadString(delimited)
return delimitedStr
}
func appendUniq(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}
func (memory memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedName string, resources drivers.BucketResourcesMetadata) (drivers.BucketResourcesMetadata, []string) {
switch true {
case key == resources.Prefix:
keys = appendUniq(keys, key)
// DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow
case key == resources.Prefix+delimitedName:
keys = appendUniq(keys, key)
case delimitedName != "":
if delimitedName == resources.Delimiter {
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, resources.Prefix+delimitedName)
} else {
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
}
return resources, keys
}
// ListObjects - list objects from memory
func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
if _, ok := memory.bucketdata[bucket]; ok == false {
return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket}
}
var results []drivers.ObjectMetadata
var keys []string
for key := range memory.objectdata {
switch true {
// Prefix absent, delimit object key based on delimiter
case resources.IsDelimiterSet():
delimitedName := delimiter(key, resources.Delimiter)
switch true {
case delimitedName == "" || delimitedName == key:
keys = appendUniq(keys, key)
case delimitedName != "":
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
// Prefix present, delimit object key with prefix key based on delimiter
case resources.IsDelimiterPrefixSet():
if strings.HasPrefix(key, resources.Prefix) {
trimmedName := strings.TrimPrefix(key, resources.Prefix)
delimitedName := delimiter(trimmedName, resources.Delimiter)
resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
}
// Prefix present, nothing to delimit
case resources.IsPrefixSet():
keys = appendUniq(keys, key)
// Prefix and delimiter absent
case resources.IsDefault():
keys = appendUniq(keys, key)
}
}
sort.Strings(keys)
for _, key := range keys {
if len(results) == resources.Maxkeys {
return results, drivers.BucketResourcesMetadata{IsTruncated: true}, nil
}
object := memory.objectdata[key]
if bucket == object.metadata.Bucket {
results = append(results, object.metadata)
}
}
return results, resources, nil
}
// ByBucketName is a type for sorting bucket metadata by bucket name
type ByBucketName []drivers.BucketMetadata
// Len of bucket name
func (b ByBucketName) Len() int { return len(b) }
// Swap bucket i, j
func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Less
func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
// ListBuckets - List buckets from memory
func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
var results []drivers.BucketMetadata
for _, bucket := range memory.bucketdata {
results = append(results, bucket.metadata)
}
sort.Sort(ByBucketName(results))
return results, nil
}
// GetObjectMetadata - get object metadata from memory
func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (drivers.ObjectMetadata, error) {
// check if bucket exists
if _, ok := memory.bucketdata[bucket]; ok == false {
return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket}
}
if object, ok := memory.objectdata[key]; ok == true {
return object.metadata, nil
}
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key}
}

39
pkg/drivers/memory/memory_test.go
View File

@@ -1,39 +0,0 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package memory
import (
"testing"
"github.com/minio-io/minio/pkg/drivers"
. "github.com/minio-io/check"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
create := func() drivers.Driver {
_, _, store := Start()
return store
}
drivers.APITestSuite(c, create)
}

128
pkg/drivers/mocks/Driver.go
View File

@@ -1,128 +0,0 @@
package mocks
import "github.com/minio-io/minio/pkg/drivers"
import "github.com/stretchr/testify/mock"
import (
"bytes"
"io"
"github.com/minio-io/iodine"
)
// Driver is a mock
type Driver struct {
mock.Mock
ObjectWriterData map[string][]byte
}
// ListBuckets is a mock
func (m *Driver) ListBuckets() ([]drivers.BucketMetadata, error) {
ret := m.Called()
r0 := ret.Get(0).([]drivers.BucketMetadata)
r1 := ret.Error(1)
return r0, r1
}
// CreateBucket is a mock
func (m *Driver) CreateBucket(bucket string) error {
ret := m.Called(bucket)
r0 := ret.Error(0)
return r0
}
// CreateBucketPolicy is a mock
func (m *Driver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error {
ret := m.Called(bucket, p)
r0 := ret.Error(0)
return r0
}
// GetBucketPolicy is a mock
func (m *Driver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
ret := m.Called(bucket)
r0 := ret.Get(0).(drivers.BucketPolicy)
r1 := ret.Error(1)
return r0, r1
}
// SetGetObjectWriter is a mock
func (m *Driver) SetGetObjectWriter(bucket, object string, data []byte) {
m.ObjectWriterData[bucket+":"+object] = data
// println(string(m.ObjectWriterData["bucket:object"]))
}
// GetObject is a mock
func (m *Driver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
ret := m.Called(w, bucket, object)
r0 := ret.Get(0).(int64)
r1 := ret.Error(1)
if r1 == nil {
if obj, ok := m.ObjectWriterData[bucket+":"+object]; ok {
n, _ := io.Copy(w, bytes.NewBuffer(obj))
r0 = n
}
}
return r0, r1
}
// GetPartialObject is a mock
func (m *Driver) GetPartialObject(w io.Writer, bucket string, object string, start int64, length int64) (int64, error) {
ret := m.Called(w, bucket, object, start, length)
r0 := ret.Get(0).(int64)
r1 := ret.Error(1)
if r1 == nil {
if obj, ok := m.ObjectWriterData[bucket+":"+object]; ok {
source := bytes.NewBuffer(obj)
var nilSink bytes.Buffer
io.CopyN(&nilSink, source, start)
n, _ := io.CopyN(w, source, length)
r0 = n
}
}
r1 = iodine.New(r1, nil)
return r0, r1
}
// GetObjectMetadata is a mock
func (m *Driver) GetObjectMetadata(bucket string, object string, prefix string) (drivers.ObjectMetadata, error) {
ret := m.Called(bucket, object, prefix)
r0 := ret.Get(0).(drivers.ObjectMetadata)
r1 := ret.Error(1)
return r0, r1
}
// ListObjects is a mock
func (m *Driver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
ret := m.Called(bucket, resources)
r0 := ret.Get(0).([]drivers.ObjectMetadata)
r1 := ret.Get(1).(drivers.BucketResourcesMetadata)
r2 := ret.Error(2)
return r0, r1, r2
}
// CreateObject is a mock
func (m *Driver) CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error {
ret := m.Called(bucket, key, contentType, md5sum, data)
r0 := ret.Error(0)
return r0
}

8
pkg/server/server.go
View File

@@ -27,12 +27,12 @@ import (
"github.com/minio-io/iodine"
"github.com/minio-io/minio/pkg/api"
"github.com/minio-io/minio/pkg/api/web"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/drivers/donut"
"github.com/minio-io/minio/pkg/drivers/file"
"github.com/minio-io/minio/pkg/drivers/memory"
"github.com/minio-io/minio/pkg/server/httpserver"
"github.com/minio-io/minio/pkg/utils/log"
"github.com/minio-io/objectdriver"
"github.com/minio-io/objectdriver/donut"
"github.com/minio-io/objectdriver/file"
"github.com/minio-io/objectdriver/memory"
)
// Config - http server parameters