MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 04:42:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Code restructuring, assigning proper subsystems to each

Browse Source 
- Drivers contain
    * donut/*
    * file/*
    * memory/*

 - Storage format contains
    * donut/*
      - GetObject() --> renamed to GetObjectReader()
      - Deleted stale objectwriter.go, renamed donutwriter.go to object_writer.go

Simplifying, and documenting codebase further
This commit is contained in:
Harshavardhana
2015-03-23 20:40:21 -07:00
parent 6dd7a9d654
commit 0474439b43
38 changed files with 936 additions and 949 deletions
Download Patch File Download Diff File Expand all files Collapse all files

391
pkg/drivers/api_testsuite.go Normal file
View File

@@ -0,0 +1,391 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
import (
"bytes"
"math/rand"
"strconv"
"gopkg.in/check.v1"
"time"
)
// APITestSuite - collection of API tests
func APITestSuite(c *check.C, create func() Driver) {
testCreateBucket(c, create)
testMultipleObjectCreation(c, create)
testPaging(c, create)
testObjectOverwriteFails(c, create)
testNonExistantBucketOperations(c, create)
testBucketRecreateFails(c, create)
testPutObjectInSubdir(c, create)
testListBuckets(c, create)
testListBucketsOrder(c, create)
testListObjectsTestsForNonExistantBucket(c, create)
testNonExistantObjectInBucket(c, create)
testGetDirectoryReturnsObjectNotFound(c, create)
testDefaultContentType(c, create)
//testContentMd5Set(c, create) TODO
}
func testCreateBucket(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
}
func testMultipleObjectCreation(c *check.C, create func() Driver) {
objects := make(map[string][]byte)
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
for i := 0; i < 10; i++ {
randomPerm := rand.Perm(10)
randomString := ""
for _, num := range randomPerm {
randomString = randomString + strconv.Itoa(num)
}
key := "obj" + strconv.Itoa(i)
objects[key] = []byte(randomString)
err := drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(randomString))
c.Assert(err, check.IsNil)
}
// ensure no duplicate etags
etags := make(map[string]string)
for key, value := range objects {
var byteBuffer bytes.Buffer
_, err := drivers.GetObject(&byteBuffer, "bucket", key)
c.Assert(err, check.IsNil)
c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
metadata, err := drivers.GetObjectMetadata("bucket", key, "")
c.Assert(err, check.IsNil)
c.Assert(metadata.Size, check.Equals, int64(len(value)))
_, ok := etags[metadata.Md5]
c.Assert(ok, check.Equals, false)
etags[metadata.Md5] = metadata.Md5
}
}
func testPaging(c *check.C, create func() Driver) {
drivers := create()
drivers.CreateBucket("bucket")
resources := BucketResourcesMetadata{}
objects, resources, err := drivers.ListObjects("bucket", resources)
c.Assert(err, check.IsNil)
c.Assert(len(objects), check.Equals, 0)
c.Assert(resources.IsTruncated, check.Equals, false)
// check before paging occurs
for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i)
drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
resources.Maxkeys = 5
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(len(objects), check.Equals, i+1)
c.Assert(resources.IsTruncated, check.Equals, false)
c.Assert(err, check.IsNil)
}
// check after paging occurs pages work
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
drivers.CreateObject("bucket", key, "", "", bytes.NewBufferString(key))
resources.Maxkeys = 5
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(len(objects), check.Equals, 5)
c.Assert(resources.IsTruncated, check.Equals, true)
c.Assert(err, check.IsNil)
}
// check paging with prefix at end returns less objects
{
drivers.CreateObject("bucket", "newPrefix", "", "", bytes.NewBufferString("prefix1"))
drivers.CreateObject("bucket", "newPrefix2", "", "", bytes.NewBufferString("prefix2"))
resources.Prefix = "new"
resources.Maxkeys = 5
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(len(objects), check.Equals, 2)
}
// check ordering of pages
{
resources.Prefix = ""
resources.Maxkeys = 1000
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "newPrefix")
c.Assert(objects[1].Key, check.Equals, "newPrefix2")
c.Assert(objects[2].Key, check.Equals, "obj0")
c.Assert(objects[3].Key, check.Equals, "obj1")
c.Assert(objects[4].Key, check.Equals, "obj10")
}
// check delimited results with delimiter and prefix
{
drivers.CreateObject("bucket", "this/is/delimited", "", "", bytes.NewBufferString("prefix1"))
drivers.CreateObject("bucket", "this/is/also/a/delimited/file", "", "", bytes.NewBufferString("prefix2"))
var prefixes []string
resources.CommonPrefixes = prefixes // allocate new everytime
resources.Delimiter = "/"
resources.Prefix = "this/is/"
resources.Maxkeys = 10
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(err, check.IsNil)
c.Assert(len(objects), check.Equals, 1)
c.Assert(resources.CommonPrefixes[0], check.Equals, "also/")
}
time.Sleep(time.Second)
// check delimited results with delimiter without prefix
{
var prefixes []string
resources.CommonPrefixes = prefixes // allocate new everytime
resources.Delimiter = "/"
resources.Prefix = ""
resources.Maxkeys = 1000
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "newPrefix")
c.Assert(objects[1].Key, check.Equals, "newPrefix2")
c.Assert(objects[2].Key, check.Equals, "obj0")
c.Assert(objects[3].Key, check.Equals, "obj1")
c.Assert(objects[4].Key, check.Equals, "obj10")
c.Assert(resources.CommonPrefixes[0], check.Equals, "this/")
}
// check ordering of results with prefix
{
resources.Prefix = "obj"
resources.Delimiter = ""
resources.Maxkeys = 1000
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "obj0")
c.Assert(objects[1].Key, check.Equals, "obj1")
c.Assert(objects[2].Key, check.Equals, "obj10")
c.Assert(objects[3].Key, check.Equals, "obj2")
c.Assert(objects[4].Key, check.Equals, "obj3")
}
// check ordering of results with prefix and no paging
{
resources.Prefix = "new"
resources.Maxkeys = 5
objects, resources, err = drivers.ListObjects("bucket", resources)
c.Assert(objects[0].Key, check.Equals, "newPrefix")
c.Assert(objects[1].Key, check.Equals, "newPrefix2")
}
}
func testObjectOverwriteFails(c *check.C, create func() Driver) {
drivers := create()
drivers.CreateBucket("bucket")
err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
c.Assert(err, check.IsNil)
err = drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("three"))
c.Assert(err, check.Not(check.IsNil))
var bytesBuffer bytes.Buffer
length, err := drivers.GetObject(&bytesBuffer, "bucket", "object")
c.Assert(length, check.Equals, int64(len("one")))
c.Assert(err, check.IsNil)
c.Assert(string(bytesBuffer.Bytes()), check.Equals, "one")
}
func testNonExistantBucketOperations(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateObject("bucket", "object", "", "", bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil))
}
func testBucketRecreateFails(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("string")
c.Assert(err, check.IsNil)
err = drivers.CreateBucket("string")
c.Assert(err, check.Not(check.IsNil))
}
func testPutObjectInSubdir(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil)
var bytesBuffer bytes.Buffer
length, err := drivers.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object")
c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length)
c.Assert(err, check.IsNil)
}
func testListBuckets(c *check.C, create func() Driver) {
drivers := create()
// test empty list
buckets, err := drivers.ListBuckets()
c.Assert(err, check.IsNil)
c.Assert(len(buckets), check.Equals, 0)
// add one and test exists
err = drivers.CreateBucket("bucket1")
c.Assert(err, check.IsNil)
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 1)
c.Assert(err, check.IsNil)
// add two and test exists
err = drivers.CreateBucket("bucket2")
c.Assert(err, check.IsNil)
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 2)
c.Assert(err, check.IsNil)
// add three and test exists + prefix
err = drivers.CreateBucket("bucket22")
buckets, err = drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 3)
c.Assert(err, check.IsNil)
}
func testListBucketsOrder(c *check.C, create func() Driver) {
// if implementation contains a map, order of map keys will vary.
// this ensures they return in the same order each time
for i := 0; i < 10; i++ {
drivers := create()
// add one and test exists
drivers.CreateBucket("bucket1")
drivers.CreateBucket("bucket2")
buckets, err := drivers.ListBuckets()
c.Assert(len(buckets), check.Equals, 2)
c.Assert(err, check.IsNil)
c.Assert(buckets[0].Name, check.Equals, "bucket1")
c.Assert(buckets[1].Name, check.Equals, "bucket2")
}
}
func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Driver) {
drivers := create()
resources := BucketResourcesMetadata{Prefix: "", Maxkeys: 1000}
objects, resources, err := drivers.ListObjects("bucket", resources)
c.Assert(err, check.Not(check.IsNil))
c.Assert(resources.IsTruncated, check.Equals, false)
c.Assert(len(objects), check.Equals, 0)
}
func testNonExistantObjectInBucket(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
length, err := drivers.GetObject(&byteBuffer, "bucket", "dir1")
c.Assert(length, check.Equals, int64(0))
c.Assert(err, check.Not(check.IsNil))
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
switch err := err.(type) {
case ObjectNotFound:
{
c.Assert(err, check.ErrorMatches, "Object not Found: bucket#dir1")
}
default:
{
c.Assert(err, check.Equals, "fails")
}
}
}
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
err = drivers.CreateObject("bucket", "dir1/dir2/object", "", "", bytes.NewBufferString("hello world"))
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
length, err := drivers.GetObject(&byteBuffer, "bucket", "dir1")
c.Assert(length, check.Equals, int64(0))
switch err := err.(type) {
case ObjectNotFound:
{
c.Assert(err.Bucket, check.Equals, "bucket")
c.Assert(err.Object, check.Equals, "dir1")
}
default:
{
// force a failure with a line number
c.Assert(err, check.Equals, "ObjectNotFound")
}
}
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
var byteBuffer2 bytes.Buffer
length, err = drivers.GetObject(&byteBuffer, "bucket", "dir1/")
c.Assert(length, check.Equals, int64(0))
switch err := err.(type) {
case ObjectNotFound:
{
c.Assert(err.Bucket, check.Equals, "bucket")
c.Assert(err.Object, check.Equals, "dir1/")
}
default:
{
// force a failure with a line number
c.Assert(err, check.Equals, "ObjectNotFound")
}
}
c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
}
func testDefaultContentType(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
// test empty
err = drivers.CreateObject("bucket", "one", "", "", bytes.NewBufferString("one"))
metadata, err := drivers.GetObjectMetadata("bucket", "one", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
// test custom
drivers.CreateObject("bucket", "two", "application/text", "", bytes.NewBufferString("two"))
metadata, err = drivers.GetObjectMetadata("bucket", "two", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/text")
// test trim space
drivers.CreateObject("bucket", "three", "\tapplication/json ", "", bytes.NewBufferString("three"))
metadata, err = drivers.GetObjectMetadata("bucket", "three", "")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/json")
}
/*
func testContentMd5Set(c *check.C, create func() Driver) {
drivers := create()
err := drivers.CreateBucket("bucket")
c.Assert(err, check.IsNil)
// test md5 invalid
err = drivers.CreateObject("bucket", "one", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA", bytes.NewBufferString("one"))
c.Assert(err, check.Not(check.IsNil))
err = drivers.CreateObject("bucket", "two", "", "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA=", bytes.NewBufferString("one"))
c.Assert(err, check.IsNil)
}
*/

194
pkg/drivers/bucket_policy.go Normal file
View File

@@ -0,0 +1,194 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
import (
"encoding/json"
"io"
"strings"
)
// User - canonical
type User struct {
AWS string
}
// Statement - minio policy statement
type Statement struct {
Sid string
Effect string
Principal User
Action []string
Resource []string
// add Condition struct/var TODO - fix it in future if necessary
}
// BucketPolicy - minio policy collection
type BucketPolicy struct {
Version string // date in 0000-00-00 format
Statement []Statement
}
// Resource delimiter
const (
MinioResource = "minio:::"
)
// TODO support canonical user
// Principal delimiter
const (
MinioPrincipal = "minio::"
)
// Action map
var SupportedActionMap = map[string]bool{
"*": true,
"minio:GetObject": true,
"minio:ListBucket": true,
"minio:PutObject": true,
"minio:CreateBucket": true,
"minio:GetBucketPolicy": true,
"minio:DeleteBucketPolicy": true,
"minio:ListAllMyBuckets": true,
"minio:PutBucketPolicy": true,
}
// Effect map
var SupportedEffectMap = map[string]bool{
"Allow": true,
"Deny": true,
}
func isValidAction(action []string) bool {
for _, a := range action {
if !SupportedActionMap[a] {
return false
}
}
return true
}
func isValidEffect(effect string) bool {
if SupportedEffectMap[effect] {
return true
}
return false
}
func isValidResource(resources []string) bool {
var ok bool
for _, resource := range resources {
switch true {
case strings.HasPrefix(resource, AwsResource):
bucket := strings.SplitAfter(resource, AwsResource)[1]
ok = true
if len(bucket) == 0 {
ok = false
}
case strings.HasPrefix(resource, MinioResource):
bucket := strings.SplitAfter(resource, MinioResource)[1]
ok = true
if len(bucket) == 0 {
ok = false
}
default:
ok = false
}
}
return ok
}
func isValidPrincipal(principal string) bool {
var ok bool
if principal == "*" {
return true
}
switch true {
case strings.HasPrefix(principal, AwsPrincipal):
username := strings.SplitAfter(principal, AwsPrincipal)[1]
ok = true
if len(username) == 0 {
ok = false
}
case strings.HasPrefix(principal, MinioPrincipal):
username := strings.SplitAfter(principal, MinioPrincipal)[1]
ok = true
if len(username) == 0 {
ok = false
}
default:
ok = false
}
return ok
}
// Parsepolicy - validate request body is proper JSON and in accordance with policy standards
func Parsepolicy(data io.Reader) (BucketPolicy, bool) {
var policy BucketPolicy
decoder := json.NewDecoder(data)
err := decoder.Decode(&policy)
if err != nil {
goto error
}
if len(policy.Version) == 0 {
goto error
}
_, err = parseDate(policy.Version)
if err != nil {
goto error
}
if len(policy.Statement) == 0 {
goto error
}
for _, statement := range policy.Statement {
if len(statement.Sid) == 0 {
goto error
}
if len(statement.Effect) == 0 {
goto error
}
if !isValidEffect(statement.Effect) {
goto error
}
if len(statement.Principal.AWS) == 0 {
goto error
}
if !isValidPrincipal(statement.Principal.AWS) {
goto error
}
if len(statement.Action) == 0 {
goto error
}
if !isValidAction(statement.Action) && !isValidActionS3(statement.Action) {
goto error
}
if len(statement.Resource) == 0 {
goto error
}
if !isValidResource(statement.Resource) {
goto error
}
}
return policy, true
error:
return BucketPolicy{}, false
}

52
pkg/drivers/bucket_policy_compat.go Normal file
View File

@@ -0,0 +1,52 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
// This file implements compatability layer for AWS clients
// Resource delimiter
const (
AwsResource = "arn:aws:s3:::"
)
// TODO support canonical user
// Principal delimiter
const (
AwsPrincipal = "arn:aws:iam::"
)
// Action map
var SupportedActionMapCompat = map[string]bool{
"*": true,
"s3:GetObject": true,
"s3:ListBucket": true,
"s3:PutObject": true,
"s3:CreateBucket": true,
"s3:GetBucketPolicy": true,
"s3:DeleteBucketPolicy": true,
"s3:ListAllMyBuckets": true,
"s3:PutBucketPolicy": true,
}
func isValidActionS3(action []string) bool {
for _, a := range action {
if !SupportedActionMapCompat[a] {
return false
}
}
return true
}

78
pkg/drivers/date.go Normal file
View File

@@ -0,0 +1,78 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
import (
"errors"
"fmt"
"strconv"
"strings"
)
// Date - [0000-00-00]
type Date struct {
Year int16
Month byte
Day byte
}
// String output in yyyy-mm-dd format
func (d Date) String() string {
return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day)
}
// IsZero true if date is 0000-00-00
func (d Date) IsZero() bool {
return d.Day == 0 && d.Month == 0 && d.Year == 0
}
// Convert string date in format YYYY-MM-DD to Date.
// Leading and trailing spaces are ignored. If format is invalid returns zero.
func parseDate(str string) (d Date, err error) {
str = strings.TrimSpace(str)
if str == "0000-00-00" {
return
}
var (
y, m, n int
)
if len(str) != 10 || str[4] != '-' || str[7] != '-' {
err = errors.New("Invalid 0000-00-000 style DATE string: " + str)
return
}
if y, err = strconv.Atoi(str[0:4]); err != nil {
return
}
if m, err = strconv.Atoi(str[5:7]); err != nil {
return
}
if m < 1 || m > 12 {
err = errors.New("Invalid 0000-00-000 style DATE string: " + str)
return
}
if n, err = strconv.Atoi(str[8:10]); err != nil {
return
}
if n < 1 || n > 31 {
err = errors.New("Invalid 0000-00-000 style DATE string: " + str)
return
}
d.Year = int16(y)
d.Month = byte(m)
d.Day = byte(n)
return
}

260
pkg/drivers/donut/donut.go Normal file
View File

@@ -0,0 +1,260 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package donut
import (
"errors"
"io"
"sort"
"strconv"
"strings"
"time"
"github.com/minio-io/minio/pkg/drivers"
"github.com/minio-io/minio/pkg/storage/donut"
)
// donutDriver - creates a new single disk drivers driver using donut
type donutDriver struct {
donut donut.Donut
}
const (
blockSize = 10 * 1024 * 1024
)
// Start a single disk subsystem
func Start(path string) (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
s := new(donutDriver)
// TODO donut driver should be passed in as Start param and driven by config
s.donut = donut.NewDonut(path)
go start(ctrlChannel, errorChannel, s)
return ctrlChannel, errorChannel, s
}
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *donutDriver) {
close(errorChannel)
}
// ListBuckets returns a list of buckets
func (d donutDriver) ListBuckets() (results []drivers.BucketMetadata, err error) {
buckets, err := d.donut.ListBuckets()
if err != nil {
return nil, err
}
for _, bucket := range buckets {
if err != nil {
return nil, err
}
result := drivers.BucketMetadata{
Name: bucket,
// TODO Add real created date
Created: time.Now(),
}
results = append(results, result)
}
return results, nil
}
// CreateBucket creates a new bucket
func (d donutDriver) CreateBucket(bucket string) error {
return d.donut.CreateBucket(bucket)
}
// GetBucketMetadata retrieves an bucket's metadata
func (d donutDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
return drivers.BucketMetadata{}, errors.New("Not Implemented")
}
// CreateBucketPolicy sets a bucket's access policy
func (d donutDriver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error {
return errors.New("Not Implemented")
}
// GetBucketPolicy returns a bucket's access policy
func (d donutDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
return drivers.BucketPolicy{}, errors.New("Not Implemented")
}
// GetObject retrieves an object and writes it to a writer
func (d donutDriver) GetObject(target io.Writer, bucket, key string) (int64, error) {
reader, err := d.donut.GetObjectReader(bucket, key)
if err != nil {
return 0, drivers.ObjectNotFound{
Bucket: bucket,
Object: key,
}
}
return io.Copy(target, reader)
}
// GetPartialObject retrieves an object range and writes it to a writer
func (d donutDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
return 0, errors.New("Not Implemented")
}
// GetObjectMetadata retrieves an object's metadata
func (d donutDriver) GetObjectMetadata(bucket, key string, prefix string) (drivers.ObjectMetadata, error) {
metadata, err := d.donut.GetObjectMetadata(bucket, key)
created, err := time.Parse(time.RFC3339Nano, metadata["sys.created"])
if err != nil {
return drivers.ObjectMetadata{}, err
}
size, err := strconv.ParseInt(metadata["sys.size"], 10, 64)
if err != nil {
return drivers.ObjectMetadata{}, err
}
objectMetadata := drivers.ObjectMetadata{
Bucket: bucket,
Key: key,
ContentType: metadata["contentType"],
Created: created,
Md5: metadata["sys.md5"],
Size: size,
}
return objectMetadata, nil
}
// ListObjects - returns list of objects
func (d donutDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
// TODO Fix IsPrefixSet && IsDelimiterSet and use them
objects, err := d.donut.ListObjects(bucket)
if err != nil {
return nil, drivers.BucketResourcesMetadata{}, err
}
sort.Strings(objects)
if resources.Prefix != "" {
objects = filterPrefix(objects, resources.Prefix)
objects = removePrefix(objects, resources.Prefix)
}
if resources.Maxkeys <= 0 || resources.Maxkeys > 1000 {
resources.Maxkeys = 1000
}
var actualObjects []string
var commonPrefixes []string
if strings.TrimSpace(resources.Delimiter) != "" {
actualObjects = filterDelimited(objects, resources.Delimiter)
commonPrefixes = filterNotDelimited(objects, resources.Delimiter)
commonPrefixes = extractDir(commonPrefixes, resources.Delimiter)
commonPrefixes = uniqueObjects(commonPrefixes)
resources.CommonPrefixes = commonPrefixes
} else {
actualObjects = objects
}
var results []drivers.ObjectMetadata
for _, object := range actualObjects {
if len(results) >= resources.Maxkeys {
resources.IsTruncated = true
break
}
metadata, err := d.GetObjectMetadata(bucket, resources.Prefix+object, "")
if err != nil {
return nil, drivers.BucketResourcesMetadata{}, err
}
results = append(results, metadata)
}
return results, resources, nil
}
func filterPrefix(objects []string, prefix string) []string {
var results []string
for _, object := range objects {
if strings.HasPrefix(object, prefix) {
results = append(results, object)
}
}
return results
}
func removePrefix(objects []string, prefix string) []string {
var results []string
for _, object := range objects {
results = append(results, strings.TrimPrefix(object, prefix))
}
return results
}
func filterDelimited(objects []string, delim string) []string {
var results []string
for _, object := range objects {
if !strings.Contains(object, delim) {
results = append(results, object)
}
}
return results
}
func filterNotDelimited(objects []string, delim string) []string {
var results []string
for _, object := range objects {
if strings.Contains(object, delim) {
results = append(results, object)
}
}
return results
}
func extractDir(objects []string, delim string) []string {
var results []string
for _, object := range objects {
parts := strings.Split(object, delim)
results = append(results, parts[0]+"/")
}
return results
}
func uniqueObjects(objects []string) []string {
objectMap := make(map[string]string)
for _, v := range objects {
objectMap[v] = v
}
var results []string
for k := range objectMap {
results = append(results, k)
}
sort.Strings(results)
return results
}
// CreateObject creates a new object
func (d donutDriver) CreateObject(bucketKey, objectKey, contentType, expectedMd5sum string, reader io.Reader) error {
writer, err := d.donut.GetObjectWriter(bucketKey, objectKey)
if err != nil {
return err
}
if _, err := io.Copy(writer, reader); err != nil {
return err
}
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
metadata := make(map[string]string)
metadata["bucket"] = bucketKey
metadata["object"] = objectKey
metadata["contentType"] = contentType
if err = writer.SetMetadata(metadata); err != nil {
return err
}
return writer.Close()
}

54
pkg/drivers/donut/donut_test.go Normal file
View File

@@ -0,0 +1,54 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package donut
import (
"io/ioutil"
"os"
"testing"
"github.com/minio-io/minio/pkg/drivers"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
// c.Skip("Not Implemented")
var storageList []string
create := func() drivers.Driver {
path, err := ioutil.TempDir(os.TempDir(), "minio-fs-")
c.Check(err, IsNil)
storageList = append(storageList, path)
_, _, store := Start(path) // TODO Make InMemory driver
return store
}
drivers.APITestSuite(c, create)
removeRoots(c, storageList)
}
func removeRoots(c *C, roots []string) {
for _, root := range roots {
err := os.RemoveAll(root)
c.Check(err, IsNil)
}
}

150
pkg/drivers/driver.go Normal file
View File

@@ -0,0 +1,150 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
import (
"io"
"regexp"
"time"
"unicode/utf8"
)
// Driver - generic API interface for various drivers - donut, file, memory
type Driver interface {
// Bucket Operations
ListBuckets() ([]BucketMetadata, error)
CreateBucket(bucket string) error
CreateBucketPolicy(bucket string, p BucketPolicy) error
GetBucketPolicy(bucket string) (BucketPolicy, error)
// Object Operations
GetObject(w io.Writer, bucket, object string) (int64, error)
GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error)
GetObjectMetadata(bucket string, object string, prefix string) (ObjectMetadata, error)
ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, error)
CreateObject(bucket string, key string, contentType string, md5sum string, data io.Reader) error
}
// BucketMetadata - name and create date
type BucketMetadata struct {
Name string
Created time.Time
}
// ObjectMetadata - object key and its relevant metadata
type ObjectMetadata struct {
Bucket string
Key string
ContentType string
Created time.Time
Md5 string
Size int64
}
// FilterMode type
type FilterMode int
// FilterMode list
const (
DelimiterPrefixMode FilterMode = iota
DelimiterMode
PrefixMode
DefaultMode
)
// BucketResourcesMetadata - various types of bucket resources
type BucketResourcesMetadata struct {
Prefix string
Marker string
Maxkeys int
Delimiter string
IsTruncated bool
CommonPrefixes []string
Mode FilterMode
Policy bool
// TODO
Logging string
Notification string
}
// GetMode - Populate filter mode
func GetMode(resources BucketResourcesMetadata) FilterMode {
var f FilterMode
switch true {
case resources.Delimiter != "" && resources.Prefix != "":
f = DelimiterPrefixMode
case resources.Delimiter != "" && resources.Prefix == "":
f = DelimiterMode
case resources.Delimiter == "" && resources.Prefix != "":
f = PrefixMode
case resources.Delimiter == "" && resources.Prefix == "":
f = DefaultMode
}
return f
}
// IsDelimiterPrefixSet Delimiter and Prefix set
func (b BucketResourcesMetadata) IsDelimiterPrefixSet() bool {
return b.Mode == DelimiterPrefixMode
}
// IsDelimiterSet Delimiter set
func (b BucketResourcesMetadata) IsDelimiterSet() bool {
return b.Mode == DelimiterMode
}
// IsPrefixSet Prefix set
func (b BucketResourcesMetadata) IsPrefixSet() bool {
return b.Mode == PrefixMode
}
// IsDefault No query values
func (b BucketResourcesMetadata) IsDefault() bool {
return b.Mode == DefaultMode
}
// IsValidBucket - verify bucket name in accordance with
// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
func IsValidBucket(bucket string) bool {
if len(bucket) < 3 || len(bucket) > 63 {
return false
}
if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
return false
}
if match, _ := regexp.MatchString("\\.\\.", bucket); match == true {
return false
}
// We don't support buckets with '.' in them
match, _ := regexp.MatchString("^[a-zA-Z][a-zA-Z0-9\\-]+[a-zA-Z0-9]$", bucket)
return match
}
// IsValidObject - verify object name in accordance with
// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
func IsValidObject(object string) bool {
if len(object) > 1024 || len(object) == 0 {
return false
}
if !utf8.ValidString(object) {
return false
}
return true
}

163
pkg/drivers/errors.go Normal file
View File

@@ -0,0 +1,163 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package drivers
// BackendError - generic disk backend error
type BackendError struct {
Path string
}
// BackendCorrupted - path has corrupted data
type BackendCorrupted BackendError
// APINotImplemented - generic API not implemented error
type APINotImplemented struct {
API string
}
// GenericBucketError - generic bucket error
type GenericBucketError struct {
Bucket string
}
// GenericObjectError - generic object error
type GenericObjectError struct {
Bucket string
Object string
}
// ImplementationError - generic implementation error
type ImplementationError struct {
Bucket string
Object string
Err error
}
// DigestError - Generic Md5 error
type DigestError struct {
Bucket string
Key string
Md5 string
}
/// Bucket related errors
// BucketPolicyNotFound - missing bucket policy
type BucketPolicyNotFound GenericBucketError
// BucketNameInvalid - bucketname provided is invalid
type BucketNameInvalid GenericBucketError
// BucketExists - bucket already exists
type BucketExists GenericBucketError
// BucketNotFound - requested bucket not found
type BucketNotFound GenericBucketError
/// Object related errors
// ObjectNotFound - requested object not found
type ObjectNotFound GenericObjectError
// ObjectExists - object already exists
type ObjectExists GenericObjectError
// ObjectNameInvalid - object name provided is invalid
type ObjectNameInvalid GenericObjectError
// BadDigest - md5 mismatch from data received
type BadDigest DigestError
// InvalidDigest - md5 in request header invalid
type InvalidDigest DigestError
// Return string an error formatted as the given text
func (e ImplementationError) Error() string {
error := ""
if e.Bucket != "" {
error = error + "Bucket: " + e.Bucket + " "
}
if e.Object != "" {
error = error + "Object: " + e.Object + " "
}
error = error + "Error: " + e.Err.Error()
return error
}
// EmbedError - wrapper function for error object
func EmbedError(bucket, object string, err error) ImplementationError {
return ImplementationError{
Bucket: bucket,
Object: object,
Err: err,
}
}
// Return string an error formatted as the given text
func (e BucketPolicyNotFound) Error() string {
return "Bucket policy not found for: " + e.Bucket
}
// Return string an error formatted as the given text
func (e ObjectNotFound) Error() string {
return "Object not Found: " + e.Bucket + "#" + e.Object
}
// Return string an error formatted as the given text
func (e APINotImplemented) Error() string {
return "Api not implemented: " + e.API
}
// Return string an error formatted as the given text
func (e ObjectExists) Error() string {
return "Object exists: " + e.Bucket + "#" + e.Object
}
// Return string an error formatted as the given text
func (e BucketNameInvalid) Error() string {
return "Bucket name invalid: " + e.Bucket
}
// Return string an error formatted as the given text
func (e BucketExists) Error() string {
return "Bucket exists: " + e.Bucket
}
// Return string an error formatted as the given text
func (e BucketNotFound) Error() string {
return "Bucket not Found: " + e.Bucket
}
// Return string an error formatted as the given text
func (e ObjectNameInvalid) Error() string {
return "Object name invalid: " + e.Bucket + "#" + e.Object
}
// Return string an error formatted as the given text
func (e BackendCorrupted) Error() string {
return "Backend corrupted: " + e.Path
}
// Return string an error formatted as the given text
func (e BadDigest) Error() string {
return "Md5 provided " + e.Md5 + " mismatches for: " + e.Bucket + "#" + e.Key
}
// Return string an error formatted as the given text
func (e InvalidDigest) Error() string {
return "Md5 provided " + e.Md5 + " is invalid"
}

41
pkg/drivers/file/file.go Normal file
View File

@@ -0,0 +1,41 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"sync"
"github.com/minio-io/minio/pkg/drivers"
)
// Start filesystem channel
func Start(root string) (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
s := new(fileDriver)
s.root = root
s.lock = new(sync.Mutex)
go start(ctrlChannel, errorChannel, s)
return ctrlChannel, errorChannel, s
}
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *fileDriver) {
err := os.MkdirAll(s.root, 0700)
errorChannel <- err
close(errorChannel)
}

133
pkg/drivers/file/file_bucket.go Normal file
View File

@@ -0,0 +1,133 @@
/*
* Minimalist Object File, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"path"
"sort"
"strings"
"io/ioutil"
"path/filepath"
"github.com/minio-io/minio/pkg/drivers"
)
/// Bucket Operations
// ListBuckets - Get service
func (file *fileDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
files, err := ioutil.ReadDir(file.root)
if err != nil {
return []drivers.BucketMetadata{}, drivers.EmbedError("bucket", "", err)
}
var metadataList []drivers.BucketMetadata
for _, fileName := range files {
// Skip policy files
if strings.HasSuffix(fileName.Name(), "_policy.json") {
continue
}
if !fileName.IsDir() {
return []drivers.BucketMetadata{}, drivers.BackendCorrupted{Path: file.root}
}
metadata := drivers.BucketMetadata{
Name: fileName.Name(),
Created: fileName.ModTime(), // TODO - provide real created time
}
metadataList = append(metadataList, metadata)
}
return metadataList, nil
}
// CreateBucket - PUT Bucket
func (file *fileDriver) CreateBucket(bucket string) error {
file.lock.Lock()
defer file.lock.Unlock()
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(file.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err == nil {
return drivers.BucketExists{
Bucket: bucket,
}
}
// make bucket
err := os.Mkdir(bucketDir, 0700)
if err != nil {
return drivers.EmbedError(bucket, "", err)
}
return nil
}
// ListObjects - GET bucket (list objects)
func (file *fileDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
p := bucketDir{}
p.files = make(map[string]os.FileInfo)
if drivers.IsValidBucket(bucket) == false {
return []drivers.ObjectMetadata{}, resources, drivers.BucketNameInvalid{Bucket: bucket}
}
if resources.Prefix != "" && drivers.IsValidObject(resources.Prefix) == false {
return []drivers.ObjectMetadata{}, resources, drivers.ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix}
}
rootPrefix := path.Join(file.root, bucket)
// check bucket exists
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
return []drivers.ObjectMetadata{}, resources, drivers.BucketNotFound{Bucket: bucket}
}
p.root = rootPrefix
err := filepath.Walk(rootPrefix, p.getAllFiles)
if err != nil {
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
var metadataList []drivers.ObjectMetadata
var metadata drivers.ObjectMetadata
// Populate filtering mode
resources.Mode = drivers.GetMode(resources)
for name, f := range p.files {
if len(metadataList) >= resources.Maxkeys {
resources.IsTruncated = true
goto ret
}
metadata, resources, err = file.filter(bucket, name, f, resources)
if err != nil {
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
if metadata.Bucket != "" {
metadataList = append(metadataList, metadata)
}
}
ret:
sort.Sort(byObjectKey(metadataList))
return metadataList, resources, nil
}

89
pkg/drivers/file/file_common.go Normal file
View File

@@ -0,0 +1,89 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"bufio"
"bytes"
"os"
"strings"
"sync"
"github.com/minio-io/minio/pkg/drivers"
)
// fileDriver - file local variables
type fileDriver struct {
root string
lock *sync.Mutex
}
// fileMetadata - carries metadata about object
type fileMetadata struct {
Md5sum []byte
ContentType string
}
func appendUniq(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}
type bucketDir struct {
files map[string]os.FileInfo
root string
}
func (p *bucketDir) getAllFiles(object string, fl os.FileInfo, err error) error {
if err != nil {
return err
}
if fl.Mode().IsRegular() {
if strings.HasSuffix(object, "$metadata") {
return nil
}
_p := strings.Split(object, p.root+"/")
if len(_p) > 1 {
p.files[_p[1]] = fl
}
}
return nil
}
func delimiter(object, delimiter string) string {
readBuffer := bytes.NewBufferString(object)
reader := bufio.NewReader(readBuffer)
stringReader := strings.NewReader(delimiter)
delimited, _ := stringReader.ReadByte()
delimitedStr, _ := reader.ReadString(delimited)
return delimitedStr
}
type byObjectKey []drivers.ObjectMetadata
// Len
func (b byObjectKey) Len() int { return len(b) }
// Swap
func (b byObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Less
func (b byObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key }

94
pkg/drivers/file/file_filter.go Normal file
View File

@@ -0,0 +1,94 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"strings"
"github.com/minio-io/minio/pkg/drivers"
)
// TODO handle resources.Marker
func (file *fileDriver) filter(bucket, name string, f os.FileInfo, resources drivers.BucketResourcesMetadata) (drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
var err error
var metadata drivers.ObjectMetadata
switch true {
// Both delimiter and Prefix is present
case resources.IsDelimiterPrefixSet():
if strings.HasPrefix(name, resources.Prefix) {
trimmedName := strings.TrimPrefix(name, resources.Prefix)
delimitedName := delimiter(trimmedName, resources.Delimiter)
switch true {
case name == resources.Prefix:
// Use resources.Prefix to filter out delimited files
metadata, err = file.GetObjectMetadata(bucket, name, resources.Prefix)
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName == f.Name():
// Use resources.Prefix to filter out delimited files
metadata, err = file.GetObjectMetadata(bucket, name, resources.Prefix)
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName != "":
if delimitedName == resources.Delimiter {
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, resources.Prefix+delimitedName)
} else {
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
}
}
// Delimiter present and Prefix is absent
case resources.IsDelimiterSet():
delimitedName := delimiter(name, resources.Delimiter)
switch true {
case delimitedName == "":
// Do not strip prefix object output
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName == f.Name():
// Do not strip prefix object output
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
case delimitedName != "":
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
// Delimiter is absent and only Prefix is present
case resources.IsPrefixSet():
if strings.HasPrefix(name, resources.Prefix) {
// Do not strip prefix object output
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
}
case resources.IsDefault():
metadata, err = file.GetObjectMetadata(bucket, name, "")
if err != nil {
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
}
}
return metadata, resources, nil
}

276
pkg/drivers/file/file_object.go Normal file
View File

@@ -0,0 +1,276 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"bytes"
"io"
"os"
"path"
"strings"
"crypto/md5"
"encoding/base64"
"encoding/gob"
"encoding/hex"
"github.com/minio-io/minio/pkg/drivers"
)
/// Object Operations
// GetPartialObject - GET object from range
func (file *fileDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
// validate bucket
if drivers.IsValidBucket(bucket) == false {
return 0, drivers.BucketNameInvalid{Bucket: bucket}
}
// validate object
if drivers.IsValidObject(object) == false {
return 0, drivers.ObjectNameInvalid{Bucket: bucket, Object: object}
}
objectPath := path.Join(file.root, bucket, object)
filestat, err := os.Stat(objectPath)
switch err := err.(type) {
case nil:
{
if filestat.IsDir() {
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
}
default:
{
if os.IsNotExist(err) {
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
return 0, drivers.EmbedError(bucket, object, err)
}
}
f, err := os.Open(objectPath)
defer f.Close()
if err != nil {
return 0, drivers.EmbedError(bucket, object, err)
}
_, err = f.Seek(start, os.SEEK_SET)
if err != nil {
return 0, drivers.EmbedError(bucket, object, err)
}
count, err := io.CopyN(w, f, length)
if err != nil {
return count, drivers.EmbedError(bucket, object, err)
}
return count, nil
}
// GetObject - GET object from key
func (file *fileDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
// validate bucket
if drivers.IsValidBucket(bucket) == false {
return 0, drivers.BucketNameInvalid{Bucket: bucket}
}
// validate object
if drivers.IsValidObject(object) == false {
return 0, drivers.ObjectNameInvalid{Bucket: bucket, Object: object}
}
objectPath := path.Join(file.root, bucket, object)
filestat, err := os.Stat(objectPath)
switch err := err.(type) {
case nil:
{
if filestat.IsDir() {
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
}
default:
{
if os.IsNotExist(err) {
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
return 0, drivers.EmbedError(bucket, object, err)
}
}
f, err := os.Open(objectPath)
defer f.Close()
if err != nil {
return 0, drivers.EmbedError(bucket, object, err)
}
count, err := io.Copy(w, f)
if err != nil {
return count, drivers.EmbedError(bucket, object, err)
}
return count, nil
}
// GetObjectMetadata - HEAD object
func (file *fileDriver) GetObjectMetadata(bucket, object, prefix string) (drivers.ObjectMetadata, error) {
if drivers.IsValidBucket(bucket) == false {
return drivers.ObjectMetadata{}, drivers.BucketNameInvalid{Bucket: bucket}
}
if drivers.IsValidObject(object) == false {
return drivers.ObjectMetadata{}, drivers.ObjectNameInvalid{Bucket: bucket, Object: bucket}
}
// Do not use path.Join() since path.Join strips off any object names with '/', use them as is
// in a static manner so that we can send a proper 'ObjectNotFound' reply back upon os.Stat()
objectPath := file.root + "/" + bucket + "/" + object
stat, err := os.Stat(objectPath)
if os.IsNotExist(err) {
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
_, err = os.Stat(objectPath + "$metadata")
if os.IsNotExist(err) {
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
f, err := os.Open(objectPath + "$metadata")
defer f.Close()
if err != nil {
return drivers.ObjectMetadata{}, drivers.EmbedError(bucket, object, err)
}
var deserializedMetadata fileMetadata
decoder := gob.NewDecoder(f)
err = decoder.Decode(&deserializedMetadata)
if err != nil {
return drivers.ObjectMetadata{}, drivers.EmbedError(bucket, object, err)
}
contentType := "application/octet-stream"
if deserializedMetadata.ContentType != "" {
contentType = deserializedMetadata.ContentType
}
contentType = strings.TrimSpace(contentType)
etag := bucket + "#" + path.Base(object)
if len(deserializedMetadata.Md5sum) != 0 {
etag = hex.EncodeToString(deserializedMetadata.Md5sum)
}
trimmedObject := strings.TrimPrefix(object, prefix)
metadata := drivers.ObjectMetadata{
Bucket: bucket,
Key: trimmedObject,
Created: stat.ModTime(),
Size: stat.Size(),
Md5: etag,
ContentType: contentType,
}
return metadata, nil
}
// CreateObject - PUT object
func (file *fileDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
// TODO Commits should stage then move instead of writing directly
file.lock.Lock()
defer file.lock.Unlock()
// check bucket name valid
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketNameInvalid{Bucket: bucket}
}
// check bucket exists
if _, err := os.Stat(path.Join(file.root, bucket)); os.IsNotExist(err) {
return drivers.BucketNotFound{Bucket: bucket}
}
// verify object path legal
if drivers.IsValidObject(key) == false {
return drivers.ObjectNameInvalid{Bucket: bucket, Object: key}
}
// verify content type
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
// get object path
objectPath := path.Join(file.root, bucket, key)
objectDir := path.Dir(objectPath)
if _, err := os.Stat(objectDir); os.IsNotExist(err) {
err = os.MkdirAll(objectDir, 0700)
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
}
// check if object exists
if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
return drivers.ObjectExists{
Bucket: bucket,
Object: key,
}
}
// write object
f, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
defer f.Close()
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
h := md5.New()
mw := io.MultiWriter(f, h)
_, err = io.Copy(mw, data)
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
//
f, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
defer f.Close()
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
metadata := &fileMetadata{
ContentType: contentType,
Md5sum: h.Sum(nil),
}
// serialize metadata to gob
encoder := gob.NewEncoder(f)
err = encoder.Encode(metadata)
if err != nil {
return drivers.EmbedError(bucket, key, err)
}
// Verify data received to be correct, Content-MD5 received
if md5sum != "" {
var data []byte
data, err = base64.StdEncoding.DecodeString(md5sum)
if err != nil {
return drivers.InvalidDigest{Bucket: bucket, Key: key, Md5: md5sum}
}
if !bytes.Equal(metadata.Md5sum, data) {
return drivers.BadDigest{Bucket: bucket, Key: key, Md5: md5sum}
}
}
return nil
}

111
pkg/drivers/file/file_policy.go Normal file
View File

@@ -0,0 +1,111 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"os"
"path"
"encoding/json"
"github.com/minio-io/minio/pkg/drivers"
)
// GetBucketPolicy - GET bucket policy
func (file *fileDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
file.lock.Lock()
defer file.lock.Unlock()
var p drivers.BucketPolicy
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketPolicy{}, drivers.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(file.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err != nil {
return drivers.BucketPolicy{}, drivers.BucketNotFound{Bucket: bucket}
}
// get policy path
bucketPolicy := path.Join(file.root, bucket+"_policy.json")
filestat, err := os.Stat(bucketPolicy)
if os.IsNotExist(err) {
return drivers.BucketPolicy{}, drivers.BucketPolicyNotFound{Bucket: bucket}
}
if filestat.IsDir() {
return drivers.BucketPolicy{}, drivers.BackendCorrupted{Path: bucketPolicy}
}
f, err := os.OpenFile(bucketPolicy, os.O_RDONLY, 0666)
defer f.Close()
if err != nil {
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err)
}
encoder := json.NewDecoder(f)
err = encoder.Decode(&p)
if err != nil {
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err)
}
return p, nil
}
// CreateBucketPolicy - PUT bucket policy
func (file *fileDriver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error {
file.lock.Lock()
defer file.lock.Unlock()
// verify bucket path legal
if drivers.IsValidBucket(bucket) == false {
return drivers.BucketNameInvalid{Bucket: bucket}
}
// get bucket path
bucketDir := path.Join(file.root, bucket)
// check if bucket exists
if _, err := os.Stat(bucketDir); err != nil {
return drivers.BucketNotFound{
Bucket: bucket,
}
}
// get policy path
bucketPolicy := path.Join(file.root, bucket+"_policy.json")
filestat, ret := os.Stat(bucketPolicy)
if !os.IsNotExist(ret) {
if filestat.IsDir() {
return drivers.BackendCorrupted{Path: bucketPolicy}
}
}
f, err := os.OpenFile(bucketPolicy, os.O_WRONLY|os.O_CREATE, 0600)
defer f.Close()
if err != nil {
return drivers.EmbedError(bucket, "", err)
}
encoder := json.NewEncoder(f)
err = encoder.Encode(p)
if err != nil {
return drivers.EmbedError(bucket, "", err)
}
return nil
}

53
pkg/drivers/file/file_test.go Normal file
View File

@@ -0,0 +1,53 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package file
import (
"io/ioutil"
"os"
"testing"
"github.com/minio-io/minio/pkg/drivers"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
var storageList []string
create := func() drivers.Driver {
path, err := ioutil.TempDir(os.TempDir(), "minio-file-")
c.Check(err, IsNil)
storageList = append(storageList, path)
_, _, store := Start(path)
return store
}
drivers.APITestSuite(c, create)
removeRoots(c, storageList)
}
func removeRoots(c *C, roots []string) {
for _, root := range roots {
err := os.RemoveAll(root)
c.Check(err, IsNil)
}
}

261
pkg/drivers/memory/memory.go Normal file
View File

@@ -0,0 +1,261 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package memory
import (
"bufio"
"bytes"
"io"
"sort"
"strings"
"sync"
"time"
"crypto/md5"
"encoding/hex"
"github.com/minio-io/minio/pkg/drivers"
)
// memoryDriver - local variables
type memoryDriver struct {
bucketdata map[string]storedBucket
objectdata map[string]storedObject
lock *sync.RWMutex
}
type storedBucket struct {
metadata drivers.BucketMetadata
// owner string // TODO
// id string // TODO
}
type storedObject struct {
metadata drivers.ObjectMetadata
data []byte
}
// Start memory object server
func Start() (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
memory := new(memoryDriver)
memory.bucketdata = make(map[string]storedBucket)
memory.objectdata = make(map[string]storedObject)
memory.lock = new(sync.RWMutex)
go start(ctrlChannel, errorChannel)
return ctrlChannel, errorChannel, memory
}
func start(ctrlChannel <-chan string, errorChannel chan<- error) {
close(errorChannel)
}
// GetObject - GET object from memory buffer
func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
// get object
key := object
if val, ok := memory.objectdata[key]; ok {
objectBuffer := bytes.NewBuffer(val.data)
written, err := io.Copy(w, objectBuffer)
return written, err
}
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
// GetPartialObject - GET object from memory buffer range
func (memory memoryDriver) GetPartialObject(w io.Writer, bucket, object string, start, end int64) (int64, error) {
return 0, drivers.APINotImplemented{API: "GetPartialObject"}
}
// CreateBucketPolicy - Not implemented
func (memory memoryDriver) CreateBucketPolicy(bucket string, policy drivers.BucketPolicy) error {
return drivers.APINotImplemented{API: "PutBucketPolicy"}
}
// GetBucketPolicy - Not implemented
func (memory memoryDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
return drivers.BucketPolicy{}, drivers.APINotImplemented{API: "GetBucketPolicy"}
}
// CreateObject - PUT object to memory buffer
func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
memory.lock.Lock()
defer memory.lock.Unlock()
if _, ok := memory.bucketdata[bucket]; ok == false {
return drivers.BucketNotFound{Bucket: bucket}
}
if _, ok := memory.objectdata[key]; ok == true {
return drivers.ObjectExists{Bucket: bucket, Object: key}
}
if contentType == "" {
contentType = "application/octet-stream"
}
contentType = strings.TrimSpace(contentType)
var bytesBuffer bytes.Buffer
var newObject = storedObject{}
if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
size := bytesBuffer.Len()
md5SumBytes := md5.Sum(bytesBuffer.Bytes())
md5Sum := hex.EncodeToString(md5SumBytes[:])
newObject.metadata = drivers.ObjectMetadata{
Bucket: bucket,
Key: key,
ContentType: contentType,
Created: time.Now(),
Md5: md5Sum,
Size: int64(size),
}
newObject.data = bytesBuffer.Bytes()
}
memory.objectdata[key] = newObject
return nil
}
// CreateBucket - create bucket in memory
func (memory memoryDriver) CreateBucket(bucketName string) error {
memory.lock.Lock()
defer memory.lock.Unlock()
if !drivers.IsValidBucket(bucketName) {
return drivers.BucketNameInvalid{Bucket: bucketName}
}
if _, ok := memory.bucketdata[bucketName]; ok == true {
return drivers.BucketExists{Bucket: bucketName}
}
var newBucket = storedBucket{}
newBucket.metadata = drivers.BucketMetadata{}
newBucket.metadata.Name = bucketName
newBucket.metadata.Created = time.Now()
memory.bucketdata[bucketName] = newBucket
return nil
}
func delimiter(object, delimiter string) string {
readBuffer := bytes.NewBufferString(object)
reader := bufio.NewReader(readBuffer)
stringReader := strings.NewReader(delimiter)
delimited, _ := stringReader.ReadByte()
delimitedStr, _ := reader.ReadString(delimited)
return delimitedStr
}
func appendUniq(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}
// ListObjects - list objects from memory
func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
if _, ok := memory.bucketdata[bucket]; ok == false {
return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket}
}
var results []drivers.ObjectMetadata
var keys []string
for key := range memory.objectdata {
switch true {
// Prefix absent, delimit object key based on delimiter
case resources.Delimiter != "" && resources.Prefix == "":
delimitedName := delimiter(key, resources.Delimiter)
switch true {
case delimitedName == "" || delimitedName == key:
keys = appendUniq(keys, key)
case delimitedName != "":
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
// Prefix present, delimit object key with prefix key based on delimiter
case resources.Delimiter != "" && resources.Prefix != "" && strings.HasPrefix(key, resources.Prefix):
trimmedName := strings.TrimPrefix(key, resources.Prefix)
delimitedName := delimiter(trimmedName, resources.Delimiter)
switch true {
case key == resources.Prefix:
keys = appendUniq(keys, key)
// DelimitedName - requires resources.Prefix as it was trimmed off earlier in the flow
case key == resources.Prefix+delimitedName:
keys = appendUniq(keys, key)
case delimitedName != "":
if delimitedName == resources.Delimiter {
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, resources.Prefix+delimitedName)
} else {
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
}
// Prefix present, nothing to delimit
case resources.Delimiter == "" && resources.Prefix != "" && strings.HasPrefix(key, resources.Prefix):
keys = appendUniq(keys, key)
// Prefix and delimiter absent
case resources.Prefix == "" && resources.Delimiter == "":
keys = appendUniq(keys, key)
}
}
sort.Strings(keys)
for _, key := range keys {
if len(results) == resources.Maxkeys {
return results, drivers.BucketResourcesMetadata{IsTruncated: true}, nil
}
object := memory.objectdata[key]
if bucket == object.metadata.Bucket {
results = append(results, object.metadata)
}
}
return results, resources, nil
}
// ByBucketName is a type for sorting bucket metadata by bucket name
type ByBucketName []drivers.BucketMetadata
// Len of bucket name
func (b ByBucketName) Len() int { return len(b) }
// Swap bucket i, j
func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Less
func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
// ListBuckets - List buckets from memory
func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
var results []drivers.BucketMetadata
for _, bucket := range memory.bucketdata {
results = append(results, bucket.metadata)
}
sort.Sort(ByBucketName(results))
return results, nil
}
// GetObjectMetadata - get object metadata from memory
func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (drivers.ObjectMetadata, error) {
if object, ok := memory.objectdata[key]; ok == true {
return object.metadata, nil
}
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key}
}

39
pkg/drivers/memory/memory_test.go Normal file
View File

@@ -0,0 +1,39 @@
/*
* Minimalist Object Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package memory
import (
"testing"
"github.com/minio-io/minio/pkg/drivers"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) { TestingT(t) }
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
create := func() drivers.Driver {
_, _, store := Start()
return store
}
drivers.APITestSuite(c, create)
}