mirror of https://github.com/minio/minio.git
Merge pull request #469 from harshavardhana/pr_out_file_driver_to_take_the_high_road
This commit is contained in:
Harshavardhana
commit
ed67a1269d
2
main.go
2
main.go
|
|
@ -82,8 +82,6 @@ func init() {
|
||||||
|
|
||||||
func getDriverType(input string) server.DriverType {
|
func getDriverType(input string) server.DriverType {
|
||||||
switch {
|
switch {
|
||||||
case input == "file":
|
|
||||||
return server.File
|
|
||||||
case input == "memory":
|
case input == "memory":
|
||||||
return server.Memory
|
return server.Memory
|
||||||
case input == "donut":
|
case input == "donut":
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,6 @@ import (
|
||||||
"github.com/minio-io/minio/pkg/api"
|
"github.com/minio-io/minio/pkg/api"
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
"github.com/minio-io/minio/pkg/storage/drivers"
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers/donut"
|
"github.com/minio-io/minio/pkg/storage/drivers/donut"
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers/file"
|
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers/memory"
|
"github.com/minio-io/minio/pkg/storage/drivers/memory"
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers/mocks"
|
"github.com/minio-io/minio/pkg/storage/drivers/mocks"
|
||||||
"github.com/stretchr/testify/mock"
|
"github.com/stretchr/testify/mock"
|
||||||
|
|
@ -65,14 +64,6 @@ var _ = Suite(&MySuite{
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
var _ = Suite(&MySuite{
|
|
||||||
initDriver: func() (drivers.Driver, string) {
|
|
||||||
root, _ := ioutil.TempDir(os.TempDir(), "minio-api")
|
|
||||||
_, _, driver := file.Start(root)
|
|
||||||
return driver, root
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
var _ = Suite(&MySuite{
|
var _ = Suite(&MySuite{
|
||||||
initDriver: func() (drivers.Driver, string) {
|
initDriver: func() (drivers.Driver, string) {
|
||||||
root, _ := ioutil.TempDir(os.TempDir(), "minio-api")
|
root, _ := ioutil.TempDir(os.TempDir(), "minio-api")
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,6 @@ import (
|
||||||
"github.com/minio-io/minio/pkg/server/httpserver"
|
"github.com/minio-io/minio/pkg/server/httpserver"
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
"github.com/minio-io/minio/pkg/storage/drivers"
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers/donut"
|
"github.com/minio-io/minio/pkg/storage/drivers/donut"
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers/file"
|
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers/memory"
|
"github.com/minio-io/minio/pkg/storage/drivers/memory"
|
||||||
"github.com/minio-io/minio/pkg/utils/log"
|
"github.com/minio-io/minio/pkg/utils/log"
|
||||||
)
|
)
|
||||||
|
|
@ -61,7 +60,6 @@ type DriverType int
|
||||||
// Driver types
|
// Driver types
|
||||||
const (
|
const (
|
||||||
Memory DriverType = iota
|
Memory DriverType = iota
|
||||||
File
|
|
||||||
Donut
|
Donut
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -140,18 +138,6 @@ func getDriverChannels(driverType DriverType) (ctrlChans []chan<- string, status
|
||||||
ctrlChans = append(ctrlChans, ctrlChan)
|
ctrlChans = append(ctrlChans, ctrlChan)
|
||||||
statusChans = append(statusChans, statusChan)
|
statusChans = append(statusChans, statusChan)
|
||||||
}
|
}
|
||||||
case driverType == File:
|
|
||||||
{
|
|
||||||
u, err := user.Current()
|
|
||||||
if err != nil {
|
|
||||||
log.Error.Println(iodine.New(err, nil))
|
|
||||||
return nil, nil, nil
|
|
||||||
}
|
|
||||||
root := path.Join(u.HomeDir, "minio-storage", "file")
|
|
||||||
ctrlChan, statusChan, driver = file.Start(root)
|
|
||||||
ctrlChans = append(ctrlChans, ctrlChan)
|
|
||||||
statusChans = append(statusChans, statusChan)
|
|
||||||
}
|
|
||||||
case driverType == Donut:
|
case driverType == Donut:
|
||||||
{
|
{
|
||||||
u, err := user.Current()
|
u, err := user.Current()
|
||||||
|
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
/*
|
|
||||||
* Minimalist Object Storage, (C) 2015 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package file
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Start filesystem channel
|
|
||||||
func Start(root string) (chan<- string, <-chan error, drivers.Driver) {
|
|
||||||
ctrlChannel := make(chan string)
|
|
||||||
errorChannel := make(chan error)
|
|
||||||
s := new(fileDriver)
|
|
||||||
s.root = root
|
|
||||||
s.lock = new(sync.Mutex)
|
|
||||||
go start(ctrlChannel, errorChannel, s)
|
|
||||||
return ctrlChannel, errorChannel, s
|
|
||||||
}
|
|
||||||
|
|
||||||
func start(ctrlChannel <-chan string, errorChannel chan<- error, s *fileDriver) {
|
|
||||||
err := os.MkdirAll(s.root, 0700)
|
|
||||||
errorChannel <- err
|
|
||||||
close(errorChannel)
|
|
||||||
}
|
|
||||||
|
|
@ -1,146 +0,0 @@
|
||||||
/*
|
|
||||||
* Minimalist Object File, (C) 2015 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package file
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"io/ioutil"
|
|
||||||
"path/filepath"
|
|
||||||
|
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
|
||||||
)
|
|
||||||
|
|
||||||
/// Bucket Operations
|
|
||||||
|
|
||||||
// GetBucketMetadata - head
|
|
||||||
func (file *fileDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
|
|
||||||
st, err := os.Stat(path.Join(file.root, bucket))
|
|
||||||
if err != nil {
|
|
||||||
return drivers.BucketMetadata{}, drivers.BucketNotFound{Bucket: bucket}
|
|
||||||
}
|
|
||||||
bucketMetadata := drivers.BucketMetadata{
|
|
||||||
Name: st.Name(),
|
|
||||||
Created: st.ModTime(),
|
|
||||||
}
|
|
||||||
return bucketMetadata, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListBuckets - Get service
|
|
||||||
func (file *fileDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
|
|
||||||
files, err := ioutil.ReadDir(file.root)
|
|
||||||
if err != nil {
|
|
||||||
return []drivers.BucketMetadata{}, drivers.EmbedError("bucket", "", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var metadataList []drivers.BucketMetadata
|
|
||||||
for _, fileName := range files {
|
|
||||||
// Skip policy files
|
|
||||||
if strings.HasSuffix(fileName.Name(), "_policy.json") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !fileName.IsDir() {
|
|
||||||
return []drivers.BucketMetadata{}, drivers.BackendCorrupted{Path: file.root}
|
|
||||||
}
|
|
||||||
metadata := drivers.BucketMetadata{
|
|
||||||
Name: fileName.Name(),
|
|
||||||
Created: fileName.ModTime(), // TODO - provide real created time
|
|
||||||
}
|
|
||||||
metadataList = append(metadataList, metadata)
|
|
||||||
}
|
|
||||||
return metadataList, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// CreateBucket - PUT Bucket
|
|
||||||
func (file *fileDriver) CreateBucket(bucket string) error {
|
|
||||||
file.lock.Lock()
|
|
||||||
defer file.lock.Unlock()
|
|
||||||
|
|
||||||
// verify bucket path legal
|
|
||||||
if drivers.IsValidBucket(bucket) == false {
|
|
||||||
return drivers.BucketNameInvalid{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
// get bucket path
|
|
||||||
bucketDir := path.Join(file.root, bucket)
|
|
||||||
|
|
||||||
// check if bucket exists
|
|
||||||
if _, err := os.Stat(bucketDir); err == nil {
|
|
||||||
return drivers.BucketExists{
|
|
||||||
Bucket: bucket,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// make bucket
|
|
||||||
err := os.Mkdir(bucketDir, 0700)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListObjects - GET bucket (list objects)
|
|
||||||
func (file *fileDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
|
|
||||||
p := bucketDir{}
|
|
||||||
p.files = make(map[string]os.FileInfo)
|
|
||||||
|
|
||||||
if drivers.IsValidBucket(bucket) == false {
|
|
||||||
return []drivers.ObjectMetadata{}, resources, drivers.BucketNameInvalid{Bucket: bucket}
|
|
||||||
}
|
|
||||||
if resources.Prefix != "" && drivers.IsValidObject(resources.Prefix) == false {
|
|
||||||
return []drivers.ObjectMetadata{}, resources, drivers.ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix}
|
|
||||||
}
|
|
||||||
|
|
||||||
rootPrefix := path.Join(file.root, bucket)
|
|
||||||
// check bucket exists
|
|
||||||
if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
|
|
||||||
return []drivers.ObjectMetadata{}, resources, drivers.BucketNotFound{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
p.root = rootPrefix
|
|
||||||
err := filepath.Walk(rootPrefix, p.getAllFiles)
|
|
||||||
if err != nil {
|
|
||||||
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var metadataList []drivers.ObjectMetadata
|
|
||||||
var metadata drivers.ObjectMetadata
|
|
||||||
|
|
||||||
// Populate filtering mode
|
|
||||||
resources.Mode = drivers.GetMode(resources)
|
|
||||||
|
|
||||||
for name, f := range p.files {
|
|
||||||
if len(metadataList) >= resources.Maxkeys {
|
|
||||||
resources.IsTruncated = true
|
|
||||||
goto ret
|
|
||||||
}
|
|
||||||
metadata, resources, err = file.filter(bucket, name, f, resources)
|
|
||||||
if err != nil {
|
|
||||||
return []drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
if metadata.Bucket != "" {
|
|
||||||
metadataList = append(metadataList, metadata)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ret:
|
|
||||||
sort.Sort(byObjectKey(metadataList))
|
|
||||||
return metadataList, resources, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,89 +0,0 @@
|
||||||
/*
|
|
||||||
* Minimalist Object Storage, (C) 2015 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package file
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
|
||||||
)
|
|
||||||
|
|
||||||
// fileDriver - file local variables
|
|
||||||
type fileDriver struct {
|
|
||||||
root string
|
|
||||||
lock *sync.Mutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// fileMetadata - carries metadata about object
|
|
||||||
type fileMetadata struct {
|
|
||||||
Md5sum []byte
|
|
||||||
ContentType string
|
|
||||||
}
|
|
||||||
|
|
||||||
func appendUniq(slice []string, i string) []string {
|
|
||||||
for _, ele := range slice {
|
|
||||||
if ele == i {
|
|
||||||
return slice
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return append(slice, i)
|
|
||||||
}
|
|
||||||
|
|
||||||
type bucketDir struct {
|
|
||||||
files map[string]os.FileInfo
|
|
||||||
root string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *bucketDir) getAllFiles(object string, fl os.FileInfo, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if fl.Mode().IsRegular() {
|
|
||||||
if strings.HasSuffix(object, "$metadata") {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
_p := strings.Split(object, p.root+"/")
|
|
||||||
if len(_p) > 1 {
|
|
||||||
p.files[_p[1]] = fl
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func delimiter(object, delimiter string) string {
|
|
||||||
readBuffer := bytes.NewBufferString(object)
|
|
||||||
reader := bufio.NewReader(readBuffer)
|
|
||||||
stringReader := strings.NewReader(delimiter)
|
|
||||||
delimited, _ := stringReader.ReadByte()
|
|
||||||
delimitedStr, _ := reader.ReadString(delimited)
|
|
||||||
return delimitedStr
|
|
||||||
}
|
|
||||||
|
|
||||||
type byObjectKey []drivers.ObjectMetadata
|
|
||||||
|
|
||||||
// Len
|
|
||||||
func (b byObjectKey) Len() int { return len(b) }
|
|
||||||
|
|
||||||
// Swap
|
|
||||||
func (b byObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
|
|
||||||
|
|
||||||
// Less
|
|
||||||
func (b byObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key }
|
|
||||||
|
|
@ -1,100 +0,0 @@
|
||||||
/*
|
|
||||||
* Minimalist Object Storage, (C) 2015 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package file
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (file *fileDriver) filterDelimiterPrefix(bucket, name, fname, delimitedName string, resources drivers.BucketResourcesMetadata) (drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
|
|
||||||
var err error
|
|
||||||
var metadata drivers.ObjectMetadata
|
|
||||||
switch true {
|
|
||||||
case name == resources.Prefix:
|
|
||||||
// Use resources.Prefix to filter out delimited files
|
|
||||||
metadata, err = file.GetObjectMetadata(bucket, name, resources.Prefix)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
case delimitedName == fname:
|
|
||||||
// Use resources.Prefix to filter out delimited files
|
|
||||||
metadata, err = file.GetObjectMetadata(bucket, name, resources.Prefix)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
case delimitedName != "":
|
|
||||||
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, resources.Prefix+delimitedName)
|
|
||||||
}
|
|
||||||
return metadata, resources, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO handle resources.Marker
|
|
||||||
func (file *fileDriver) filter(bucket, name string, f os.FileInfo, resources drivers.BucketResourcesMetadata) (drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
|
|
||||||
var err error
|
|
||||||
var metadata drivers.ObjectMetadata
|
|
||||||
|
|
||||||
switch true {
|
|
||||||
// Both delimiter and Prefix is present
|
|
||||||
case resources.IsDelimiterPrefixSet():
|
|
||||||
if strings.HasPrefix(name, resources.Prefix) {
|
|
||||||
trimmedName := strings.TrimPrefix(name, resources.Prefix)
|
|
||||||
delimitedName := delimiter(trimmedName, resources.Delimiter)
|
|
||||||
metadata, resources, err = file.filterDelimiterPrefix(bucket, name, f.Name(), delimitedName, resources)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.ObjectMetadata{}, resources, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Delimiter present and Prefix is absent
|
|
||||||
case resources.IsDelimiterSet():
|
|
||||||
delimitedName := delimiter(name, resources.Delimiter)
|
|
||||||
switch true {
|
|
||||||
case delimitedName == "":
|
|
||||||
// Do not strip prefix object output
|
|
||||||
metadata, err = file.GetObjectMetadata(bucket, name, "")
|
|
||||||
if err != nil {
|
|
||||||
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
case delimitedName == f.Name():
|
|
||||||
// Do not strip prefix object output
|
|
||||||
metadata, err = file.GetObjectMetadata(bucket, name, "")
|
|
||||||
if err != nil {
|
|
||||||
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
case delimitedName != "":
|
|
||||||
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
|
|
||||||
}
|
|
||||||
// Delimiter is absent and only Prefix is present
|
|
||||||
case resources.IsPrefixSet():
|
|
||||||
if strings.HasPrefix(name, resources.Prefix) {
|
|
||||||
// Do not strip prefix object output
|
|
||||||
metadata, err = file.GetObjectMetadata(bucket, name, "")
|
|
||||||
if err != nil {
|
|
||||||
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case resources.IsDefault():
|
|
||||||
metadata, err = file.GetObjectMetadata(bucket, name, "")
|
|
||||||
if err != nil {
|
|
||||||
return drivers.ObjectMetadata{}, resources, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return metadata, resources, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,283 +0,0 @@
|
||||||
/*
|
|
||||||
* Minimalist Object Storage, (C) 2015 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package file
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
|
||||||
|
|
||||||
"crypto/md5"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/gob"
|
|
||||||
"encoding/hex"
|
|
||||||
)
|
|
||||||
|
|
||||||
/// Object Operations
|
|
||||||
|
|
||||||
// GetPartialObject - GET object from range
|
|
||||||
func (file *fileDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) {
|
|
||||||
// validate bucket
|
|
||||||
if drivers.IsValidBucket(bucket) == false {
|
|
||||||
return 0, drivers.BucketNameInvalid{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
// validate object
|
|
||||||
if drivers.IsValidObject(object) == false {
|
|
||||||
return 0, drivers.ObjectNameInvalid{Bucket: bucket, Object: object}
|
|
||||||
}
|
|
||||||
|
|
||||||
objectPath := path.Join(file.root, bucket, object)
|
|
||||||
filestat, err := os.Stat(objectPath)
|
|
||||||
switch err := err.(type) {
|
|
||||||
case nil:
|
|
||||||
{
|
|
||||||
if filestat.IsDir() {
|
|
||||||
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
{
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
|
|
||||||
}
|
|
||||||
return 0, drivers.EmbedError(bucket, object, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f, err := os.Open(objectPath)
|
|
||||||
defer f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return 0, drivers.EmbedError(bucket, object, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = f.Seek(start, os.SEEK_SET)
|
|
||||||
if err != nil {
|
|
||||||
return 0, drivers.EmbedError(bucket, object, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
count, err := io.CopyN(w, f, length)
|
|
||||||
if err != nil {
|
|
||||||
return count, drivers.EmbedError(bucket, object, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return count, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetObject - GET object from key
|
|
||||||
func (file *fileDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
|
|
||||||
// validate bucket
|
|
||||||
if drivers.IsValidBucket(bucket) == false {
|
|
||||||
return 0, drivers.BucketNameInvalid{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
// check bucket exists
|
|
||||||
if _, err := os.Stat(path.Join(file.root, bucket)); os.IsNotExist(err) {
|
|
||||||
return 0, drivers.BucketNotFound{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
// validate object
|
|
||||||
if drivers.IsValidObject(object) == false {
|
|
||||||
return 0, drivers.ObjectNameInvalid{Bucket: bucket, Object: object}
|
|
||||||
}
|
|
||||||
|
|
||||||
objectPath := path.Join(file.root, bucket, object)
|
|
||||||
|
|
||||||
filestat, err := os.Stat(objectPath)
|
|
||||||
switch err := err.(type) {
|
|
||||||
case nil:
|
|
||||||
{
|
|
||||||
if filestat.IsDir() {
|
|
||||||
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
{
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
|
|
||||||
}
|
|
||||||
return 0, drivers.EmbedError(bucket, object, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f, err := os.Open(objectPath)
|
|
||||||
defer f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return 0, drivers.EmbedError(bucket, object, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
count, err := io.Copy(w, f)
|
|
||||||
if err != nil {
|
|
||||||
return count, drivers.EmbedError(bucket, object, err)
|
|
||||||
}
|
|
||||||
return count, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetObjectMetadata - HEAD object
|
|
||||||
func (file *fileDriver) GetObjectMetadata(bucket, object, prefix string) (drivers.ObjectMetadata, error) {
|
|
||||||
if drivers.IsValidBucket(bucket) == false {
|
|
||||||
return drivers.ObjectMetadata{}, drivers.BucketNameInvalid{Bucket: bucket}
|
|
||||||
}
|
|
||||||
if drivers.IsValidObject(object) == false {
|
|
||||||
return drivers.ObjectMetadata{}, drivers.ObjectNameInvalid{Bucket: bucket, Object: bucket}
|
|
||||||
}
|
|
||||||
// check bucket exists
|
|
||||||
if _, err := os.Stat(path.Join(file.root, bucket)); os.IsNotExist(err) {
|
|
||||||
return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket}
|
|
||||||
}
|
|
||||||
// Do not use path.Join() since path.Join strips off any object names with '/', use them as is
|
|
||||||
// in a static manner so that we can send a proper 'ObjectNotFound' reply back upon os.Stat()
|
|
||||||
objectPath := file.root + "/" + bucket + "/" + object
|
|
||||||
stat, err := os.Stat(objectPath)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: object}
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = os.Stat(objectPath + "$metadata")
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: object}
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := os.Open(objectPath + "$metadata")
|
|
||||||
defer f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return drivers.ObjectMetadata{}, drivers.EmbedError(bucket, object, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var deserializedMetadata fileMetadata
|
|
||||||
decoder := gob.NewDecoder(f)
|
|
||||||
err = decoder.Decode(&deserializedMetadata)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.ObjectMetadata{}, drivers.EmbedError(bucket, object, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
contentType := "application/octet-stream"
|
|
||||||
if deserializedMetadata.ContentType != "" {
|
|
||||||
contentType = deserializedMetadata.ContentType
|
|
||||||
}
|
|
||||||
contentType = strings.TrimSpace(contentType)
|
|
||||||
|
|
||||||
etag := bucket + "#" + path.Base(object)
|
|
||||||
if len(deserializedMetadata.Md5sum) != 0 {
|
|
||||||
etag = hex.EncodeToString(deserializedMetadata.Md5sum)
|
|
||||||
}
|
|
||||||
trimmedObject := strings.TrimPrefix(object, prefix)
|
|
||||||
metadata := drivers.ObjectMetadata{
|
|
||||||
Bucket: bucket,
|
|
||||||
Key: trimmedObject,
|
|
||||||
Created: stat.ModTime(),
|
|
||||||
Size: stat.Size(),
|
|
||||||
Md5: etag,
|
|
||||||
ContentType: contentType,
|
|
||||||
}
|
|
||||||
|
|
||||||
return metadata, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// CreateObject - PUT object
|
|
||||||
func (file *fileDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
|
|
||||||
// TODO Commits should stage then move instead of writing directly
|
|
||||||
file.lock.Lock()
|
|
||||||
defer file.lock.Unlock()
|
|
||||||
|
|
||||||
// check bucket name valid
|
|
||||||
if drivers.IsValidBucket(bucket) == false {
|
|
||||||
return drivers.BucketNameInvalid{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
// check bucket exists
|
|
||||||
if _, err := os.Stat(path.Join(file.root, bucket)); os.IsNotExist(err) {
|
|
||||||
return drivers.BucketNotFound{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
// verify object path legal
|
|
||||||
if drivers.IsValidObject(key) == false {
|
|
||||||
return drivers.ObjectNameInvalid{Bucket: bucket, Object: key}
|
|
||||||
}
|
|
||||||
|
|
||||||
// verify content type
|
|
||||||
if contentType == "" {
|
|
||||||
contentType = "application/octet-stream"
|
|
||||||
}
|
|
||||||
contentType = strings.TrimSpace(contentType)
|
|
||||||
|
|
||||||
// get object path
|
|
||||||
objectPath := path.Join(file.root, bucket, key)
|
|
||||||
objectDir := path.Dir(objectPath)
|
|
||||||
if _, err := os.Stat(objectDir); os.IsNotExist(err) {
|
|
||||||
err = os.MkdirAll(objectDir, 0700)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.EmbedError(bucket, key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// check if object exists
|
|
||||||
if _, err := os.Stat(objectPath); !os.IsNotExist(err) {
|
|
||||||
return drivers.ObjectExists{
|
|
||||||
Bucket: bucket,
|
|
||||||
Object: key,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// write object
|
|
||||||
f, err := os.OpenFile(objectPath, os.O_WRONLY|os.O_CREATE, 0600)
|
|
||||||
defer f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return drivers.EmbedError(bucket, key, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
h := md5.New()
|
|
||||||
mw := io.MultiWriter(f, h)
|
|
||||||
|
|
||||||
_, err = io.Copy(mw, data)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.EmbedError(bucket, key, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
|
||||||
f, err = os.OpenFile(objectPath+"$metadata", os.O_WRONLY|os.O_CREATE, 0600)
|
|
||||||
defer f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return drivers.EmbedError(bucket, key, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
metadata := &fileMetadata{
|
|
||||||
ContentType: contentType,
|
|
||||||
Md5sum: h.Sum(nil),
|
|
||||||
}
|
|
||||||
// serialize metadata to gob
|
|
||||||
encoder := gob.NewEncoder(f)
|
|
||||||
err = encoder.Encode(metadata)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.EmbedError(bucket, key, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify data received to be correct, Content-MD5 received
|
|
||||||
if md5sum != "" {
|
|
||||||
var data []byte
|
|
||||||
data, err = base64.StdEncoding.DecodeString(md5sum)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.InvalidDigest{Bucket: bucket, Key: key, Md5: md5sum}
|
|
||||||
}
|
|
||||||
if !bytes.Equal(metadata.Md5sum, data) {
|
|
||||||
return drivers.BadDigest{Bucket: bucket, Key: key, Md5: md5sum}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,112 +0,0 @@
|
||||||
/*
|
|
||||||
* Minimalist Object Storage, (C) 2015 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package file
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
|
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
|
||||||
|
|
||||||
"encoding/json"
|
|
||||||
)
|
|
||||||
|
|
||||||
// GetBucketPolicy - GET bucket policy
|
|
||||||
func (file *fileDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, error) {
|
|
||||||
file.lock.Lock()
|
|
||||||
defer file.lock.Unlock()
|
|
||||||
|
|
||||||
var p drivers.BucketPolicy
|
|
||||||
// verify bucket path legal
|
|
||||||
if drivers.IsValidBucket(bucket) == false {
|
|
||||||
return drivers.BucketPolicy{}, drivers.BucketNameInvalid{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
// get bucket path
|
|
||||||
bucketDir := path.Join(file.root, bucket)
|
|
||||||
// check if bucket exists
|
|
||||||
if _, err := os.Stat(bucketDir); err != nil {
|
|
||||||
return drivers.BucketPolicy{}, drivers.BucketNotFound{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
// get policy path
|
|
||||||
bucketPolicy := path.Join(file.root, bucket+"_policy.json")
|
|
||||||
filestat, err := os.Stat(bucketPolicy)
|
|
||||||
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return drivers.BucketPolicy{}, drivers.BucketPolicyNotFound{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
if filestat.IsDir() {
|
|
||||||
return drivers.BucketPolicy{}, drivers.BackendCorrupted{Path: bucketPolicy}
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := os.OpenFile(bucketPolicy, os.O_RDONLY, 0666)
|
|
||||||
defer f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
encoder := json.NewDecoder(f)
|
|
||||||
err = encoder.Decode(&p)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.BucketPolicy{}, drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return p, nil
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// CreateBucketPolicy - PUT bucket policy
|
|
||||||
func (file *fileDriver) CreateBucketPolicy(bucket string, p drivers.BucketPolicy) error {
|
|
||||||
file.lock.Lock()
|
|
||||||
defer file.lock.Unlock()
|
|
||||||
|
|
||||||
// verify bucket path legal
|
|
||||||
if drivers.IsValidBucket(bucket) == false {
|
|
||||||
return drivers.BucketNameInvalid{Bucket: bucket}
|
|
||||||
}
|
|
||||||
|
|
||||||
// get bucket path
|
|
||||||
bucketDir := path.Join(file.root, bucket)
|
|
||||||
// check if bucket exists
|
|
||||||
if _, err := os.Stat(bucketDir); err != nil {
|
|
||||||
return drivers.BucketNotFound{
|
|
||||||
Bucket: bucket,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// get policy path
|
|
||||||
bucketPolicy := path.Join(file.root, bucket+"_policy.json")
|
|
||||||
filestat, ret := os.Stat(bucketPolicy)
|
|
||||||
if !os.IsNotExist(ret) {
|
|
||||||
if filestat.IsDir() {
|
|
||||||
return drivers.BackendCorrupted{Path: bucketPolicy}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := os.OpenFile(bucketPolicy, os.O_WRONLY|os.O_CREATE, 0600)
|
|
||||||
defer f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
encoder := json.NewEncoder(f)
|
|
||||||
err = encoder.Encode(p)
|
|
||||||
if err != nil {
|
|
||||||
return drivers.EmbedError(bucket, "", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,52 +0,0 @@
|
||||||
/*
|
|
||||||
* Minimalist Object Storage, (C) 2015 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package file
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io/ioutil"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
. "github.com/minio-io/check"
|
|
||||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Test(t *testing.T) { TestingT(t) }
|
|
||||||
|
|
||||||
type MySuite struct{}
|
|
||||||
|
|
||||||
var _ = Suite(&MySuite{})
|
|
||||||
|
|
||||||
func (s *MySuite) TestAPISuite(c *C) {
|
|
||||||
var storageList []string
|
|
||||||
create := func() drivers.Driver {
|
|
||||||
path, err := ioutil.TempDir(os.TempDir(), "minio-file-")
|
|
||||||
c.Check(err, IsNil)
|
|
||||||
storageList = append(storageList, path)
|
|
||||||
_, _, store := Start(path)
|
|
||||||
return store
|
|
||||||
}
|
|
||||||
drivers.APITestSuite(c, create)
|
|
||||||
removeRoots(c, storageList)
|
|
||||||
}
|
|
||||||
|
|
||||||
func removeRoots(c *C, roots []string) {
|
|
||||||
for _, root := range roots {
|
|
||||||
err := os.RemoveAll(root)
|
|
||||||
c.Check(err, IsNil)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue