mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
api: Various fixes.
- limit list buckets to limit only 100 buckets, all uppercase buckets are now lowercase and work transparently with all calls. - Change disk.Stat to disk.GetInfo and return back disk.Info{} struct. - Introduce new ioutils package which implements ReadDirN(path, n), ReadDirNamesN(path, n)
This commit is contained in:
parent
432a073e6b
commit
497f13d733
@ -16,11 +16,11 @@
|
|||||||
|
|
||||||
package disk
|
package disk
|
||||||
|
|
||||||
// StatFS stat fs struct is container which holds following values
|
// Info stat fs struct is container which holds following values
|
||||||
// Total - total size of the volume / disk
|
// Total - total size of the volume / disk
|
||||||
// Free - free size of the volume / disk
|
// Free - free size of the volume / disk
|
||||||
// FSType - file system type string
|
// Type - file system type string
|
||||||
type StatFS struct {
|
type Info struct {
|
||||||
Total int64
|
Total int64
|
||||||
Free int64
|
Free int64
|
||||||
FSType string
|
FSType string
|
||||||
|
@ -36,9 +36,9 @@ func (s *MySuite) TestFree(c *C) {
|
|||||||
path, err := ioutil.TempDir(os.TempDir(), "minio-")
|
path, err := ioutil.TempDir(os.TempDir(), "minio-")
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
statfs, err := disk.Stat(path)
|
di, err := disk.GetInfo(path)
|
||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
c.Assert(statfs.Total, Not(Equals), 0)
|
c.Assert(di.Total, Not(Equals), 0)
|
||||||
c.Assert(statfs.Free, Not(Equals), 0)
|
c.Assert(di.Free, Not(Equals), 0)
|
||||||
c.Assert(statfs.FSType, Not(Equals), "UNKNOWN")
|
c.Assert(di.FSType, Not(Equals), "UNKNOWN")
|
||||||
}
|
}
|
||||||
|
@ -22,19 +22,19 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Stat returns total and free bytes available in a directory, e.g. `/`.
|
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
|
||||||
func Stat(path string) (statfs StatFS, err error) {
|
func GetInfo(path string) (info Info, err error) {
|
||||||
s := syscall.Statfs_t{}
|
s := syscall.Statfs_t{}
|
||||||
err = syscall.Statfs(path, &s)
|
err = syscall.Statfs(path, &s)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return StatFS{}, err
|
return Info{}, err
|
||||||
}
|
}
|
||||||
statfs = StatFS{}
|
info = Info{}
|
||||||
statfs.Total = int64(s.Bsize) * int64(s.Blocks)
|
info.Total = int64(s.Bsize) * int64(s.Blocks)
|
||||||
statfs.Free = int64(s.Bsize) * int64(s.Bfree)
|
info.Free = int64(s.Bsize) * int64(s.Bfree)
|
||||||
statfs.FSType, err = getFSType(path)
|
info.FSType, err = getFSType(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return StatFS{}, err
|
return Info{}, err
|
||||||
}
|
}
|
||||||
return statfs, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
@ -23,11 +23,11 @@ import (
|
|||||||
"unsafe"
|
"unsafe"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Stat returns total and free bytes available in a directory, e.g. `C:\`.
|
// GetInfo returns total and free bytes available in a directory, e.g. `C:\`.
|
||||||
// It returns free space available to the user (including quota limitations)
|
// It returns free space available to the user (including quota limitations)
|
||||||
//
|
//
|
||||||
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
|
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
|
||||||
func Stat(path string) (statfs StatFS, err error) {
|
func GetInfo(path string) (info Info, err error) {
|
||||||
dll := syscall.MustLoadDLL("kernel32.dll")
|
dll := syscall.MustLoadDLL("kernel32.dll")
|
||||||
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
|
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
|
||||||
// Retrieves information about the amount of space that is available on a disk volume,
|
// Retrieves information about the amount of space that is available on a disk volume,
|
||||||
@ -50,9 +50,9 @@ func Stat(path string) (statfs StatFS, err error) {
|
|||||||
uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
|
uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
|
||||||
uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
|
uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
|
||||||
uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)))
|
uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)))
|
||||||
statfs = StatFS{}
|
info = Info{}
|
||||||
statfs.Total = int64(lpTotalNumberOfBytes)
|
info.Total = int64(lpTotalNumberOfBytes)
|
||||||
statfs.Free = int64(lpFreeBytesAvailable)
|
info.Free = int64(lpFreeBytesAvailable)
|
||||||
statfs.FSType = getFSType(path)
|
info.FSType = getFSType(path)
|
||||||
return statfs, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
@ -165,6 +165,9 @@ func IsValidBucketACL(acl string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// validBucket regexp.
|
||||||
|
var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
|
||||||
|
|
||||||
// IsValidBucketName - verify bucket name in accordance with
|
// IsValidBucketName - verify bucket name in accordance with
|
||||||
// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
|
// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
|
||||||
func IsValidBucketName(bucket string) bool {
|
func IsValidBucketName(bucket string) bool {
|
||||||
@ -177,12 +180,7 @@ func IsValidBucketName(bucket string) bool {
|
|||||||
if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
|
if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if match, _ := regexp.MatchString("\\.\\.", bucket); match == true {
|
return validBucket.MatchString(bucket)
|
||||||
return false
|
|
||||||
}
|
|
||||||
// We don't support buckets with '.' in them
|
|
||||||
match, _ := regexp.MatchString("^[a-zA-Z0-9][a-zA-Z0-9\\-]+[a-zA-Z0-9]$", bucket)
|
|
||||||
return match
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsValidObjectName - verify object name in accordance with
|
// IsValidObjectName - verify object name in accordance with
|
||||||
|
@ -38,6 +38,8 @@ func (fs Filesystem) ListObjects(bucket string, resources BucketResourcesMetadat
|
|||||||
return nil, resources, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix})
|
return nil, resources, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
|
|
||||||
p := bucketDir{}
|
p := bucketDir{}
|
||||||
rootPrefix := filepath.Join(fs.path, bucket)
|
rootPrefix := filepath.Join(fs.path, bucket)
|
||||||
// check bucket exists
|
// check bucket exists
|
||||||
|
@ -17,13 +17,15 @@
|
|||||||
package fs
|
package fs
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io/ioutil"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/minio/minio-xl/pkg/probe"
|
"github.com/minio/minio-xl/pkg/probe"
|
||||||
"github.com/minio/minio/pkg/disk"
|
"github.com/minio/minio/pkg/disk"
|
||||||
|
"github.com/minio/minio/pkg/ioutils"
|
||||||
)
|
)
|
||||||
|
|
||||||
/// Bucket Operations
|
/// Bucket Operations
|
||||||
@ -36,22 +38,20 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
|
|||||||
if !IsValidBucketName(bucket) {
|
if !IsValidBucketName(bucket) {
|
||||||
return probe.NewError(BucketNameInvalid{Bucket: bucket})
|
return probe.NewError(BucketNameInvalid{Bucket: bucket})
|
||||||
}
|
}
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
bucketDir := filepath.Join(fs.path, bucket)
|
bucketDir := filepath.Join(fs.path, bucket)
|
||||||
// check bucket exists
|
// check bucket exists
|
||||||
if _, err := os.Stat(bucketDir); err != nil {
|
if _, e := os.Stat(bucketDir); e != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(e) {
|
||||||
return probe.NewError(BucketNotFound{Bucket: bucket})
|
return probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
}
|
}
|
||||||
return probe.NewError(err)
|
return probe.NewError(e)
|
||||||
}
|
}
|
||||||
if _, ok := fs.buckets.Metadata[bucket]; !ok {
|
if e := os.Remove(bucketDir); e != nil {
|
||||||
return probe.NewError(BucketNotFound{Bucket: bucket})
|
if strings.Contains(e.Error(), "directory not empty") {
|
||||||
}
|
|
||||||
if err := os.Remove(bucketDir); err != nil {
|
|
||||||
if strings.Contains(err.Error(), "directory not empty") {
|
|
||||||
return probe.NewError(BucketNotEmpty{Bucket: bucket})
|
return probe.NewError(BucketNotEmpty{Bucket: bucket})
|
||||||
}
|
}
|
||||||
return probe.NewError(err)
|
return probe.NewError(e)
|
||||||
}
|
}
|
||||||
delete(fs.buckets.Metadata, bucket)
|
delete(fs.buckets.Metadata, bucket)
|
||||||
if err := saveBucketsMetadata(fs.buckets); err != nil {
|
if err := saveBucketsMetadata(fs.buckets); err != nil {
|
||||||
@ -60,34 +60,50 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func removeDuplicateBuckets(elements []BucketMetadata) (result []BucketMetadata) {
|
||||||
|
// Use map to record duplicates as we find them.
|
||||||
|
duplicates := make(map[string]struct{})
|
||||||
|
for _, element := range elements {
|
||||||
|
if _, ok := duplicates[element.Name]; !ok {
|
||||||
|
duplicates[element.Name] = struct{}{}
|
||||||
|
result = append(result, element)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
// ListBuckets - Get service
|
// ListBuckets - Get service
|
||||||
func (fs Filesystem) ListBuckets() ([]BucketMetadata, *probe.Error) {
|
func (fs Filesystem) ListBuckets() ([]BucketMetadata, *probe.Error) {
|
||||||
fs.lock.Lock()
|
fs.lock.Lock()
|
||||||
defer fs.lock.Unlock()
|
defer fs.lock.Unlock()
|
||||||
|
|
||||||
files, err := ioutil.ReadDir(fs.path)
|
files, err := ioutils.ReadDirN(fs.path, fs.maxBuckets)
|
||||||
if err != nil {
|
if err != nil && err != io.EOF {
|
||||||
return []BucketMetadata{}, probe.NewError(err)
|
return []BucketMetadata{}, probe.NewError(err)
|
||||||
}
|
}
|
||||||
|
if err == io.EOF {
|
||||||
|
fmt.Printf("Truncating the bucket list to %d entries only.", fs.maxBuckets)
|
||||||
|
}
|
||||||
var metadataList []BucketMetadata
|
var metadataList []BucketMetadata
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
if !file.IsDir() {
|
if !file.IsDir() {
|
||||||
// if files found ignore them
|
// if files found ignore them
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
dirName := strings.ToLower(file.Name())
|
||||||
if file.IsDir() {
|
if file.IsDir() {
|
||||||
// if directories found with odd names, skip them too
|
// if directories found with odd names, skip them too
|
||||||
if !IsValidBucketName(file.Name()) {
|
if !IsValidBucketName(dirName) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
metadata := BucketMetadata{
|
metadata := BucketMetadata{
|
||||||
Name: file.Name(),
|
Name: dirName,
|
||||||
Created: file.ModTime(),
|
Created: file.ModTime(),
|
||||||
}
|
}
|
||||||
metadataList = append(metadataList, metadata)
|
metadataList = append(metadataList, metadata)
|
||||||
}
|
}
|
||||||
|
metadataList = removeDuplicateBuckets(metadataList)
|
||||||
return metadataList, nil
|
return metadataList, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -96,13 +112,13 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
|
|||||||
fs.lock.Lock()
|
fs.lock.Lock()
|
||||||
defer fs.lock.Unlock()
|
defer fs.lock.Unlock()
|
||||||
|
|
||||||
stfs, err := disk.Stat(fs.path)
|
di, err := disk.GetInfo(fs.path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return probe.NewError(err)
|
return probe.NewError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
|
// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
|
||||||
availableDiskSpace := (float64(stfs.Free) / (float64(stfs.Total) - (0.05 * float64(stfs.Total)))) * 100
|
availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
|
||||||
if int64(availableDiskSpace) <= fs.minFreeDisk {
|
if int64(availableDiskSpace) <= fs.minFreeDisk {
|
||||||
return probe.NewError(RootPathFull{Path: fs.path})
|
return probe.NewError(RootPathFull{Path: fs.path})
|
||||||
}
|
}
|
||||||
@ -116,29 +132,26 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
|
|||||||
return probe.NewError(InvalidACL{ACL: acl})
|
return probe.NewError(InvalidACL{ACL: acl})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
// get bucket path
|
// get bucket path
|
||||||
bucketDir := filepath.Join(fs.path, bucket)
|
bucketDir := filepath.Join(fs.path, bucket)
|
||||||
// check if bucket exists
|
if _, e := os.Stat(bucketDir); e == nil {
|
||||||
if _, err = os.Stat(bucketDir); err == nil {
|
return probe.NewError(BucketExists{Bucket: bucket})
|
||||||
return probe.NewError(BucketExists{
|
|
||||||
Bucket: bucket,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// make bucket
|
// make bucket
|
||||||
err = os.Mkdir(bucketDir, 0700)
|
if e := os.Mkdir(bucketDir, 0700); e != nil {
|
||||||
if err != nil {
|
|
||||||
return probe.NewError(err)
|
return probe.NewError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
bucketMetadata := &BucketMetadata{}
|
bucketMetadata := &BucketMetadata{}
|
||||||
fi, err := os.Stat(bucketDir)
|
fi, e := os.Stat(bucketDir)
|
||||||
// check if bucket exists
|
// check if bucket exists
|
||||||
if err != nil {
|
if e != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(e) {
|
||||||
return probe.NewError(BucketNotFound{Bucket: bucket})
|
return probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
}
|
}
|
||||||
return probe.NewError(err)
|
return probe.NewError(e)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(acl) == "" {
|
if strings.TrimSpace(acl) == "" {
|
||||||
acl = "private"
|
acl = "private"
|
||||||
@ -153,6 +166,19 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (fs Filesystem) denormalizeBucket(bucket string) string {
|
||||||
|
buckets, err := ioutils.ReadDirNamesN(fs.path, fs.maxBuckets)
|
||||||
|
if err != nil {
|
||||||
|
return bucket
|
||||||
|
}
|
||||||
|
for _, b := range buckets {
|
||||||
|
if strings.ToLower(b) == bucket {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return bucket
|
||||||
|
}
|
||||||
|
|
||||||
// GetBucketMetadata - get bucket metadata
|
// GetBucketMetadata - get bucket metadata
|
||||||
func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error) {
|
func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error) {
|
||||||
fs.lock.Lock()
|
fs.lock.Lock()
|
||||||
@ -160,15 +186,18 @@ func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Er
|
|||||||
if !IsValidBucketName(bucket) {
|
if !IsValidBucketName(bucket) {
|
||||||
return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
|
return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
|
|
||||||
// get bucket path
|
// get bucket path
|
||||||
bucketDir := filepath.Join(fs.path, bucket)
|
bucketDir := filepath.Join(fs.path, bucket)
|
||||||
fi, err := os.Stat(bucketDir)
|
fi, e := os.Stat(bucketDir)
|
||||||
if err != nil {
|
if e != nil {
|
||||||
// check if bucket exists
|
// check if bucket exists
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(e) {
|
||||||
return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
|
return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
}
|
}
|
||||||
return BucketMetadata{}, probe.NewError(err)
|
return BucketMetadata{}, probe.NewError(e)
|
||||||
}
|
}
|
||||||
bucketMetadata, ok := fs.buckets.Metadata[bucket]
|
bucketMetadata, ok := fs.buckets.Metadata[bucket]
|
||||||
if !ok {
|
if !ok {
|
||||||
@ -194,14 +223,15 @@ func (fs Filesystem) SetBucketMetadata(bucket string, metadata map[string]string
|
|||||||
if strings.TrimSpace(acl) == "" {
|
if strings.TrimSpace(acl) == "" {
|
||||||
acl = "private"
|
acl = "private"
|
||||||
}
|
}
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
bucketDir := filepath.Join(fs.path, bucket)
|
bucketDir := filepath.Join(fs.path, bucket)
|
||||||
fi, err := os.Stat(bucketDir)
|
fi, e := os.Stat(bucketDir)
|
||||||
if err != nil {
|
if e != nil {
|
||||||
// check if bucket exists
|
// check if bucket exists
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(e) {
|
||||||
return probe.NewError(BucketNotFound{Bucket: bucket})
|
return probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
}
|
}
|
||||||
return probe.NewError(err)
|
return probe.NewError(e)
|
||||||
}
|
}
|
||||||
bucketMetadata, ok := fs.buckets.Metadata[bucket]
|
bucketMetadata, ok := fs.buckets.Metadata[bucket]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -59,14 +59,14 @@ func (fs Filesystem) ListMultipartUploads(bucket string, resources BucketMultipa
|
|||||||
if !IsValidBucketName(bucket) {
|
if !IsValidBucketName(bucket) {
|
||||||
return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
|
return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
|
||||||
}
|
}
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
bucketPath := filepath.Join(fs.path, bucket)
|
bucketPath := filepath.Join(fs.path, bucket)
|
||||||
_, err := os.Stat(bucketPath)
|
if _, e := os.Stat(bucketPath); e != nil {
|
||||||
// check bucket exists
|
// check bucket exists
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(e) {
|
||||||
return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
|
return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
}
|
}
|
||||||
if err != nil {
|
return BucketMultipartResourcesMetadata{}, probe.NewError(e)
|
||||||
return BucketMultipartResourcesMetadata{}, probe.NewError(InternalError{})
|
|
||||||
}
|
}
|
||||||
var uploads []*UploadMetadata
|
var uploads []*UploadMetadata
|
||||||
for object, session := range fs.multiparts.ActiveSession {
|
for object, session := range fs.multiparts.ActiveSession {
|
||||||
@ -142,13 +142,13 @@ func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.E
|
|||||||
fs.lock.Lock()
|
fs.lock.Lock()
|
||||||
defer fs.lock.Unlock()
|
defer fs.lock.Unlock()
|
||||||
|
|
||||||
stfs, err := disk.Stat(fs.path)
|
di, e := disk.GetInfo(fs.path)
|
||||||
if err != nil {
|
if e != nil {
|
||||||
return "", probe.NewError(err)
|
return "", probe.NewError(e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
|
// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
|
||||||
availableDiskSpace := (float64(stfs.Free) / (float64(stfs.Total) - (0.05 * float64(stfs.Total)))) * 100
|
availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
|
||||||
if int64(availableDiskSpace) <= fs.minFreeDisk {
|
if int64(availableDiskSpace) <= fs.minFreeDisk {
|
||||||
return "", probe.NewError(RootPathFull{Path: fs.path})
|
return "", probe.NewError(RootPathFull{Path: fs.path})
|
||||||
}
|
}
|
||||||
@ -160,31 +160,35 @@ func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.E
|
|||||||
return "", probe.NewError(ObjectNameInvalid{Object: object})
|
return "", probe.NewError(ObjectNameInvalid{Object: object})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
bucketPath := filepath.Join(fs.path, bucket)
|
bucketPath := filepath.Join(fs.path, bucket)
|
||||||
_, err = os.Stat(bucketPath)
|
if _, e := os.Stat(bucketPath); e != nil {
|
||||||
// check bucket exists
|
// check bucket exists
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(e) {
|
||||||
return "", probe.NewError(BucketNotFound{Bucket: bucket})
|
return "", probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
}
|
}
|
||||||
if err != nil {
|
return "", probe.NewError(e)
|
||||||
return "", probe.NewError(InternalError{})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
objectPath := filepath.Join(bucketPath, object)
|
objectPath := filepath.Join(bucketPath, object)
|
||||||
objectDir := filepath.Dir(objectPath)
|
objectDir := filepath.Dir(objectPath)
|
||||||
if _, err = os.Stat(objectDir); os.IsNotExist(err) {
|
if _, e := os.Stat(objectDir); e != nil {
|
||||||
err = os.MkdirAll(objectDir, 0700)
|
if os.IsNotExist(e) {
|
||||||
if err != nil {
|
e = os.MkdirAll(objectDir, 0700)
|
||||||
return "", probe.NewError(err)
|
if e != nil {
|
||||||
|
return "", probe.NewError(e)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
return "", probe.NewError(e)
|
||||||
}
|
}
|
||||||
|
|
||||||
id := []byte(strconv.FormatInt(rand.Int63(), 10) + bucket + object + time.Now().String())
|
id := []byte(strconv.FormatInt(rand.Int63(), 10) + bucket + object + time.Now().String())
|
||||||
uploadIDSum := sha512.Sum512(id)
|
uploadIDSum := sha512.Sum512(id)
|
||||||
uploadID := base64.URLEncoding.EncodeToString(uploadIDSum[:])[:47]
|
uploadID := base64.URLEncoding.EncodeToString(uploadIDSum[:])[:47]
|
||||||
|
|
||||||
multiPartfile, err := os.OpenFile(objectPath+"$multiparts", os.O_WRONLY|os.O_CREATE, 0600)
|
multiPartfile, e := os.OpenFile(objectPath+"$multiparts", os.O_WRONLY|os.O_CREATE, 0600)
|
||||||
if err != nil {
|
if e != nil {
|
||||||
return "", probe.NewError(err)
|
return "", probe.NewError(e)
|
||||||
}
|
}
|
||||||
defer multiPartfile.Close()
|
defer multiPartfile.Close()
|
||||||
|
|
||||||
@ -197,9 +201,8 @@ func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.E
|
|||||||
fs.multiparts.ActiveSession[object] = mpartSession
|
fs.multiparts.ActiveSession[object] = mpartSession
|
||||||
|
|
||||||
encoder := json.NewEncoder(multiPartfile)
|
encoder := json.NewEncoder(multiPartfile)
|
||||||
err = encoder.Encode(mpartSession)
|
if e = encoder.Encode(mpartSession); e != nil {
|
||||||
if err != nil {
|
return "", probe.NewError(e)
|
||||||
return "", probe.NewError(err)
|
|
||||||
}
|
}
|
||||||
if err := saveMultipartsSession(fs.multiparts); err != nil {
|
if err := saveMultipartsSession(fs.multiparts); err != nil {
|
||||||
return "", err.Trace()
|
return "", err.Trace()
|
||||||
@ -219,13 +222,13 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s
|
|||||||
fs.lock.Lock()
|
fs.lock.Lock()
|
||||||
defer fs.lock.Unlock()
|
defer fs.lock.Unlock()
|
||||||
|
|
||||||
stfs, err := disk.Stat(fs.path)
|
di, err := disk.GetInfo(fs.path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", probe.NewError(err)
|
return "", probe.NewError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
|
// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
|
||||||
availableDiskSpace := (float64(stfs.Free) / (float64(stfs.Total) - (0.05 * float64(stfs.Total)))) * 100
|
availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
|
||||||
if int64(availableDiskSpace) <= fs.minFreeDisk {
|
if int64(availableDiskSpace) <= fs.minFreeDisk {
|
||||||
return "", probe.NewError(RootPathFull{Path: fs.path})
|
return "", probe.NewError(RootPathFull{Path: fs.path})
|
||||||
}
|
}
|
||||||
@ -257,15 +260,14 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s
|
|||||||
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
|
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
bucketPath := filepath.Join(fs.path, bucket)
|
bucketPath := filepath.Join(fs.path, bucket)
|
||||||
if _, err = os.Stat(bucketPath); err != nil {
|
if _, err = os.Stat(bucketPath); err != nil {
|
||||||
// check bucket exists
|
// check bucket exists
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return "", probe.NewError(BucketNotFound{Bucket: bucket})
|
return "", probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
}
|
}
|
||||||
if err != nil {
|
return "", probe.NewError(err)
|
||||||
return "", probe.NewError(InternalError{})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
objectPath := filepath.Join(bucketPath, object)
|
objectPath := filepath.Join(bucketPath, object)
|
||||||
@ -357,6 +359,7 @@ func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, da
|
|||||||
return ObjectMetadata{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
|
return ObjectMetadata{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
bucketPath := filepath.Join(fs.path, bucket)
|
bucketPath := filepath.Join(fs.path, bucket)
|
||||||
if _, err := os.Stat(bucketPath); err != nil {
|
if _, err := os.Stat(bucketPath); err != nil {
|
||||||
// check bucket exists
|
// check bucket exists
|
||||||
@ -470,14 +473,14 @@ func (fs Filesystem) ListObjectParts(bucket, object string, resources ObjectReso
|
|||||||
startPartNumber = objectResourcesMetadata.PartNumberMarker
|
startPartNumber = objectResourcesMetadata.PartNumberMarker
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
bucketPath := filepath.Join(fs.path, bucket)
|
bucketPath := filepath.Join(fs.path, bucket)
|
||||||
_, err := os.Stat(bucketPath)
|
if _, e := os.Stat(bucketPath); e != nil {
|
||||||
// check bucket exists
|
// check bucket exists
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(e) {
|
||||||
return ObjectResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
|
return ObjectResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
}
|
}
|
||||||
if err != nil {
|
return ObjectResourcesMetadata{}, probe.NewError(e)
|
||||||
return ObjectResourcesMetadata{}, probe.NewError(InternalError{})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
objectPath := filepath.Join(bucketPath, object)
|
objectPath := filepath.Join(bucketPath, object)
|
||||||
@ -528,27 +531,26 @@ func (fs Filesystem) AbortMultipartUpload(bucket, object, uploadID string) *prob
|
|||||||
return probe.NewError(InvalidUploadID{UploadID: uploadID})
|
return probe.NewError(InvalidUploadID{UploadID: uploadID})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
bucketPath := filepath.Join(fs.path, bucket)
|
bucketPath := filepath.Join(fs.path, bucket)
|
||||||
_, err := os.Stat(bucketPath)
|
if _, e := os.Stat(bucketPath); e != nil {
|
||||||
// check bucket exists
|
// check bucket exists
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(e) {
|
||||||
return probe.NewError(BucketNotFound{Bucket: bucket})
|
return probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
}
|
}
|
||||||
if err != nil {
|
return probe.NewError(e)
|
||||||
return probe.NewError(InternalError{})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
objectPath := filepath.Join(bucketPath, object)
|
objectPath := filepath.Join(bucketPath, object)
|
||||||
for _, part := range fs.multiparts.ActiveSession[object].Parts {
|
for _, part := range fs.multiparts.ActiveSession[object].Parts {
|
||||||
err = os.RemoveAll(objectPath + fmt.Sprintf("$%d-$multiparts", part.PartNumber))
|
e := os.RemoveAll(objectPath + fmt.Sprintf("$%d-$multiparts", part.PartNumber))
|
||||||
if err != nil {
|
if e != nil {
|
||||||
return probe.NewError(err)
|
return probe.NewError(e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
delete(fs.multiparts.ActiveSession, object)
|
delete(fs.multiparts.ActiveSession, object)
|
||||||
err = os.RemoveAll(objectPath + "$multiparts")
|
if e := os.RemoveAll(objectPath + "$multiparts"); e != nil {
|
||||||
if err != nil {
|
return probe.NewError(e)
|
||||||
return probe.NewError(err)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -52,7 +52,16 @@ func (fs Filesystem) GetObject(w io.Writer, bucket, object string, start, length
|
|||||||
return 0, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
|
return 0, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
|
||||||
}
|
}
|
||||||
|
|
||||||
objectPath := filepath.Join(fs.path, bucket, object)
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
|
bucketPath := filepath.Join(fs.path, bucket)
|
||||||
|
if _, e := os.Stat(bucketPath); e != nil {
|
||||||
|
if os.IsNotExist(e) {
|
||||||
|
return 0, probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
|
}
|
||||||
|
return 0, probe.NewError(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
objectPath := filepath.Join(bucketPath, object)
|
||||||
filestat, err := os.Stat(objectPath)
|
filestat, err := os.Stat(objectPath)
|
||||||
switch err := err.(type) {
|
switch err := err.(type) {
|
||||||
case nil:
|
case nil:
|
||||||
@ -170,13 +179,13 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in
|
|||||||
fs.lock.Lock()
|
fs.lock.Lock()
|
||||||
defer fs.lock.Unlock()
|
defer fs.lock.Unlock()
|
||||||
|
|
||||||
stfs, err := disk.Stat(fs.path)
|
di, err := disk.GetInfo(fs.path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ObjectMetadata{}, probe.NewError(err)
|
return ObjectMetadata{}, probe.NewError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
|
// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
|
||||||
availableDiskSpace := (float64(stfs.Free) / (float64(stfs.Total) - (0.05 * float64(stfs.Total)))) * 100
|
availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
|
||||||
if int64(availableDiskSpace) <= fs.minFreeDisk {
|
if int64(availableDiskSpace) <= fs.minFreeDisk {
|
||||||
return ObjectMetadata{}, probe.NewError(RootPathFull{Path: fs.path})
|
return ObjectMetadata{}, probe.NewError(RootPathFull{Path: fs.path})
|
||||||
}
|
}
|
||||||
@ -185,9 +194,14 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in
|
|||||||
if !IsValidBucketName(bucket) {
|
if !IsValidBucketName(bucket) {
|
||||||
return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
|
return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
|
||||||
}
|
}
|
||||||
// check bucket exists
|
|
||||||
if _, err = os.Stat(filepath.Join(fs.path, bucket)); os.IsNotExist(err) {
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
|
bucketPath := filepath.Join(fs.path, bucket)
|
||||||
|
if _, e := os.Stat(bucketPath); e != nil {
|
||||||
|
if os.IsNotExist(e) {
|
||||||
|
return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
|
}
|
||||||
|
return ObjectMetadata{}, probe.NewError(e)
|
||||||
}
|
}
|
||||||
// verify object path legal
|
// verify object path legal
|
||||||
if !IsValidObjectName(object) {
|
if !IsValidObjectName(object) {
|
||||||
@ -195,7 +209,7 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in
|
|||||||
}
|
}
|
||||||
|
|
||||||
// get object path
|
// get object path
|
||||||
objectPath := filepath.Join(fs.path, bucket, object)
|
objectPath := filepath.Join(bucketPath, object)
|
||||||
if strings.TrimSpace(expectedMD5Sum) != "" {
|
if strings.TrimSpace(expectedMD5Sum) != "" {
|
||||||
var expectedMD5SumBytes []byte
|
var expectedMD5SumBytes []byte
|
||||||
expectedMD5SumBytes, err = base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
|
expectedMD5SumBytes, err = base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
|
||||||
@ -308,10 +322,14 @@ func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error {
|
|||||||
return probe.NewError(BucketNameInvalid{Bucket: bucket})
|
return probe.NewError(BucketNameInvalid{Bucket: bucket})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucket = fs.denormalizeBucket(bucket)
|
||||||
bucketPath := filepath.Join(fs.path, bucket)
|
bucketPath := filepath.Join(fs.path, bucket)
|
||||||
// check bucket exists
|
// check bucket exists
|
||||||
if _, err := os.Stat(filepath.Join(fs.path, bucket)); os.IsNotExist(err) {
|
if _, e := os.Stat(bucketPath); e != nil {
|
||||||
return probe.NewError(BucketNotFound{Bucket: bucket})
|
if os.IsNotExist(e) {
|
||||||
|
return probe.NewError(BucketNotFound{Bucket: bucket})
|
||||||
|
}
|
||||||
|
return probe.NewError(e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// verify object path legal
|
// verify object path legal
|
||||||
|
30
pkg/fs/fs.go
30
pkg/fs/fs.go
@ -29,6 +29,7 @@ import (
|
|||||||
type Filesystem struct {
|
type Filesystem struct {
|
||||||
path string
|
path string
|
||||||
minFreeDisk int64
|
minFreeDisk int64
|
||||||
|
maxBuckets int
|
||||||
lock *sync.Mutex
|
lock *sync.Mutex
|
||||||
multiparts *Multiparts
|
multiparts *Multiparts
|
||||||
buckets *Buckets
|
buckets *Buckets
|
||||||
@ -91,11 +92,20 @@ func New(rootPath string) (Filesystem, *probe.Error) {
|
|||||||
return Filesystem{}, err.Trace()
|
return Filesystem{}, err.Trace()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
a := Filesystem{lock: new(sync.Mutex)}
|
fs := Filesystem{lock: new(sync.Mutex)}
|
||||||
a.path = rootPath
|
fs.path = rootPath
|
||||||
a.multiparts = multiparts
|
fs.multiparts = multiparts
|
||||||
a.buckets = buckets
|
fs.buckets = buckets
|
||||||
return a, nil
|
|
||||||
|
/// Defaults
|
||||||
|
|
||||||
|
// maximum buckets to be listed from list buckets.
|
||||||
|
fs.maxBuckets = 100
|
||||||
|
// minium free disk required for i/o operations to succeed.
|
||||||
|
fs.minFreeDisk = 10
|
||||||
|
|
||||||
|
// Return here.
|
||||||
|
return fs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetMinFreeDisk - set min free disk
|
// SetMinFreeDisk - set min free disk
|
||||||
@ -104,3 +114,13 @@ func (fs *Filesystem) SetMinFreeDisk(minFreeDisk int64) {
|
|||||||
defer fs.lock.Unlock()
|
defer fs.lock.Unlock()
|
||||||
fs.minFreeDisk = minFreeDisk
|
fs.minFreeDisk = minFreeDisk
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetMaxBuckets - set total number of buckets supported, default is 100.
|
||||||
|
func (fs *Filesystem) SetMaxBuckets(maxBuckets int) {
|
||||||
|
fs.lock.Lock()
|
||||||
|
defer fs.lock.Unlock()
|
||||||
|
if maxBuckets == 0 {
|
||||||
|
maxBuckets = 100
|
||||||
|
}
|
||||||
|
fs.maxBuckets = maxBuckets
|
||||||
|
}
|
||||||
|
45
pkg/ioutils/ioutils.go
Normal file
45
pkg/ioutils/ioutils.go
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
package ioutils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// byName implements sort.Interface for sorting os.FileInfo list.
|
||||||
|
type byName []os.FileInfo
|
||||||
|
|
||||||
|
func (f byName) Len() int { return len(f) }
|
||||||
|
func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() }
|
||||||
|
func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
|
||||||
|
|
||||||
|
// ReadDirN reads the directory named by dirname and returns
|
||||||
|
// a list of sorted directory entries of size 'n'.
|
||||||
|
func ReadDirN(dirname string, n int) ([]os.FileInfo, error) {
|
||||||
|
f, err := os.Open(dirname)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
list, err := f.Readdir(n)
|
||||||
|
f.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Sort(byName(list))
|
||||||
|
return list, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadDirNamesN reads the directory named by dirname and returns
|
||||||
|
// a list of sorted directory names of size 'n'.
|
||||||
|
func ReadDirNamesN(dirname string, n int) ([]string, error) {
|
||||||
|
f, err := os.Open(dirname)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names, err := f.Readdirnames(n)
|
||||||
|
f.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return names, nil
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user