minio/posix.go

551 lines
14 KiB
Go
Raw Normal View History

/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
fs: Break fs package to top-level and introduce ObjectAPI interface. ObjectAPI interface brings in changes needed for XL ObjectAPI layer. The new interface for any ObjectAPI layer is as below ``` // ObjectAPI interface. type ObjectAPI interface { // Bucket resource API. DeleteBucket(bucket string) *probe.Error ListBuckets() ([]BucketInfo, *probe.Error) MakeBucket(bucket string) *probe.Error GetBucketInfo(bucket string) (BucketInfo, *probe.Error) // Bucket query API. ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error) // Object resource API. GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error) DeleteObject(bucket, object string) *probe.Error // Object query API. NewMultipartUpload(bucket, object string) (string, *probe.Error) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) ListObjectParts(bucket, object string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (ObjectInfo, *probe.Error) AbortMultipartUpload(bucket, object, uploadID string) *probe.Error } ```
2016-03-30 19:15:28 -04:00
package main
import (
"io"
"os"
slashpath "path"
"runtime"
"strings"
"syscall"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/safe"
)
const (
fsMinSpacePercent = 5
)
// fsStorage - implements StorageAPI interface.
type fsStorage struct {
diskPath string
minFreeDisk int64
}
// checkPathLength - returns error if given path name length more than 255
func checkPathLength(pathName string) error {
// For MS Windows, the maximum path length is 255
if runtime.GOOS == "windows" {
if len(pathName) > 255 {
return errFileNameTooLong
}
return nil
}
// For non-windows system, check each path segment length is > 255
for len(pathName) > 0 && pathName != "." && pathName != "/" {
dir, file := slashpath.Dir(pathName), slashpath.Base(pathName)
if len(file) > 255 {
return errFileNameTooLong
}
pathName = dir
}
return nil
}
// isDirEmpty - returns whether given directory is empty or not.
func isDirEmpty(dirname string) bool {
f, err := os.Open(dirname)
if err != nil {
errorIf(err, "Unable to access directory.")
return false
}
defer f.Close()
// List one entry.
_, err = f.Readdirnames(1)
if err != nil {
if err == io.EOF {
// Returns true if we have reached EOF, directory is indeed empty.
return true
}
errorIf(err, "Unable to list directory.")
return false
}
// Directory is not empty.
return false
}
// Initialize a new storage disk.
func newPosix(diskPath string) (StorageAPI, error) {
if diskPath == "" {
return nil, errInvalidArgument
}
fs := fsStorage{
diskPath: diskPath,
minFreeDisk: fsMinSpacePercent, // Minimum 5% disk should be free.
}
st, err := os.Stat(diskPath)
if err != nil {
if os.IsNotExist(err) {
return fs, errDiskNotFound
}
return fs, err
}
if !st.IsDir() {
return fs, syscall.ENOTDIR
}
return fs, nil
}
// checkDiskFree verifies if disk path has sufficient minium free disk space.
func checkDiskFree(diskPath string, minFreeDisk int64) (err error) {
if err = checkPathLength(diskPath); err != nil {
return err
}
di, err := disk.GetInfo(diskPath)
if err != nil {
return err
}
// Remove 5% from total space for cumulative disk
// space used for journalling, inodes etc.
availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
if int64(availableDiskSpace) <= minFreeDisk {
return errDiskFull
}
// Success.
return nil
}
func removeDuplicateVols(volsInfo []VolInfo) []VolInfo {
// Use map to record duplicates as we find them.
result := []VolInfo{}
m := make(map[string]VolInfo)
for _, v := range volsInfo {
if _, found := m[v.Name]; !found {
m[v.Name] = v
}
}
result = make([]VolInfo, 0, len(m))
for _, v := range m {
result = append(result, v)
}
// Return the new slice.
return result
}
// gets all the unique directories from diskPath.
func getAllUniqueVols(dirPath string) ([]VolInfo, error) {
if err := checkPathLength(dirPath); err != nil {
return nil, err
}
entries, err := readDir(dirPath)
if err != nil {
return nil, errDiskNotFound
}
var volsInfo []VolInfo
for _, entry := range entries {
if !strings.HasSuffix(entry, slashSeparator) || !isValidVolname(slashpath.Clean(entry)) {
// Skip if entry is neither a directory not a valid volume name.
continue
}
var fi os.FileInfo
fi, err = os.Stat(pathJoin(dirPath, entry))
if err != nil {
// If the file does not exist, skip the entry.
if os.IsNotExist(err) {
continue
}
return nil, err
}
volsInfo = append(volsInfo, VolInfo{
Name: fi.Name(),
// As os.Stat() doesn't carry other than ModTime(), use
// ModTime() as CreatedTime.
Created: fi.ModTime(),
})
}
return removeDuplicateVols(volsInfo), nil
}
// getVolumeDir - will convert incoming volume names to
// corresponding valid volume names on the backend in a platform
// compatible way for all operating systems. If volume is not found
// an error is generated.
func (s fsStorage) getVolumeDir(volume string) (string, error) {
if !isValidVolname(volume) {
return "", errInvalidArgument
}
if err := checkPathLength(volume); err != nil {
return "", err
}
volumeDir := pathJoin(s.diskPath, volume)
_, err := os.Stat(volumeDir)
if err == nil {
return volumeDir, nil
}
if os.IsNotExist(err) {
var volsInfo []VolInfo
volsInfo, err = getAllUniqueVols(s.diskPath)
if err != nil {
// For any errors, treat it as disk not found.
return volumeDir, err
}
for _, vol := range volsInfo {
// Verify if lowercase version of the volume is equal to
// the incoming volume, then use the proper name.
if strings.ToLower(vol.Name) == volume {
volumeDir = pathJoin(s.diskPath, vol.Name)
return volumeDir, nil
}
}
return volumeDir, errVolumeNotFound
} else if os.IsPermission(err) {
return volumeDir, errVolumeAccessDenied
}
return volumeDir, err
}
// Make a volume entry.
func (s fsStorage) MakeVol(volume string) (err error) {
// Validate if disk is free.
if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
return err
}
volumeDir, err := s.getVolumeDir(volume)
if err == nil {
// Volume already exists, return error.
return errVolumeExists
}
// If volume not found create it.
if err == errVolumeNotFound {
// Make a volume entry.
return os.Mkdir(volumeDir, 0700)
}
// For all other errors return here.
return err
}
// ListVols - list volumes.
func (s fsStorage) ListVols() (volsInfo []VolInfo, err error) {
// Get disk info to be populated for VolInfo.
var diskInfo disk.Info
diskInfo, err = disk.GetInfo(s.diskPath)
if err != nil {
return nil, err
}
volsInfo, err = getAllUniqueVols(s.diskPath)
if err != nil {
return nil, err
}
for i, vol := range volsInfo {
// Volname on case sensitive fs backends can come in as
// capitalized, but object layer cannot consume it
// directly. Convert it as we see fit.
volName := strings.ToLower(vol.Name)
volInfo := VolInfo{
Name: volName,
Created: vol.Created,
Total: diskInfo.Total,
Free: diskInfo.Free,
FSType: diskInfo.FSType,
}
volsInfo[i] = volInfo
}
return volsInfo, nil
}
// StatVol - get volume info.
func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) {
// Verify if volume is valid and it exists.
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return VolInfo{}, err
}
// Stat a volume entry.
var st os.FileInfo
st, err = os.Stat(volumeDir)
if err != nil {
if os.IsNotExist(err) {
return VolInfo{}, errVolumeNotFound
}
return VolInfo{}, err
}
// Get disk info, to be returned back along with volume info.
var diskInfo disk.Info
diskInfo, err = disk.GetInfo(s.diskPath)
if err != nil {
return VolInfo{}, err
}
// As os.Stat() doesn't carry other than ModTime(), use ModTime()
// as CreatedTime.
createdTime := st.ModTime()
return VolInfo{
Name: volume,
Created: createdTime,
Free: diskInfo.Free,
Total: diskInfo.Total,
FSType: diskInfo.FSType,
}, nil
}
// DeleteVol - delete a volume.
func (s fsStorage) DeleteVol(volume string) error {
// Verify if volume is valid and it exists.
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return err
}
err = os.Remove(volumeDir)
if err != nil {
if os.IsNotExist(err) {
return errVolumeNotFound
} else if strings.Contains(err.Error(), "directory is not empty") {
2016-03-28 00:52:38 -04:00
// On windows the string is slightly different, handle it here.
return errVolumeNotEmpty
} else if strings.Contains(err.Error(), "directory not empty") {
// Hopefully for all other
// operating systems, this is
// assumed to be consistent.
return errVolumeNotEmpty
}
return err
}
2016-03-28 00:52:38 -04:00
return nil
}
// ListDir - return all the entries at the given directory path.
// If an entry is a directory it will be returned with a trailing "/".
func (s fsStorage) ListDir(volume, dirPath string) ([]string, error) {
// Verify if volume is valid and it exists.
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return nil, err
}
// Stat a volume entry.
_, err = os.Stat(volumeDir)
if err != nil {
if os.IsNotExist(err) {
return nil, errVolumeNotFound
}
return nil, err
}
return readDir(pathJoin(volumeDir, dirPath))
}
// ReadFile - read a file at a given offset.
func (s fsStorage) ReadFile(volume string, path string, offset int64) (readCloser io.ReadCloser, err error) {
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return nil, err
}
filePath := pathJoin(volumeDir, path)
if err = checkPathLength(filePath); err != nil {
return nil, err
}
file, err := os.Open(filePath)
if err != nil {
if os.IsNotExist(err) {
return nil, errFileNotFound
} else if os.IsPermission(err) {
return nil, errFileAccessDenied
}
return nil, err
}
st, err := file.Stat()
if err != nil {
return nil, err
}
// Verify if its not a regular file, since subsequent Seek is undefined.
if !st.Mode().IsRegular() {
return nil, errFileNotFound
}
// Seek to requested offset.
_, err = file.Seek(offset, os.SEEK_SET)
if err != nil {
return nil, err
}
return file, nil
}
// CreateFile - create a file at path.
func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) {
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return nil, err
}
if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
return nil, err
}
filePath := pathJoin(volumeDir, path)
if err = checkPathLength(filePath); err != nil {
return nil, err
}
// Verify if the file already exists and is not of regular type.
var st os.FileInfo
if st, err = os.Stat(filePath); err == nil {
if st.IsDir() {
return nil, errIsNotRegular
}
}
w, err := safe.CreateFileWithPrefix(filePath, "$tmpfile")
if err != nil {
// File path cannot be verified since one of the parents is a file.
if strings.Contains(err.Error(), "not a directory") {
return nil, errFileAccessDenied
}
return nil, err
}
return w, nil
}
// StatFile - get file info.
func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) {
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return FileInfo{}, err
}
filePath := slashpath.Join(volumeDir, path)
if err = checkPathLength(filePath); err != nil {
return FileInfo{}, err
}
st, err := os.Stat(filePath)
if err != nil {
// File is really not found.
if os.IsNotExist(err) {
return FileInfo{}, errFileNotFound
}
// File path cannot be verified since one of the parents is a file.
if strings.Contains(err.Error(), "not a directory") {
return FileInfo{}, errFileNotFound
}
// Return all errors here.
return FileInfo{}, err
}
// If its a directory its not a regular file.
if st.Mode().IsDir() {
return FileInfo{}, errFileNotFound
}
return FileInfo{
Volume: volume,
Name: path,
ModTime: st.ModTime(),
Size: st.Size(),
Mode: st.Mode(),
}, nil
}
// deleteFile - delete file path if its empty.
func deleteFile(basePath, deletePath string) error {
if basePath == deletePath {
return nil
}
// Verify if the path exists.
pathSt, err := os.Stat(deletePath)
if err != nil {
if os.IsNotExist(err) {
return errFileNotFound
} else if os.IsPermission(err) {
return errFileAccessDenied
}
return err
}
if pathSt.IsDir() && !isDirEmpty(deletePath) {
// Verify if directory is empty.
return nil
}
// Attempt to remove path.
if err := os.Remove(deletePath); err != nil {
return err
}
// Recursively go down the next path and delete again.
if err := deleteFile(basePath, slashpath.Dir(deletePath)); err != nil {
return err
}
return nil
}
// DeleteFile - delete a file at path.
func (s fsStorage) DeleteFile(volume, path string) error {
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return err
}
// Following code is needed so that we retain "/" suffix if any in
// path argument.
filePath := pathJoin(volumeDir, path)
if err = checkPathLength(filePath); err != nil {
return err
}
// Delete file and delete parent directory as well if its empty.
return deleteFile(volumeDir, filePath)
}
// RenameFile - rename file.
func (s fsStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
srcVolumeDir, err := s.getVolumeDir(srcVolume)
if err != nil {
return err
}
dstVolumeDir, err := s.getVolumeDir(dstVolume)
if err != nil {
return err
}
srcIsDir := strings.HasSuffix(srcPath, slashSeparator)
dstIsDir := strings.HasSuffix(dstPath, slashSeparator)
// Either src and dst have to be directories or files, else return error.
if !(srcIsDir && dstIsDir || !srcIsDir && !dstIsDir) {
return errFileAccessDenied
}
if srcIsDir {
// If source is a directory we expect the destination to be non-existent always.
_, err = os.Stat(slashpath.Join(dstVolumeDir, dstPath))
if err == nil {
return errFileAccessDenied
}
if !os.IsNotExist(err) {
return err
}
// Destination does not exist, hence proceed with the rename.
}
if err = os.MkdirAll(slashpath.Dir(slashpath.Join(dstVolumeDir, dstPath)), 0755); err != nil {
// File path cannot be verified since one of the parents is a file.
if strings.Contains(err.Error(), "not a directory") {
return errFileAccessDenied
}
return err
}
err = os.Rename(slashpath.Join(srcVolumeDir, srcPath), slashpath.Join(dstVolumeDir, dstPath))
if err != nil {
if os.IsNotExist(err) {
return errFileNotFound
}
return err
}
return nil
}