mirror of
https://github.com/minio/minio.git
synced 2025-01-25 21:53:16 -05:00
fs: Add proper volume and path validation.
This commit is contained in:
parent
caa35f68fa
commit
8457af5708
56
fs-utils.go
Normal file
56
fs-utils.go
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
/*
|
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// validVolname regexp.
|
||||||
|
var validVolname = regexp.MustCompile(`^.{3,63}$`)
|
||||||
|
|
||||||
|
// isValidVolname verifies a volname name in accordance with object
|
||||||
|
// layer requirements.
|
||||||
|
func isValidVolname(volname string) bool {
|
||||||
|
return validVolname.MatchString(volname)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keeping this as lower bound value supporting Linux, Darwin and Windows operating systems.
|
||||||
|
const pathMax = 4096
|
||||||
|
|
||||||
|
// isValidPath verifies if a path name is in accordance with FS limitations.
|
||||||
|
func isValidPath(path string) bool {
|
||||||
|
// TODO: Make this FSType or Operating system specific.
|
||||||
|
if len(path) > pathMax || len(path) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !utf8.ValidString(path) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// isValidPrefix verifies where the prefix is a valid path.
|
||||||
|
func isValidPrefix(prefix string) bool {
|
||||||
|
// Prefix can be empty.
|
||||||
|
if prefix == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// Verify if prefix is a valid path.
|
||||||
|
return isValidPath(prefix)
|
||||||
|
}
|
241
fs.go
241
fs.go
@ -121,26 +121,63 @@ func checkDiskFree(diskPath string, minFreeDisk int64) (err error) {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkVolumeArg - will convert incoming volume names to
|
||||||
|
// corresponding valid volume names on the backend in a platform
|
||||||
|
// compatible way for all operating systems. If volume is not found
|
||||||
|
// an error is generated.
|
||||||
|
func (s fsStorage) checkVolumeArg(volume string) (string, error) {
|
||||||
|
if !isValidVolname(volume) {
|
||||||
|
return "", errInvalidArgument
|
||||||
|
}
|
||||||
|
volumeDir := filepath.Join(s.diskPath, volume)
|
||||||
|
_, err := os.Stat(volumeDir)
|
||||||
|
if err == nil {
|
||||||
|
return volumeDir, nil
|
||||||
|
}
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
var volumes []os.FileInfo
|
||||||
|
volumes, err = ioutil.ReadDir(s.diskPath)
|
||||||
|
if err != nil {
|
||||||
|
return volumeDir, errVolumeNotFound
|
||||||
|
}
|
||||||
|
for _, vol := range volumes {
|
||||||
|
if vol.IsDir() {
|
||||||
|
// Verify if lowercase version of the volume
|
||||||
|
// is equal to the incoming volume, then use the proper name.
|
||||||
|
if strings.ToLower(vol.Name()) == volume {
|
||||||
|
volumeDir = filepath.Join(s.diskPath, vol.Name())
|
||||||
|
return volumeDir, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return volumeDir, errVolumeNotFound
|
||||||
|
} else if os.IsPermission(err) {
|
||||||
|
return volumeDir, errVolumeAccessDenied
|
||||||
|
}
|
||||||
|
return volumeDir, err
|
||||||
|
}
|
||||||
|
|
||||||
// Make a volume entry.
|
// Make a volume entry.
|
||||||
func (s fsStorage) MakeVol(volume string) (err error) {
|
func (s fsStorage) MakeVol(volume string) (err error) {
|
||||||
if volume == "" {
|
volumeDir, err := s.checkVolumeArg(volume)
|
||||||
return errInvalidArgument
|
if err == nil {
|
||||||
}
|
// Volume already exists, return error.
|
||||||
if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
volumeDir := getVolumeDir(s.diskPath, volume)
|
|
||||||
if _, err = os.Stat(volumeDir); err == nil {
|
|
||||||
return errVolumeExists
|
return errVolumeExists
|
||||||
}
|
}
|
||||||
|
|
||||||
// Make a volume entry.
|
// Validate if disk is free.
|
||||||
if err = os.Mkdir(volumeDir, 0700); err != nil {
|
if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil {
|
||||||
return err
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
// If volume not found create it.
|
||||||
|
if err == errVolumeNotFound {
|
||||||
|
// Make a volume entry.
|
||||||
|
return os.Mkdir(volumeDir, 0700)
|
||||||
|
}
|
||||||
|
|
||||||
|
// For all other errors return here.
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeDuplicateVols - remove duplicate volumes.
|
// removeDuplicateVols - remove duplicate volumes.
|
||||||
@ -175,9 +212,15 @@ func (s fsStorage) ListVols() (volsInfo []VolInfo, err error) {
|
|||||||
// If not directory, ignore all file types.
|
// If not directory, ignore all file types.
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// Volname on case sensitive fs backends can come in as
|
||||||
|
// capitalized, but object layer cannot consume it
|
||||||
|
// directly. Convert it as we see fit.
|
||||||
|
volName := strings.ToLower(file.Name())
|
||||||
|
// Modtime is used as created time.
|
||||||
|
createdTime := file.ModTime()
|
||||||
volInfo := VolInfo{
|
volInfo := VolInfo{
|
||||||
Name: file.Name(),
|
Name: volName,
|
||||||
Created: file.ModTime(),
|
Created: createdTime,
|
||||||
}
|
}
|
||||||
volsInfo = append(volsInfo, volInfo)
|
volsInfo = append(volsInfo, volInfo)
|
||||||
}
|
}
|
||||||
@ -186,30 +229,13 @@ func (s fsStorage) ListVols() (volsInfo []VolInfo, err error) {
|
|||||||
return volsInfo, nil
|
return volsInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getVolumeDir - will convert incoming volume names to
|
|
||||||
// corresponding valid volume names on the backend in a platform
|
|
||||||
// compatible way for all operating systems.
|
|
||||||
func getVolumeDir(diskPath, volume string) string {
|
|
||||||
volumes, e := ioutil.ReadDir(diskPath)
|
|
||||||
if e != nil {
|
|
||||||
return volume
|
|
||||||
}
|
|
||||||
for _, vol := range volumes {
|
|
||||||
// Verify if lowercase version of the volume
|
|
||||||
// is equal to the incoming volume, then use the proper name.
|
|
||||||
if strings.ToLower(vol.Name()) == volume {
|
|
||||||
return filepath.Join(diskPath, vol.Name())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return filepath.Join(diskPath, volume)
|
|
||||||
}
|
|
||||||
|
|
||||||
// StatVol - get volume info.
|
// StatVol - get volume info.
|
||||||
func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) {
|
func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) {
|
||||||
if volume == "" {
|
// Verify if volume is valid and it exists.
|
||||||
return VolInfo{}, errInvalidArgument
|
volumeDir, err := s.checkVolumeArg(volume)
|
||||||
|
if err != nil {
|
||||||
|
return VolInfo{}, err
|
||||||
}
|
}
|
||||||
volumeDir := getVolumeDir(s.diskPath, volume)
|
|
||||||
// Stat a volume entry.
|
// Stat a volume entry.
|
||||||
var st os.FileInfo
|
var st os.FileInfo
|
||||||
st, err = os.Stat(volumeDir)
|
st, err = os.Stat(volumeDir)
|
||||||
@ -219,18 +245,23 @@ func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) {
|
|||||||
}
|
}
|
||||||
return VolInfo{}, err
|
return VolInfo{}, err
|
||||||
}
|
}
|
||||||
|
// Modtime is used as created time since operating systems lack a
|
||||||
|
// portable way of knowing the actual created time of a directory.
|
||||||
|
createdTime := st.ModTime()
|
||||||
return VolInfo{
|
return VolInfo{
|
||||||
Name: st.Name(),
|
Name: volume,
|
||||||
Created: st.ModTime(),
|
Created: createdTime,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteVol - delete a volume.
|
// DeleteVol - delete a volume.
|
||||||
func (s fsStorage) DeleteVol(volume string) error {
|
func (s fsStorage) DeleteVol(volume string) error {
|
||||||
if volume == "" {
|
// Verify if volume is valid and it exists.
|
||||||
return errInvalidArgument
|
volumeDir, err := s.checkVolumeArg(volume)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
err := os.Remove(getVolumeDir(s.diskPath, volume))
|
err = os.Remove(volumeDir)
|
||||||
if err != nil && os.IsNotExist(err) {
|
if err != nil && os.IsNotExist(err) {
|
||||||
return errVolumeNotFound
|
return errVolumeNotFound
|
||||||
}
|
}
|
||||||
@ -281,23 +312,13 @@ var specialPrefixes = []string{
|
|||||||
|
|
||||||
// List operation.
|
// List operation.
|
||||||
func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, count int) ([]FileInfo, bool, error) {
|
func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, count int) ([]FileInfo, bool, error) {
|
||||||
if volume == "" {
|
// Verify if volume is valid and it exists.
|
||||||
return nil, true, errInvalidArgument
|
volumeDir, err := s.checkVolumeArg(volume)
|
||||||
|
if err != nil {
|
||||||
|
return nil, true, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var fileInfos []FileInfo
|
var fileInfos []FileInfo
|
||||||
|
|
||||||
volumeDir := getVolumeDir(s.diskPath, volume)
|
|
||||||
// Verify if volume directory exists
|
|
||||||
if exists, err := isDirExist(volumeDir); !exists {
|
|
||||||
if err == nil {
|
|
||||||
return nil, true, errVolumeNotFound
|
|
||||||
} else if os.IsNotExist(err) {
|
|
||||||
return nil, true, errVolumeNotFound
|
|
||||||
} else {
|
|
||||||
return nil, true, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if marker != "" {
|
if marker != "" {
|
||||||
// Verify if marker has prefix.
|
// Verify if marker has prefix.
|
||||||
if marker != "" && !strings.HasPrefix(marker, prefix) {
|
if marker != "" && !strings.HasPrefix(marker, prefix) {
|
||||||
@ -323,6 +344,7 @@ func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, coun
|
|||||||
// Prefix does not exist, not an error just respond empty list response.
|
// Prefix does not exist, not an error just respond empty list response.
|
||||||
return nil, true, nil
|
return nil, true, nil
|
||||||
} else if strings.Contains(err.Error(), "not a directory") {
|
} else if strings.Contains(err.Error(), "not a directory") {
|
||||||
|
// Prefix exists as a file.
|
||||||
return nil, true, nil
|
return nil, true, nil
|
||||||
}
|
}
|
||||||
// Rest errors should be treated as failure.
|
// Rest errors should be treated as failure.
|
||||||
@ -375,26 +397,18 @@ func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, coun
|
|||||||
|
|
||||||
// ReadFile - read a file at a given offset.
|
// ReadFile - read a file at a given offset.
|
||||||
func (s fsStorage) ReadFile(volume string, path string, offset int64) (readCloser io.ReadCloser, err error) {
|
func (s fsStorage) ReadFile(volume string, path string, offset int64) (readCloser io.ReadCloser, err error) {
|
||||||
if volume == "" || path == "" {
|
volumeDir, err := s.checkVolumeArg(volume)
|
||||||
return nil, errInvalidArgument
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
volumeDir := getVolumeDir(s.diskPath, volume)
|
|
||||||
// Verify if volume directory exists
|
filePath := filepath.Join(volumeDir, filepath.FromSlash(path))
|
||||||
var exists bool
|
|
||||||
if exists, err = isDirExist(volumeDir); !exists {
|
|
||||||
if err == nil {
|
|
||||||
return nil, errVolumeNotFound
|
|
||||||
} else if os.IsNotExist(err) {
|
|
||||||
return nil, errVolumeNotFound
|
|
||||||
} else {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
filePath := filepath.Join(volumeDir, path)
|
|
||||||
file, err := os.Open(filePath)
|
file, err := os.Open(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return nil, errFileNotFound
|
return nil, errFileNotFound
|
||||||
|
} else if os.IsPermission(err) {
|
||||||
|
return nil, errFileAccessDenied
|
||||||
}
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -416,22 +430,12 @@ func (s fsStorage) ReadFile(volume string, path string, offset int64) (readClose
|
|||||||
|
|
||||||
// CreateFile - create a file at path.
|
// CreateFile - create a file at path.
|
||||||
func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) {
|
func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) {
|
||||||
if volume == "" || path == "" {
|
volumeDir, err := s.checkVolumeArg(volume)
|
||||||
return nil, errInvalidArgument
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil {
|
if err := checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
|
||||||
return nil, e
|
return nil, err
|
||||||
}
|
|
||||||
volumeDir := getVolumeDir(s.diskPath, volume)
|
|
||||||
// Verify if volume directory exists
|
|
||||||
if exists, err := isDirExist(volumeDir); !exists {
|
|
||||||
if err == nil {
|
|
||||||
return nil, errVolumeNotFound
|
|
||||||
} else if os.IsNotExist(err) {
|
|
||||||
return nil, errVolumeNotFound
|
|
||||||
} else {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
filePath := filepath.Join(volumeDir, path)
|
filePath := filepath.Join(volumeDir, path)
|
||||||
// Verify if the file already exists and is not of regular type.
|
// Verify if the file already exists and is not of regular type.
|
||||||
@ -445,33 +449,26 @@ func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser,
|
|||||||
|
|
||||||
// StatFile - get file info.
|
// StatFile - get file info.
|
||||||
func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) {
|
func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) {
|
||||||
if volume == "" || path == "" {
|
volumeDir, err := s.checkVolumeArg(volume)
|
||||||
return FileInfo{}, errInvalidArgument
|
if err != nil {
|
||||||
}
|
return FileInfo{}, err
|
||||||
volumeDir := getVolumeDir(s.diskPath, volume)
|
|
||||||
// Verify if volume directory exists
|
|
||||||
var exists bool
|
|
||||||
if exists, err = isDirExist(volumeDir); !exists {
|
|
||||||
if err == nil {
|
|
||||||
return FileInfo{}, errVolumeNotFound
|
|
||||||
} else if os.IsNotExist(err) {
|
|
||||||
return FileInfo{}, errVolumeNotFound
|
|
||||||
} else {
|
|
||||||
return FileInfo{}, err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
filePath := filepath.Join(volumeDir, path)
|
filePath := filepath.Join(volumeDir, filepath.FromSlash(path))
|
||||||
st, err := os.Stat(filePath)
|
st, err := os.Stat(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
// File is really not found.
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return FileInfo{}, errFileNotFound
|
return FileInfo{}, errFileNotFound
|
||||||
}
|
}
|
||||||
|
// File path cannot be verified since one of the parents is a file.
|
||||||
if strings.Contains(err.Error(), "not a directory") {
|
if strings.Contains(err.Error(), "not a directory") {
|
||||||
return FileInfo{}, errIsNotRegular
|
return FileInfo{}, errIsNotRegular
|
||||||
}
|
}
|
||||||
|
// Return all errors here.
|
||||||
return FileInfo{}, err
|
return FileInfo{}, err
|
||||||
}
|
}
|
||||||
|
// If its a directory its not a regular file.
|
||||||
if st.Mode().IsDir() {
|
if st.Mode().IsDir() {
|
||||||
return FileInfo{}, errIsNotRegular
|
return FileInfo{}, errIsNotRegular
|
||||||
}
|
}
|
||||||
@ -486,49 +483,55 @@ func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// deleteFile - delete file path if its empty.
|
// deleteFile - delete file path if its empty.
|
||||||
func deleteFile(basePath, deletePath, volume, path string) error {
|
func deleteFile(basePath, deletePath string) error {
|
||||||
if basePath == deletePath {
|
if basePath == deletePath {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// Verify if the path exists.
|
// Verify if the path exists.
|
||||||
pathSt, e := os.Stat(deletePath)
|
pathSt, err := os.Stat(deletePath)
|
||||||
if e != nil {
|
if err != nil {
|
||||||
return e
|
if os.IsNotExist(err) {
|
||||||
|
return errFileNotFound
|
||||||
|
} else if os.IsPermission(err) {
|
||||||
|
return errFileAccessDenied
|
||||||
|
}
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
if pathSt.IsDir() {
|
if pathSt.IsDir() {
|
||||||
// Verify if directory is empty.
|
// Verify if directory is empty.
|
||||||
empty, e := isDirEmpty(deletePath)
|
empty, err := isDirEmpty(deletePath)
|
||||||
if e != nil {
|
if err != nil {
|
||||||
return e
|
return err
|
||||||
}
|
}
|
||||||
if !empty {
|
if !empty {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Attempt to remove path.
|
// Attempt to remove path.
|
||||||
if e := os.Remove(deletePath); e != nil {
|
if err := os.Remove(deletePath); err != nil {
|
||||||
return e
|
return err
|
||||||
}
|
}
|
||||||
// Recursively go down the next path and delete again.
|
// Recursively go down the next path and delete again.
|
||||||
if e := deleteFile(basePath, filepath.Dir(deletePath), volume, path); e != nil {
|
if err := deleteFile(basePath, filepath.Dir(deletePath)); err != nil {
|
||||||
return e
|
return err
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteFile - delete a file at path.
|
// DeleteFile - delete a file at path.
|
||||||
func (s fsStorage) DeleteFile(volume, path string) error {
|
func (s fsStorage) DeleteFile(volume, path string) error {
|
||||||
if volume == "" || path == "" {
|
volumeDir, err := s.checkVolumeArg(volume)
|
||||||
return errInvalidArgument
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
volumeDir := getVolumeDir(s.diskPath, volume)
|
|
||||||
|
|
||||||
// Following code is needed so that we retain "/" suffix if any in
|
// Following code is needed so that we retain "/" suffix if any in
|
||||||
// path argument. Do not use filepath.Join() since it would strip
|
// path argument.
|
||||||
// off any suffixes.
|
filePath := filepath.Join(volumeDir, filepath.FromSlash(path))
|
||||||
filePath := s.diskPath + string(os.PathSeparator) + volume + string(os.PathSeparator) + path
|
if strings.HasSuffix(filepath.FromSlash(path), string(os.PathSeparator)) {
|
||||||
|
filePath = filePath + string(os.PathSeparator)
|
||||||
|
}
|
||||||
|
|
||||||
// Delete file and delete parent directory as well if its empty.
|
// Delete file and delete parent directory as well if its empty.
|
||||||
return deleteFile(volumeDir, filePath, volume, path)
|
return deleteFile(volumeDir, filePath)
|
||||||
}
|
}
|
||||||
|
@ -22,7 +22,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@ -111,10 +110,13 @@ func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarke
|
|||||||
}
|
}
|
||||||
result.IsTruncated = true
|
result.IsTruncated = true
|
||||||
newMaxUploads := 0
|
newMaxUploads := 0
|
||||||
prefixPath := bucket + slashPathSeparator + prefix // do not use filepath.Join so that we retain trailing '/' if any
|
prefixPath := path.Join(bucket, prefix)
|
||||||
|
if strings.HasSuffix(prefix, slashPathSeparator) {
|
||||||
|
// Add back the slash separator removed after 'path.Join'.
|
||||||
|
prefixPath = prefixPath + slashPathSeparator
|
||||||
|
}
|
||||||
if recursive {
|
if recursive {
|
||||||
keyMarkerPath := filepath.Join(keyMarker, uploadIDMarker)
|
keyMarkerPath := path.Join(keyMarker, uploadIDMarker)
|
||||||
outerLoop:
|
outerLoop:
|
||||||
for {
|
for {
|
||||||
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, keyMarkerPath, recursive, maxUploads-newMaxUploads)
|
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, keyMarkerPath, recursive, maxUploads-newMaxUploads)
|
||||||
@ -123,17 +125,17 @@ func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarke
|
|||||||
}
|
}
|
||||||
for _, fi := range fileInfos {
|
for _, fi := range fileInfos {
|
||||||
keyMarkerPath = fi.Name
|
keyMarkerPath = fi.Name
|
||||||
fileName := filepath.Base(fi.Name)
|
fileName := path.Base(fi.Name)
|
||||||
if strings.Contains(fileName, ".") {
|
if strings.Contains(fileName, ".") {
|
||||||
// fileName contains partnumber and md5sum info, skip this.
|
// fileName contains partnumber and md5sum info, skip this.
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
result.Uploads = append(result.Uploads, uploadMetadata{
|
result.Uploads = append(result.Uploads, uploadMetadata{
|
||||||
Object: filepath.Dir(fi.Name),
|
Object: path.Dir(fi.Name),
|
||||||
UploadID: fileName,
|
UploadID: fileName,
|
||||||
Initiated: fi.ModTime,
|
Initiated: fi.ModTime,
|
||||||
})
|
})
|
||||||
result.NextKeyMarker = filepath.Dir(fi.Name)
|
result.NextKeyMarker = path.Dir(fi.Name)
|
||||||
result.NextUploadIDMarker = fileName
|
result.NextUploadIDMarker = fileName
|
||||||
newMaxUploads++
|
newMaxUploads++
|
||||||
if newMaxUploads == maxUploads {
|
if newMaxUploads == maxUploads {
|
||||||
@ -244,7 +246,7 @@ func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Err
|
|||||||
return "", probe.NewError(e)
|
return "", probe.NewError(e)
|
||||||
}
|
}
|
||||||
uploadID := uuid.String()
|
uploadID := uuid.String()
|
||||||
uploadIDFile := filepath.Join(bucket, object, uploadID)
|
uploadIDFile := path.Join(bucket, object, uploadID)
|
||||||
if _, e = o.storage.StatFile(minioMetaVolume, uploadIDFile); e != nil {
|
if _, e = o.storage.StatFile(minioMetaVolume, uploadIDFile); e != nil {
|
||||||
if e != errFileNotFound {
|
if e != errFileNotFound {
|
||||||
return "", probe.NewError(e)
|
return "", probe.NewError(e)
|
||||||
@ -266,7 +268,7 @@ func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Err
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (o objectAPI) isUploadIDExist(bucket, object, uploadID string) (bool, error) {
|
func (o objectAPI) isUploadIDExist(bucket, object, uploadID string) (bool, error) {
|
||||||
st, e := o.storage.StatFile(minioMetaVolume, filepath.Join(bucket, object, uploadID))
|
st, e := o.storage.StatFile(minioMetaVolume, path.Join(bucket, object, uploadID))
|
||||||
if e != nil {
|
if e != nil {
|
||||||
if e == errFileNotFound {
|
if e == errFileNotFound {
|
||||||
return false, nil
|
return false, nil
|
||||||
@ -291,7 +293,7 @@ func (o objectAPI) PutObjectPart(bucket, object, uploadID string, partID int, si
|
|||||||
}
|
}
|
||||||
|
|
||||||
partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partID, md5Hex)
|
partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partID, md5Hex)
|
||||||
fileWriter, e := o.storage.CreateFile(minioMetaVolume, filepath.Join(bucket, object, partSuffix))
|
fileWriter, e := o.storage.CreateFile(minioMetaVolume, path.Join(bucket, object, partSuffix))
|
||||||
if e != nil {
|
if e != nil {
|
||||||
if e == errVolumeNotFound {
|
if e == errVolumeNotFound {
|
||||||
return "", probe.NewError(BucketNotFound{
|
return "", probe.NewError(BucketNotFound{
|
||||||
@ -356,7 +358,7 @@ func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMa
|
|||||||
marker := ""
|
marker := ""
|
||||||
nextPartNumberMarker := 0
|
nextPartNumberMarker := 0
|
||||||
if partNumberMarker > 0 {
|
if partNumberMarker > 0 {
|
||||||
fileInfos, _, e := o.storage.ListFiles(minioMetaVolume, filepath.Join(bucket, object, uploadID)+"."+strconv.Itoa(partNumberMarker)+".", "", false, 1)
|
fileInfos, _, e := o.storage.ListFiles(minioMetaVolume, path.Join(bucket, object, uploadID)+"."+strconv.Itoa(partNumberMarker)+".", "", false, 1)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return result, probe.NewError(e)
|
return result, probe.NewError(e)
|
||||||
}
|
}
|
||||||
@ -365,12 +367,12 @@ func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMa
|
|||||||
}
|
}
|
||||||
marker = fileInfos[0].Name
|
marker = fileInfos[0].Name
|
||||||
}
|
}
|
||||||
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, filepath.Join(bucket, object, uploadID)+".", marker, false, maxParts)
|
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, path.Join(bucket, object, uploadID)+".", marker, false, maxParts)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return result, probe.NewError(InvalidPart{})
|
return result, probe.NewError(InvalidPart{})
|
||||||
}
|
}
|
||||||
for _, fileInfo := range fileInfos {
|
for _, fileInfo := range fileInfos {
|
||||||
fileName := filepath.Base(fileInfo.Name)
|
fileName := path.Base(fileInfo.Name)
|
||||||
splitResult := strings.Split(fileName, ".")
|
splitResult := strings.Split(fileName, ".")
|
||||||
partNum, e := strconv.Atoi(splitResult[1])
|
partNum, e := strconv.Atoi(splitResult[1])
|
||||||
if e != nil {
|
if e != nil {
|
||||||
@ -415,7 +417,7 @@ func (o objectAPI) CompleteMultipartUpload(bucket string, object string, uploadI
|
|||||||
for _, part := range parts {
|
for _, part := range parts {
|
||||||
partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, part.PartNumber, part.ETag)
|
partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, part.PartNumber, part.ETag)
|
||||||
var fileReader io.ReadCloser
|
var fileReader io.ReadCloser
|
||||||
fileReader, e = o.storage.ReadFile(minioMetaVolume, filepath.Join(bucket, object, partSuffix), 0)
|
fileReader, e = o.storage.ReadFile(minioMetaVolume, path.Join(bucket, object, partSuffix), 0)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return ObjectInfo{}, probe.NewError(e)
|
return ObjectInfo{}, probe.NewError(e)
|
||||||
}
|
}
|
||||||
@ -456,7 +458,8 @@ func (o objectAPI) removeMultipartUpload(bucket, object, uploadID string) *probe
|
|||||||
}
|
}
|
||||||
marker := ""
|
marker := ""
|
||||||
for {
|
for {
|
||||||
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, filepath.Join(bucket, object, uploadID), marker, false, 1000)
|
uploadIDFile := path.Join(bucket, object, uploadID)
|
||||||
|
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, uploadIDFile, marker, false, 1000)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
|
return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
|
||||||
}
|
}
|
||||||
|
@ -80,10 +80,15 @@ func (o objectAPI) ListBuckets() ([]BucketInfo, *probe.Error) {
|
|||||||
return nil, probe.NewError(e)
|
return nil, probe.NewError(e)
|
||||||
}
|
}
|
||||||
for _, vol := range vols {
|
for _, vol := range vols {
|
||||||
|
// StorageAPI can send volume names which are incompatible
|
||||||
|
// with buckets, handle it and skip them.
|
||||||
if !IsValidBucketName(vol.Name) {
|
if !IsValidBucketName(vol.Name) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bucketInfos = append(bucketInfos, BucketInfo{vol.Name, vol.Created})
|
bucketInfos = append(bucketInfos, BucketInfo{
|
||||||
|
Name: vol.Name,
|
||||||
|
Created: vol.Created,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
return bucketInfos, nil
|
return bucketInfos, nil
|
||||||
}
|
}
|
||||||
|
@ -1012,7 +1012,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
|||||||
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
|
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Println(string(completeMultipartBytes))
|
|
||||||
complMultipartUpload := &completeMultipartUpload{}
|
complMultipartUpload := &completeMultipartUpload{}
|
||||||
if e = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); e != nil {
|
if e = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); e != nil {
|
||||||
errorIf(probe.NewError(e), "XML Unmarshal failed", nil)
|
errorIf(probe.NewError(e), "XML Unmarshal failed", nil)
|
||||||
|
@ -18,6 +18,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"strings"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -41,7 +42,27 @@ func IsValidBucketName(bucket string) bool {
|
|||||||
// IsValidObjectName verifies an object name in accordance with Amazon's
|
// IsValidObjectName verifies an object name in accordance with Amazon's
|
||||||
// requirements. It cannot exceed 1024 characters and must be a valid UTF8
|
// requirements. It cannot exceed 1024 characters and must be a valid UTF8
|
||||||
// string.
|
// string.
|
||||||
// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
|
//
|
||||||
|
// See:
|
||||||
|
// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
|
||||||
|
//
|
||||||
|
// You should avoid the following characters in a key name because of
|
||||||
|
// significant special handling for consistency across all
|
||||||
|
// applications.
|
||||||
|
//
|
||||||
|
// Rejects strings with following characters.
|
||||||
|
//
|
||||||
|
// - Backslash ("\")
|
||||||
|
// - Left curly brace ("{")
|
||||||
|
// - Caret ("^")
|
||||||
|
// - Right curly brace ("}")
|
||||||
|
// - Grave accent / back tick ("`")
|
||||||
|
// - Right square bracket ("]")
|
||||||
|
// - Left square bracket ("[")
|
||||||
|
// - Tilde ("~")
|
||||||
|
// - 'Greater Than' symbol (">")
|
||||||
|
// - 'Less Than' symbol ("<")
|
||||||
|
// - Vertical bar / pipe ("|")
|
||||||
func IsValidObjectName(object string) bool {
|
func IsValidObjectName(object string) bool {
|
||||||
if len(object) > 1024 || len(object) == 0 {
|
if len(object) > 1024 || len(object) == 0 {
|
||||||
return false
|
return false
|
||||||
@ -49,7 +70,8 @@ func IsValidObjectName(object string) bool {
|
|||||||
if !utf8.ValidString(object) {
|
if !utf8.ValidString(object) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return true
|
// Reject unsupported characters in object name.
|
||||||
|
return !strings.ContainsAny(object, "`^*{}[]|\\\"'")
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsValidObjectPrefix verifies whether the prefix is a valid object name.
|
// IsValidObjectPrefix verifies whether the prefix is a valid object name.
|
||||||
|
@ -1172,6 +1172,7 @@ func (s *MyAPISuite) TestValidateObjectMultipartUploadID(c *C) {
|
|||||||
c.Assert(err, IsNil)
|
c.Assert(err, IsNil)
|
||||||
|
|
||||||
response, err = client.Do(request)
|
response, err = client.Do(request)
|
||||||
|
c.Assert(err, IsNil)
|
||||||
c.Assert(response.StatusCode, Equals, http.StatusOK)
|
c.Assert(response.StatusCode, Equals, http.StatusOK)
|
||||||
|
|
||||||
decoder := xml.NewDecoder(response.Body)
|
decoder := xml.NewDecoder(response.Body)
|
||||||
|
@ -32,3 +32,11 @@ var errIsNotRegular = errors.New("Not a regular file type.")
|
|||||||
|
|
||||||
// errVolumeNotFound - cannot find the volume.
|
// errVolumeNotFound - cannot find the volume.
|
||||||
var errVolumeNotFound = errors.New("Volume not found.")
|
var errVolumeNotFound = errors.New("Volume not found.")
|
||||||
|
|
||||||
|
// errVolumeAccessDenied - cannot access volume, insufficient
|
||||||
|
// permissions.
|
||||||
|
var errVolumeAccessDenied = errors.New("Volume access denied.")
|
||||||
|
|
||||||
|
// errVolumeAccessDenied - cannot access file, insufficient
|
||||||
|
// permissions.
|
||||||
|
var errFileAccessDenied = errors.New("File access denied.")
|
||||||
|
Loading…
x
Reference in New Issue
Block a user