diff --git a/api-headers.go b/api-headers.go index 4eb95681f..64a025817 100644 --- a/api-headers.go +++ b/api-headers.go @@ -63,7 +63,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h setCommonHeaders(w) // set object-related metadata headers - lastModified := objInfo.ModifiedTime.UTC().Format(http.TimeFormat) + lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat) w.Header().Set("Last-Modified", lastModified) w.Header().Set("Content-Type", objInfo.ContentType) diff --git a/api-response.go b/api-response.go index 11a3ba351..0fc424387 100644 --- a/api-response.go +++ b/api-response.go @@ -260,7 +260,7 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe continue } content.Key = object.Name - content.LastModified = object.ModifiedTime.UTC().Format(timeFormatAMZ) + content.LastModified = object.ModTime.UTC().Format(timeFormatAMZ) if object.MD5Sum != "" { content.ETag = "\"" + object.MD5Sum + "\"" } diff --git a/api-router.go b/api-router.go index 50986f687..b263a20c3 100644 --- a/api-router.go +++ b/api-router.go @@ -18,13 +18,13 @@ package main import router "github.com/gorilla/mux" -// objectStorageAPI container for S3 compatible API. -type objectStorageAPI struct { - ObjectAPI ObjectAPI +// objectAPIHandler implements and provides http handlers for S3 API. +type objectAPIHandlers struct { + ObjectAPI *objectAPI } // registerAPIRouter - registers S3 compatible APIs. -func registerAPIRouter(mux *router.Router, api objectStorageAPI) { +func registerAPIRouter(mux *router.Router, api objectAPIHandlers) { // API Router apiRouter := mux.NewRoute().PathPrefix("/").Subrouter() diff --git a/bucket-handlers.go b/bucket-handlers.go index b4c92a349..22401f1ed 100644 --- a/bucket-handlers.go +++ b/bucket-handlers.go @@ -74,7 +74,7 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error // GetBucketLocationHandler - GET Bucket location. // ------------------------- // This operation returns bucket location. -func (api objectStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -152,7 +152,7 @@ func (api objectStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *h // completed or aborted. This operation returns at most 1,000 multipart // uploads in the response. // -func (api objectStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -223,7 +223,7 @@ func (api objectStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r // of the objects in a bucket. You can use the request parameters as selection // criteria to return a subset of the objects in a bucket. // -func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -289,8 +289,6 @@ func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Re writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) case BucketNotFound: writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) - case ObjectNotFound: - writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) case ObjectNameInvalid: writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) default: @@ -303,7 +301,7 @@ func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Re // ----------- // This implementation of the GET operation returns a list of all buckets // owned by the authenticated sender of the request. -func (api objectStorageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { // List buckets does not support bucket policies. switch getRequestAuthType(r) { default: @@ -354,7 +352,7 @@ func (api objectStorageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Re } // DeleteMultipleObjectsHandler - deletes multiple objects. -func (api objectStorageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -465,7 +463,7 @@ func (api objectStorageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, // PutBucketHandler - PUT Bucket // ---------- // This implementation of the PUT operation creates a new bucket for authenticated request -func (api objectStorageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -530,7 +528,7 @@ func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]stri // ---------- // This implementation of the POST operation handles object creation with a specified // signature policy in multipart/form-data -func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { // Here the parameter is the size of the form data that should // be loaded in memory, the remaining being put in temporary files. reader, e := r.MultipartReader() @@ -560,7 +558,7 @@ func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *ht writeErrorResponse(w, r, apiErr, r.URL.Path) return } - objInfo, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, nil) + md5Sum, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, nil) if err != nil { errorIf(err.Trace(), "PutObject failed.", nil) switch err.ToGoError().(type) { @@ -579,8 +577,8 @@ func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *ht } return } - if objInfo.MD5Sum != "" { - w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"") + if md5Sum != "" { + w.Header().Set("ETag", "\""+md5Sum+"\"") } writeSuccessResponse(w, nil) } @@ -591,7 +589,7 @@ func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *ht // The operation returns a 200 OK if the bucket exists and you // have permission to access it. Otherwise, the operation might // return responses such as 404 Not Found and 403 Forbidden. -func (api objectStorageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -630,7 +628,7 @@ func (api objectStorageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Req } // DeleteBucketHandler - Delete bucket -func (api objectStorageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] diff --git a/bucket-policy-handlers.go b/bucket-policy-handlers.go index eab8949a7..35f674d81 100644 --- a/bucket-policy-handlers.go +++ b/bucket-policy-handlers.go @@ -127,7 +127,7 @@ func bucketPolicyConditionMatch(conditions map[string]string, statement policySt // ----------------- // This implementation of the PUT operation uses the policy // subresource to add to or replace a policy on a bucket -func (api objectStorageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -201,7 +201,7 @@ func (api objectStorageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *htt // ----------------- // This implementation of the DELETE operation uses the policy // subresource to add to remove a policy on a bucket. -func (api objectStorageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] @@ -238,7 +238,7 @@ func (api objectStorageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r * // ----------------- // This operation uses the policy // subresource to return the policy of a specified bucket. -func (api objectStorageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] diff --git a/fs-bucket-listobjects.go b/fs-bucket-listobjects.go deleted file mode 100644 index c5b234b20..000000000 --- a/fs-bucket-listobjects.go +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2015-2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/minio/minio/pkg/probe" -) - -const ( - // listObjectsLimit - maximum list objects limit. - listObjectsLimit = 1000 -) - -// isDirExist - returns whether given directory is exist or not. -func isDirExist(dirname string) (bool, error) { - fi, e := os.Lstat(dirname) - if e != nil { - if os.IsNotExist(e) { - return false, nil - } - return false, e - } - return fi.IsDir(), nil -} - -func (fs *Filesystem) saveTreeWalk(params listObjectParams, walker *treeWalker) { - fs.listObjectMapMutex.Lock() - defer fs.listObjectMapMutex.Unlock() - - walkers, _ := fs.listObjectMap[params] - walkers = append(walkers, walker) - - fs.listObjectMap[params] = walkers -} - -func (fs *Filesystem) lookupTreeWalk(params listObjectParams) *treeWalker { - fs.listObjectMapMutex.Lock() - defer fs.listObjectMapMutex.Unlock() - - if walkChs, ok := fs.listObjectMap[params]; ok { - for i, walkCh := range walkChs { - if !walkCh.timedOut { - newWalkChs := walkChs[i+1:] - if len(newWalkChs) > 0 { - fs.listObjectMap[params] = newWalkChs - } else { - delete(fs.listObjectMap, params) - } - return walkCh - } - } - // As all channels are timed out, delete the map entry - delete(fs.listObjectMap, params) - } - return nil -} - -// ListObjects - lists all objects for a given prefix, returns up to -// maxKeys number of objects per call. -func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error) { - result := ListObjectsInfo{} - - // Input validation. - bucket, e := fs.checkBucketArg(bucket) - if e != nil { - return result, probe.NewError(e) - } - bucketDir := filepath.Join(fs.diskPath, bucket) - - if !IsValidObjectPrefix(prefix) { - return result, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) - } - - // Verify if delimiter is anything other than '/', which we do not support. - if delimiter != "" && delimiter != "/" { - return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported. Only '/' is supported", delimiter)) - } - - // Verify if marker has prefix. - if marker != "" { - if !strings.HasPrefix(marker, prefix) { - return result, probe.NewError(fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", marker, prefix)) - } - } - - // Return empty response for a valid request when maxKeys is 0. - if maxKeys == 0 { - return result, nil - } - - // Over flowing maxkeys - reset to listObjectsLimit. - if maxKeys < 0 || maxKeys > listObjectsLimit { - maxKeys = listObjectsLimit - } - - // Verify if prefix exists. - prefixDir := filepath.Dir(filepath.FromSlash(prefix)) - rootDir := filepath.Join(bucketDir, prefixDir) - if status, e := isDirExist(rootDir); !status { - if e == nil { - // Prefix does not exist, not an error just respond empty - // list response. - return result, nil - } - // Rest errors should be treated as failure. - return result, probe.NewError(e) - } - - recursive := true - if delimiter == "/" { - recursive = false - } - - // Maximum 1000 objects returned in a single to listObjects. - // Further calls will set right marker value to continue reading the rest of the objectList. - // popTreeWalker returns nil if the call to ListObject is done for the first time. - // On further calls to ListObjects to retrive more objects within the timeout period, - // popTreeWalker returns the channel from which rest of the objects can be retrieved. - walker := fs.lookupTreeWalk(listObjectParams{bucket, delimiter, marker, prefix}) - if walker == nil { - walker = startTreeWalk(fs.diskPath, bucket, filepath.FromSlash(prefix), filepath.FromSlash(marker), recursive) - } - - nextMarker := "" - for i := 0; i < maxKeys; { - walkResult, ok := <-walker.ch - if !ok { - // Closed channel. - return result, nil - } - // For any walk error return right away. - if walkResult.err != nil { - return ListObjectsInfo{}, probe.NewError(walkResult.err) - } - objInfo := walkResult.objectInfo - objInfo.Name = filepath.ToSlash(objInfo.Name) - - // Skip temporary files. - if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") { - continue - } - - // For objects being directory and delimited we set Prefixes. - if objInfo.IsDir { - result.Prefixes = append(result.Prefixes, objInfo.Name) - } else { - result.Objects = append(result.Objects, objInfo) - } - - // We have listed everything return. - if walkResult.end { - return result, nil - } - nextMarker = objInfo.Name - i++ - } - // We haven't exhaused the list yet, set IsTruncated to 'true' so - // that the client can send another request. - result.IsTruncated = true - result.NextMarker = nextMarker - fs.saveTreeWalk(listObjectParams{bucket, delimiter, nextMarker, prefix}, walker) - return result, nil -} diff --git a/fs-bucket.go b/fs-bucket.go deleted file mode 100644 index bebc36e9e..000000000 --- a/fs-bucket.go +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/minio/minio/pkg/probe" -) - -/// Bucket Operations - -// DeleteBucket - delete a bucket. -func (fs Filesystem) DeleteBucket(bucket string) *probe.Error { - // Verify bucket is valid. - if !IsValidBucketName(bucket) { - return probe.NewError(BucketNameInvalid{Bucket: bucket}) - } - bucket = getActualBucketname(fs.diskPath, bucket) - bucketDir := filepath.Join(fs.diskPath, bucket) - if e := os.Remove(bucketDir); e != nil { - // Error if there was no bucket in the first place. - if os.IsNotExist(e) { - return probe.NewError(BucketNotFound{Bucket: bucket}) - } - // On windows the string is slightly different, handle it here. - if strings.Contains(e.Error(), "directory is not empty") { - return probe.NewError(BucketNotEmpty{Bucket: bucket}) - } - // Hopefully for all other operating systems, this is - // assumed to be consistent. - if strings.Contains(e.Error(), "directory not empty") { - return probe.NewError(BucketNotEmpty{Bucket: bucket}) - } - return probe.NewError(e) - } - return nil -} - -// ListBuckets - Get service. -func (fs Filesystem) ListBuckets() ([]BucketInfo, *probe.Error) { - files, e := ioutil.ReadDir(fs.diskPath) - if e != nil { - return []BucketInfo{}, probe.NewError(e) - } - var buckets []BucketInfo - for _, file := range files { - if !file.IsDir() { - // If not directory, ignore all file types. - continue - } - // If directories are found with odd names, skip them. - dirName := strings.ToLower(file.Name()) - if !IsValidBucketName(dirName) { - continue - } - bucket := BucketInfo{ - Name: dirName, - Created: file.ModTime(), - } - buckets = append(buckets, bucket) - } - // Remove duplicated entries. - buckets = removeDuplicateBuckets(buckets) - return buckets, nil -} - -// removeDuplicateBuckets - remove duplicate buckets. -func removeDuplicateBuckets(buckets []BucketInfo) []BucketInfo { - length := len(buckets) - 1 - for i := 0; i < length; i++ { - for j := i + 1; j <= length; j++ { - if buckets[i].Name == buckets[j].Name { - if buckets[i].Created.Sub(buckets[j].Created) > 0 { - buckets[i] = buckets[length] - } else { - buckets[j] = buckets[length] - } - buckets = buckets[0:length] - length-- - j-- - } - } - } - return buckets -} - -// MakeBucket - PUT Bucket -func (fs Filesystem) MakeBucket(bucket string) *probe.Error { - if _, e := fs.checkBucketArg(bucket); e == nil { - return probe.NewError(BucketExists{Bucket: bucket}) - } else if _, ok := e.(BucketNameInvalid); ok { - return probe.NewError(BucketNameInvalid{Bucket: bucket}) - } - bucketDir := filepath.Join(fs.diskPath, bucket) - - // Make bucket. - if e := os.Mkdir(bucketDir, 0700); e != nil { - return probe.NewError(e) - } - return nil -} - -// getActualBucketname - will convert incoming bucket names to -// corresponding actual bucketnames on the backend in a platform -// compatible way for all operating systems. -func getActualBucketname(fsPath, bucket string) string { - fd, e := os.Open(fsPath) - if e != nil { - return bucket - } - buckets, e := fd.Readdirnames(-1) - if e != nil { - return bucket - } - for _, b := range buckets { - // Verify if lowercase version of the bucket is equal - // to the incoming bucket, then use the proper name. - if strings.ToLower(b) == bucket { - return b - } - } - return bucket -} - -// GetBucketInfo - get bucket metadata. -func (fs Filesystem) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) { - if !IsValidBucketName(bucket) { - return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) - } - bucket = getActualBucketname(fs.diskPath, bucket) - // Get bucket path. - bucketDir := filepath.Join(fs.diskPath, bucket) - fi, e := os.Stat(bucketDir) - if e != nil { - // Check if bucket exists. - if os.IsNotExist(e) { - return BucketInfo{}, probe.NewError(BucketNotFound{Bucket: bucket}) - } - return BucketInfo{}, probe.NewError(e) - } - bucketMetadata := BucketInfo{} - bucketMetadata.Name = fi.Name() - bucketMetadata.Created = fi.ModTime() - return bucketMetadata, nil -} diff --git a/fs-bucket_test.go b/fs-bucket_test.go deleted file mode 100644 index 15b5c1d41..000000000 --- a/fs-bucket_test.go +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "io/ioutil" - "os" - "strconv" - "strings" - "testing" -) - -// The test not just includes asserting the correctness of the output, -// But also includes test cases for which the function should fail. -// For those cases for which it fails, its also asserted whether the function fails as expected. -func TestGetBucketInfo(t *testing.T) { - // Make a temporary directory to use as the fs. - directory, e := ioutil.TempDir("", "minio-metadata-test") - if e != nil { - t.Fatal(e) - } - defer os.RemoveAll(directory) - - // Create the fs. - fs, err := newFS(directory) - if err != nil { - t.Fatal(err) - } - - // Creating few buckets. - for i := 0; i < 4; i++ { - err = fs.MakeBucket("meta-test-bucket." + strconv.Itoa(i)) - if err != nil { - t.Fatal(err) - } - } - testCases := []struct { - bucketName string - metaData BucketInfo - e error - shouldPass bool - }{ - // Test cases with invalid bucket names. - {".test", BucketInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, - {"Test", BucketInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, - {"---", BucketInfo{}, BucketNameInvalid{Bucket: "---"}, false}, - {"ad", BucketInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, - // Test cases with non-existent buckets. - {"volatile-bucket-1", BucketInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, - {"volatile-bucket-2", BucketInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, - // Test cases with existing buckets. - {"meta-test-bucket.0", BucketInfo{Name: "meta-test-bucket.0"}, nil, true}, - {"meta-test-bucket.1", BucketInfo{Name: "meta-test-bucket.1"}, nil, true}, - {"meta-test-bucket.2", BucketInfo{Name: "meta-test-bucket.2"}, nil, true}, - {"meta-test-bucket.3", BucketInfo{Name: "meta-test-bucket.3"}, nil, true}, - } - for i, testCase := range testCases { - // The err returned is of type *probe.Error. - bucketInfo, err := fs.GetBucketInfo(testCase.bucketName) - - if err != nil && testCase.shouldPass { - t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Cause.Error()) - } - if err == nil && !testCase.shouldPass { - t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.e.Error()) - - } - // Failed as expected, but does it fail for the expected reason. - if err != nil && !testCase.shouldPass { - if testCase.e.Error() != err.Cause.Error() { - t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.e.Error(), err.Cause.Error()) - } - } - // Since there are cases for which GetBucketInfo fails, this is necessary. - // Test passes as expected, but the output values are verified for correctness here. - if err == nil && testCase.shouldPass { - if testCase.bucketName != bucketInfo.Name { - t.Errorf("Test %d: Expected the bucket name to be \"%s\", but found \"%s\" instead", i+1, testCase.bucketName, bucketInfo.Name) - } - } - } -} - -func TestListBuckets(t *testing.T) { - // Make a temporary directory to use as the fs. - directory, e := ioutil.TempDir("", "minio-benchmark") - if e != nil { - t.Fatal(e) - } - defer os.RemoveAll(directory) - - // Create the fs. - fs, err := newFS(directory) - if err != nil { - t.Fatal(err) - } - - // Create a few buckets. - for i := 0; i < 10; i++ { - err = fs.MakeBucket("testbucket." + strconv.Itoa(i)) - if err != nil { - t.Fatal(err) - } - } - - // List, and ensure that they are all there. - metadatas, err := fs.ListBuckets() - if err != nil { - t.Fatal(err) - } - - if len(metadatas) != 10 { - t.Errorf("incorrect length of metadatas (%d)\n", len(metadatas)) - } - - // Iterate over the buckets, ensuring that the name is correct. - for i := 0; i < len(metadatas); i++ { - if !strings.Contains(metadatas[i].Name, "testbucket") { - t.Fail() - } - } -} - -func TestDeleteBucket(t *testing.T) { - // Make a temporary directory to use as the fs. - directory, e := ioutil.TempDir("", "minio-benchmark") - if e != nil { - t.Fatal(e) - } - defer os.RemoveAll(directory) - - // Create the fs. - fs, err := newFS(directory) - if err != nil { - t.Fatal(err) - } - - // Deleting a bucket that doesn't exist should error. - err = fs.DeleteBucket("bucket") - if !strings.Contains(err.Cause.Error(), "Bucket not found:") { - t.Fail() - } -} - -func BenchmarkListBuckets(b *testing.B) { - // Make a temporary directory to use as the fs. - directory, e := ioutil.TempDir("", "minio-benchmark") - if e != nil { - b.Fatal(e) - } - defer os.RemoveAll(directory) - - // Create the fs. - fs, err := newFS(directory) - if err != nil { - b.Fatal(err) - } - - // Create a few buckets. - for i := 0; i < 20; i++ { - err = fs.MakeBucket("bucket." + strconv.Itoa(i)) - if err != nil { - b.Fatal(err) - } - } - - b.ResetTimer() - - // List the buckets over and over and over. - for i := 0; i < b.N; i++ { - _, err = fs.ListBuckets() - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkDeleteBucket(b *testing.B) { - // Make a temporary directory to use as the fs. - directory, e := ioutil.TempDir("", "minio-benchmark") - if e != nil { - b.Fatal(e) - } - defer os.RemoveAll(directory) - - // Create the fs. - fs, err := newFS(directory) - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - // Creating buckets takes time, so stop and start the timer. - b.StopTimer() - - // Create and delete the bucket over and over. - err = fs.MakeBucket("bucket") - if err != nil { - b.Fatal(err) - } - - b.StartTimer() - - err = fs.DeleteBucket("bucket") - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkGetBucketInfo(b *testing.B) { - // Make a temporary directory to use as the fs. - directory, e := ioutil.TempDir("", "minio-benchmark") - if e != nil { - b.Fatal(e) - } - defer os.RemoveAll(directory) - - // Create the fs. - fs, err := newFS(directory) - if err != nil { - b.Fatal(err) - } - - // Put up a bucket with some metadata. - err = fs.MakeBucket("bucket") - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - // Retrieve the metadata! - _, err := fs.GetBucketInfo("bucket") - if err != nil { - b.Fatal(err) - } - } -} diff --git a/fs-dir-common.go b/fs-dir-common.go index 96d247a45..6db10d0e8 100644 --- a/fs-dir-common.go +++ b/fs-dir-common.go @@ -54,7 +54,8 @@ func (d byDirentName) Len() int { return len(d) } func (d byDirentName) Swap(i, j int) { d[i], d[j] = d[j], d[i] } func (d byDirentName) Less(i, j int) bool { return d[i].name < d[j].name } -// Using sort.Search() internally to jump to the file entry containing the prefix. +// Using sort.Search() internally to jump to the file entry containing +// the prefix. func searchDirents(dirents []fsDirent, x string) int { processFunc := func(i int) bool { return dirents[i].name >= x @@ -64,9 +65,9 @@ func searchDirents(dirents []fsDirent, x string) int { // Tree walk result carries results of tree walking. type treeWalkResult struct { - objectInfo ObjectInfo - err error - end bool + fileInfo FileInfo + err error + end bool } // Tree walk notify carries a channel which notifies tree walk @@ -77,42 +78,42 @@ type treeWalker struct { timedOut bool } -// treeWalk walks FS directory tree recursively pushing ObjectInfo into the channel as and when it encounters files. +// treeWalk walks FS directory tree recursively pushing fileInfo into the channel as and when it encounters files. func treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker string, recursive bool, send func(treeWalkResult) bool, count *int) bool { // Example: // if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively // called with prefixDir="one/two/three/four/" and marker="five.txt" - // Convert dirent to ObjectInfo - direntToObjectInfo := func(dirent fsDirent) (ObjectInfo, error) { - objectInfo := ObjectInfo{} + // Convert dirent to FileInfo + direntToFileInfo := func(dirent fsDirent) (FileInfo, error) { + fileInfo := FileInfo{} // Convert to full object name. - objectInfo.Name = filepath.Join(prefixDir, dirent.name) + fileInfo.Name = filepath.Join(prefixDir, dirent.name) if dirent.modTime.IsZero() && dirent.size == 0 { // ModifiedTime and Size are zero, Stat() and figure out // the actual values that need to be set. fi, err := os.Stat(filepath.Join(bucketDir, prefixDir, dirent.name)) if err != nil { - return ObjectInfo{}, err + return FileInfo{}, err } // Fill size and modtime. - objectInfo.ModifiedTime = fi.ModTime() - objectInfo.Size = fi.Size() - objectInfo.IsDir = fi.IsDir() + fileInfo.ModTime = fi.ModTime() + fileInfo.Size = fi.Size() + fileInfo.Mode = fi.Mode() } else { - // If ModifiedTime or Size are set then use them + // If ModTime or Size are set then use them // without attempting another Stat operation. - objectInfo.ModifiedTime = dirent.modTime - objectInfo.Size = dirent.size - objectInfo.IsDir = dirent.IsDir() + fileInfo.ModTime = dirent.modTime + fileInfo.Size = dirent.size + fileInfo.Mode = dirent.mode } - if objectInfo.IsDir { + if fileInfo.Mode.IsDir() { // Add os.PathSeparator suffix again for directories as // filepath.Join would have removed it. - objectInfo.Size = 0 - objectInfo.Name += string(os.PathSeparator) + fileInfo.Size = 0 + fileInfo.Name += string(os.PathSeparator) } - return objectInfo, nil + return fileInfo, nil } var markerBase, markerDir string @@ -126,12 +127,25 @@ func treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker string, recursive b } } - // readDirAll returns entries that begins with entryPrefixMatch - dirents, err := readDirAll(filepath.Join(bucketDir, prefixDir), entryPrefixMatch) + // Entry prefix match function. + prefixMatchFn := func(dirent fsDirent) bool { + if dirent.IsDir() || dirent.IsRegular() { + // Does dirent name has reserved prefixes or suffixes. + hasReserved := hasReservedPrefix(dirent.name) || hasReservedSuffix(dirent.name) + // All dirents which match prefix and do not have reserved + // keywords in them are valid entries. + return strings.HasPrefix(dirent.name, entryPrefixMatch) && !hasReserved + } + return false + } + + // scandir returns entries that begins with entryPrefixMatch + dirents, err := scandir(filepath.Join(bucketDir, prefixDir), prefixMatchFn, true) if err != nil { send(treeWalkResult{err: err}) return false } + // example: // If markerDir="four/" searchDirents() returns the index of "four/" in the sorted // dirents list. We skip all the dirent entries till "four/" @@ -158,13 +172,13 @@ func treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker string, recursive b } continue } - objectInfo, err := direntToObjectInfo(dirent) + fileInfo, err := direntToFileInfo(dirent) if err != nil { send(treeWalkResult{err: err}) return false } *count-- - if !send(treeWalkResult{objectInfo: objectInfo}) { + if !send(treeWalkResult{fileInfo: fileInfo}) { return false } } @@ -182,7 +196,7 @@ func startTreeWalk(fsPath, bucket, prefix, marker string, recursive bool) *treeW // if prefix is "one/two/th" and marker is "one/two/three/four/five.txt" // treeWalk is called with prefixDir="one/two/" and marker="three/four/five.txt" // and entryPrefixMatch="th" - ch := make(chan treeWalkResult, listObjectsLimit) + ch := make(chan treeWalkResult, fsListLimit) walkNotify := treeWalker{ch: ch} entryPrefixMatch := prefix prefixDir := "" @@ -196,8 +210,6 @@ func startTreeWalk(fsPath, bucket, prefix, marker string, recursive bool) *treeW go func() { defer close(ch) send := func(walkResult treeWalkResult) bool { - // Add the bucket. - walkResult.objectInfo.Bucket = bucket if count == 0 { walkResult.end = true } diff --git a/fs-dir-nix.go b/fs-dir-nix.go index 132c8d681..e50970519 100644 --- a/fs-dir-nix.go +++ b/fs-dir-nix.go @@ -23,7 +23,6 @@ import ( "path/filepath" "runtime" "sort" - "strings" "syscall" "unsafe" ) @@ -48,7 +47,7 @@ func clen(n []byte) int { // parseDirents - inspired from // https://golang.org/src/syscall/syscall_.go -func parseDirents(buf []byte) []fsDirent { +func parseDirents(dirPath string, buf []byte) []fsDirent { bufidx := 0 dirents := []fsDirent{} for bufidx < len(buf) { @@ -87,7 +86,15 @@ func parseDirents(buf []byte) []fsDirent { case syscall.DT_SOCK: mode = os.ModeSocket case syscall.DT_UNKNOWN: - mode = 0xffffffff + // On Linux XFS does not implement d_type for on disk + // format << v5. Fall back to Stat(). + if fi, err := os.Stat(filepath.Join(dirPath, name)); err == nil { + mode = fi.Mode() + } else { + // Caller listing would fail, if Stat failed but we + // won't crash the server. + mode = 0xffffffff + } } dirents = append(dirents, fsDirent{ @@ -98,37 +105,6 @@ func parseDirents(buf []byte) []fsDirent { return dirents } -// Read all directory entries, returns a list of lexically sorted entries. -func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) { - buf := make([]byte, readDirentBufSize) - f, err := os.Open(readDirPath) - if err != nil { - return nil, err - } - defer f.Close() - dirents := []fsDirent{} - for { - nbuf, err := syscall.ReadDirent(int(f.Fd()), buf) - if err != nil { - return nil, err - } - if nbuf <= 0 { - break - } - for _, dirent := range parseDirents(buf[:nbuf]) { - if dirent.IsDir() { - dirent.name += string(os.PathSeparator) - dirent.size = 0 - } - if strings.HasPrefix(dirent.name, entryPrefixMatch) { - dirents = append(dirents, dirent) - } - } - } - sort.Sort(byDirentName(dirents)) - return dirents, nil -} - // scans the directory dirPath, calling filter() on each directory // entry. Entries for which filter() returns true are stored, lexically // sorted using sort.Sort(). If filter is NULL, all entries are selected. @@ -151,12 +127,13 @@ func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDi if nbuf <= 0 { break } - for _, dirent := range parseDirents(buf[:nbuf]) { + for _, dirent := range parseDirents(dirPath, buf[:nbuf]) { if !namesOnly { dirent.name = filepath.Join(dirPath, dirent.name) } if dirent.IsDir() { dirent.name += string(os.PathSeparator) + dirent.size = 0 } if filter == nil || filter(dirent) { dirents = append(dirents, dirent) @@ -165,6 +142,5 @@ func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDi } sort.Sort(byDirentName(dirents)) - return dirents, nil } diff --git a/fs-dir-others.go b/fs-dir-others.go index b8e331284..210fa2361 100644 --- a/fs-dir-others.go +++ b/fs-dir-others.go @@ -23,46 +23,8 @@ import ( "os" "path/filepath" "sort" - "strings" ) -// Read all directory entries, returns a list of lexically sorted entries. -func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) { - f, err := os.Open(readDirPath) - if err != nil { - return nil, err - } - defer f.Close() - var dirents []fsDirent - for { - fis, err := f.Readdir(1000) - if err != nil { - if err == io.EOF { - break - } - return nil, err - } - for _, fi := range fis { - dirent := fsDirent{ - name: fi.Name(), - modTime: fi.ModTime(), - size: fi.Size(), - mode: fi.Mode(), - } - if dirent.IsDir() { - dirent.name += string(os.PathSeparator) - dirent.size = 0 - } - if strings.HasPrefix(fi.Name(), entryPrefixMatch) { - dirents = append(dirents, dirent) - } - } - } - // Sort dirents. - sort.Sort(byDirentName(dirents)) - return dirents, nil -} - // scans the directory dirPath, calling filter() on each directory // entry. Entries for which filter() returns true are stored, lexically // sorted using sort.Sort(). If filter is NULL, all entries are selected. @@ -103,6 +65,5 @@ func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDi } sort.Sort(byDirentName(dirents)) - return dirents, nil } diff --git a/fs-multipart-dir.go b/fs-multipart-dir.go deleted file mode 100644 index 84268a46a..000000000 --- a/fs-multipart-dir.go +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "errors" - "os" - "path/filepath" - "strings" - "time" -) - -func scanMultipartDir(bucketDir, prefixPath, markerPath, uploadIDMarker string, recursive bool) <-chan multipartObjectInfo { - objectInfoCh := make(chan multipartObjectInfo, listObjectsLimit) - - // TODO: check if bucketDir is absolute path - scanDir := bucketDir - dirDepth := bucketDir - - if prefixPath != "" { - if !filepath.IsAbs(prefixPath) { - tmpPrefixPath := filepath.Join(bucketDir, prefixPath) - if strings.HasSuffix(prefixPath, string(os.PathSeparator)) { - tmpPrefixPath += string(os.PathSeparator) - } - prefixPath = tmpPrefixPath - } - - // TODO: check if prefixPath starts with bucketDir - - // Case #1: if prefixPath is /mnt/mys3/mybucket/2012/photos/paris, then - // dirDepth is /mnt/mys3/mybucket/2012/photos - // Case #2: if prefixPath is /mnt/mys3/mybucket/2012/photos/, then - // dirDepth is /mnt/mys3/mybucket/2012/photos - dirDepth = filepath.Dir(prefixPath) - scanDir = dirDepth - } else { - prefixPath = bucketDir - } - - if markerPath != "" { - if !filepath.IsAbs(markerPath) { - tmpMarkerPath := filepath.Join(bucketDir, markerPath) - if strings.HasSuffix(markerPath, string(os.PathSeparator)) { - tmpMarkerPath += string(os.PathSeparator) - } - - markerPath = tmpMarkerPath - } - - // TODO: check markerPath must be a file - if uploadIDMarker != "" { - markerPath = filepath.Join(markerPath, uploadIDMarker+multipartUploadIDSuffix) - } - - // TODO: check if markerPath starts with bucketDir - // TODO: check if markerPath starts with prefixPath - - // Case #1: if markerPath is /mnt/mys3/mybucket/2012/photos/gophercon.png, then - // scanDir is /mnt/mys3/mybucket/2012/photos - // Case #2: if markerPath is /mnt/mys3/mybucket/2012/photos/gophercon.png/1fbd117a-268a-4ed0-85c9-8cc3888cbf20.uploadid, then - // scanDir is /mnt/mys3/mybucket/2012/photos/gophercon.png - // Case #3: if markerPath is /mnt/mys3/mybucket/2012/photos/, then - // scanDir is /mnt/mys3/mybucket/2012/photos - - scanDir = filepath.Dir(markerPath) - } else { - markerPath = bucketDir - } - - // Have bucketDir ends with os.PathSeparator - if !strings.HasSuffix(bucketDir, string(os.PathSeparator)) { - bucketDir += string(os.PathSeparator) - } - - // Remove os.PathSeparator if scanDir ends with - if strings.HasSuffix(scanDir, string(os.PathSeparator)) { - scanDir = filepath.Dir(scanDir) - } - - // goroutine - retrieves directory entries, makes ObjectInfo and sends into the channel. - go func() { - defer close(objectInfoCh) - - // send function - returns true if ObjectInfo is sent - // within (time.Second * 15) else false on timeout. - send := func(oi multipartObjectInfo) bool { - timer := time.After(time.Second * 15) - select { - case objectInfoCh <- oi: - return true - case <-timer: - return false - } - } - - // filter function - filters directory entries matching multipart uploadids, prefix and marker - direntFilterFn := func(dirent fsDirent) bool { - // check if dirent is a directory (or) dirent is a regular file and it's name ends with Upload ID suffix string - if dirent.IsDir() || (dirent.IsRegular() && strings.HasSuffix(dirent.name, multipartUploadIDSuffix)) { - // return if dirent's name starts with prefixPath and lexically higher than markerPath - return strings.HasPrefix(dirent.name, prefixPath) && dirent.name > markerPath - } - return false - } - - // filter function - filters directory entries matching multipart uploadids - subDirentFilterFn := func(dirent fsDirent) bool { - // check if dirent is a directory (or) dirent is a regular file and it's name ends with Upload ID suffix string - return dirent.IsDir() || (dirent.IsRegular() && strings.HasSuffix(dirent.name, multipartUploadIDSuffix)) - } - - // lastObjInfo is used to save last object info which is sent at last with End=true - var lastObjInfo *multipartObjectInfo - - sendError := func(err error) { - if lastObjInfo != nil { - if !send(*lastObjInfo) { - // as we got error sending lastObjInfo, we can't send the error - return - } - } - - send(multipartObjectInfo{Err: err, End: true}) - return - } - - for { - dirents, err := scandir(scanDir, direntFilterFn, false) - if err != nil { - sendError(err) - return - } - - var dirent fsDirent - for len(dirents) > 0 { - dirent, dirents = dirents[0], dirents[1:] - if dirent.IsRegular() { - // Handle uploadid file - name := strings.Replace(filepath.Dir(dirent.name), bucketDir, "", 1) - if name == "" { - // This should not happen ie uploadid file should not be in bucket directory - sendError(errors.New("Corrupted metadata")) - return - } - - uploadID := strings.Split(filepath.Base(dirent.name), multipartUploadIDSuffix)[0] - - // Solaris and older unixes have modTime to be - // empty, fallback to os.Stat() to fill missing values. - if dirent.modTime.IsZero() { - if fi, e := os.Stat(dirent.name); e == nil { - dirent.modTime = fi.ModTime() - } else { - sendError(e) - return - } - } - - objInfo := multipartObjectInfo{ - Name: name, - UploadID: uploadID, - ModifiedTime: dirent.modTime, - } - - // as we got new object info, send last object info and keep new object info as last object info - if lastObjInfo != nil { - if !send(*lastObjInfo) { - return - } - } - lastObjInfo = &objInfo - - continue - } - - // Fetch sub dirents. - subDirents, err := scandir(dirent.name, subDirentFilterFn, false) - if err != nil { - sendError(err) - return - } - - subDirFound := false - uploadIDDirents := []fsDirent{} - // If subDirents has a directory, then current dirent needs to be sent - for _, subdirent := range subDirents { - if subdirent.IsDir() { - subDirFound = true - - if recursive { - break - } - } - - if !recursive && subdirent.IsRegular() { - uploadIDDirents = append(uploadIDDirents, subdirent) - } - } - - // Send directory only for non-recursive listing - if !recursive && (subDirFound || len(subDirents) == 0) { - // Solaris and older unixes have modTime to be - // empty, fallback to os.Stat() to fill missing values. - if dirent.modTime.IsZero() { - if fi, e := os.Stat(dirent.name); e == nil { - dirent.modTime = fi.ModTime() - } else { - sendError(e) - return - } - } - - objInfo := multipartObjectInfo{ - Name: strings.Replace(dirent.name, bucketDir, "", 1), - ModifiedTime: dirent.modTime, - IsDir: true, - } - - // as we got new object info, send last object info and keep new object info as last object info - if lastObjInfo != nil { - if !send(*lastObjInfo) { - return - } - } - lastObjInfo = &objInfo - } - - if recursive { - dirents = append(subDirents, dirents...) - } else { - dirents = append(uploadIDDirents, dirents...) - } - } - - if !recursive { - break - } - - markerPath = scanDir + string(os.PathSeparator) - if scanDir = filepath.Dir(scanDir); scanDir < dirDepth { - break - } - } - - if lastObjInfo != nil { - // we got last object - lastObjInfo.End = true - if !send(*lastObjInfo) { - return - } - } - }() - - return objectInfoCh -} - -// multipartObjectInfo - Multipart object info -type multipartObjectInfo struct { - Name string - UploadID string - ModifiedTime time.Time - IsDir bool - Err error - End bool -} diff --git a/fs-multipart.go b/fs-multipart.go deleted file mode 100644 index 4ceec5af7..000000000 --- a/fs-multipart.go +++ /dev/null @@ -1,685 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2015,2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "crypto/md5" - "encoding/hex" - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/minio/minio/pkg/mimedb" - "github.com/minio/minio/pkg/probe" - "github.com/minio/minio/pkg/safe" - "github.com/skyrings/skyring-common/tools/uuid" -) - -const ( - minioMetaDir = ".minio" - multipartUploadIDSuffix = ".uploadid" -) - -// Removes files and its parent directories up to a given level. -func removeFileTree(fileName string, level string) error { - if e := os.Remove(fileName); e != nil { - return e - } - - for fileDir := filepath.Dir(fileName); fileDir > level; fileDir = filepath.Dir(fileDir) { - if status, e := isDirEmpty(fileDir); e != nil { - return e - } else if !status { - break - } - if e := os.Remove(fileDir); e != nil { - return e - } - } - - return nil -} - -// Takes an input stream and safely writes to disk, additionally -// verifies checksum. -func safeWriteFile(fileName string, data io.Reader, size int64, md5sum string) error { - safeFile, e := safe.CreateFileWithSuffix(fileName, "-") - if e != nil { - return e - } - - md5Hasher := md5.New() - multiWriter := io.MultiWriter(md5Hasher, safeFile) - if size > 0 { - if _, e = io.CopyN(multiWriter, data, size); e != nil { - // Closes the file safely and removes it in a single atomic operation. - safeFile.CloseAndRemove() - return e - } - } else { - if _, e = io.Copy(multiWriter, data); e != nil { - // Closes the file safely and removes it in a single atomic operation. - safeFile.CloseAndRemove() - return e - } - } - - dataMd5sum := hex.EncodeToString(md5Hasher.Sum(nil)) - if md5sum != "" && !isMD5SumEqual(md5sum, dataMd5sum) { - // Closes the file safely and removes it in a single atomic operation. - safeFile.CloseAndRemove() - return BadDigest{ExpectedMD5: md5sum, CalculatedMD5: dataMd5sum} - } - - // Safely close the file and atomically renames it the actual filePath. - safeFile.Close() - - // Safely wrote the file. - return nil -} - -func isFileExist(filename string) (bool, error) { - fi, e := os.Lstat(filename) - if e != nil { - if os.IsNotExist(e) { - return false, nil - } - - return false, e - } - - return fi.Mode().IsRegular(), nil -} - -// Create an s3 compatible MD5sum for complete multipart transaction. -func makeS3MD5(md5Strs ...string) (string, *probe.Error) { - var finalMD5Bytes []byte - for _, md5Str := range md5Strs { - md5Bytes, e := hex.DecodeString(md5Str) - if e != nil { - return "", probe.NewError(e) - } - finalMD5Bytes = append(finalMD5Bytes, md5Bytes...) - } - md5Hasher := md5.New() - md5Hasher.Write(finalMD5Bytes) - s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(md5Strs)) - return s3MD5, nil -} - -func (fs Filesystem) newUploadID(bucket, object string) (string, error) { - metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object) - - // create metaObjectDir if not exist - if status, e := isDirExist(metaObjectDir); e != nil { - return "", e - } else if !status { - if e := os.MkdirAll(metaObjectDir, 0755); e != nil { - return "", e - } - } - - for { - uuid, e := uuid.New() - if e != nil { - return "", e - } - - uploadID := uuid.String() - uploadIDFile := filepath.Join(metaObjectDir, uploadID+multipartUploadIDSuffix) - if _, e := os.Lstat(uploadIDFile); e != nil { - if !os.IsNotExist(e) { - return "", e - } - - // uploadIDFile doesn't exist, so create empty file to reserve the name - if e := ioutil.WriteFile(uploadIDFile, []byte{}, 0644); e != nil { - return "", e - } - - return uploadID, nil - } - // uploadIDFile already exists. - // loop again to try with different uuid generated. - } -} - -func (fs Filesystem) isUploadIDExist(bucket, object, uploadID string) (bool, error) { - return isFileExist(filepath.Join(fs.diskPath, minioMetaDir, bucket, object, uploadID+multipartUploadIDSuffix)) -} - -func (fs Filesystem) cleanupUploadID(bucket, object, uploadID string) error { - metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object) - uploadIDPrefix := uploadID + "." - - dirents, e := scandir(metaObjectDir, - func(dirent fsDirent) bool { - return dirent.IsRegular() && strings.HasPrefix(dirent.name, uploadIDPrefix) - }, - true) - - if e != nil { - return e - } - - for _, dirent := range dirents { - if e := os.Remove(filepath.Join(metaObjectDir, dirent.name)); e != nil { - return e - } - } - - if status, e := isDirEmpty(metaObjectDir); e != nil { - return e - } else if status { - if e := removeFileTree(metaObjectDir, filepath.Join(fs.diskPath, minioMetaDir, bucket)); e != nil { - return e - } - } - - return nil -} - -func (fs Filesystem) checkMultipartArgs(bucket, object string) (string, error) { - bucket, e := fs.checkBucketArg(bucket) - if e != nil { - return "", e - } - - if !IsValidObjectName(object) { - return "", ObjectNameInvalid{Object: object} - } - - return bucket, nil -} - -// NewMultipartUpload - initiate a new multipart session -func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.Error) { - if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil { - bucket = bucketDirName - } else { - return "", probe.NewError(e) - } - - if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil { - return "", probe.NewError(e) - } - - uploadID, e := fs.newUploadID(bucket, object) - if e != nil { - return "", probe.NewError(e) - } - - return uploadID, nil -} - -// PutObjectPart - create a part in a multipart session -func (fs Filesystem) PutObjectPart(bucket, object, uploadID string, partNumber int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) { - if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil { - bucket = bucketDirName - } else { - return "", probe.NewError(e) - } - - if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil { - //return "", probe.NewError(InternalError{Err: err}) - return "", probe.NewError(e) - } else if !status { - return "", probe.NewError(InvalidUploadID{UploadID: uploadID}) - } - - // Part id cannot be negative. - if partNumber <= 0 { - return "", probe.NewError(errors.New("invalid part id, cannot be zero or less than zero")) - } - - if partNumber > 10000 { - return "", probe.NewError(errors.New("invalid part id, should be not more than 10000")) - } - - if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil { - return "", probe.NewError(e) - } - - partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partNumber, md5Hex) - partFilePath := filepath.Join(fs.diskPath, minioMetaDir, bucket, object, partSuffix) - if e := safeWriteFile(partFilePath, data, size, md5Hex); e != nil { - return "", probe.NewError(e) - } - return md5Hex, nil -} - -// AbortMultipartUpload - abort an incomplete multipart session -func (fs Filesystem) AbortMultipartUpload(bucket, object, uploadID string) *probe.Error { - if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil { - bucket = bucketDirName - } else { - return probe.NewError(e) - } - - if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil { - //return probe.NewError(InternalError{Err: err}) - return probe.NewError(e) - } else if !status { - return probe.NewError(InvalidUploadID{UploadID: uploadID}) - } - - if e := fs.cleanupUploadID(bucket, object, uploadID); e != nil { - return probe.NewError(e) - } - - return nil -} - -// CompleteMultipartUpload - complete a multipart upload and persist the data -func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, parts []completePart) (ObjectInfo, *probe.Error) { - if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil { - bucket = bucketDirName - } else { - return ObjectInfo{}, probe.NewError(e) - } - - if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil { - //return probe.NewError(InternalError{Err: err}) - return ObjectInfo{}, probe.NewError(e) - } else if !status { - return ObjectInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID}) - } - - if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil { - return ObjectInfo{}, probe.NewError(e) - } - - metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object) - - var md5Sums []string - for _, part := range parts { - partNumber := part.PartNumber - md5sum := strings.Trim(part.ETag, "\"") - partFile := filepath.Join(metaObjectDir, uploadID+"."+strconv.Itoa(partNumber)+"."+md5sum) - if status, err := isFileExist(partFile); err != nil { - return ObjectInfo{}, probe.NewError(err) - } else if !status { - return ObjectInfo{}, probe.NewError(InvalidPart{}) - } - md5Sums = append(md5Sums, md5sum) - } - - // Save the s3 md5. - s3MD5, err := makeS3MD5(md5Sums...) - if err != nil { - return ObjectInfo{}, err.Trace(md5Sums...) - } - - completeObjectFile := filepath.Join(metaObjectDir, uploadID+".complete.") - safeFile, e := safe.CreateFileWithSuffix(completeObjectFile, "-") - if e != nil { - return ObjectInfo{}, probe.NewError(e) - } - for _, part := range parts { - partNumber := part.PartNumber - // Trim off the odd double quotes from ETag in the beginning and end. - md5sum := strings.TrimPrefix(part.ETag, "\"") - md5sum = strings.TrimSuffix(md5sum, "\"") - partFileStr := filepath.Join(metaObjectDir, fmt.Sprintf("%s.%d.%s", uploadID, partNumber, md5sum)) - var partFile *os.File - partFile, e = os.Open(partFileStr) - if e != nil { - // Remove the complete file safely. - safeFile.CloseAndRemove() - return ObjectInfo{}, probe.NewError(e) - } else if _, e = io.Copy(safeFile, partFile); e != nil { - // Remove the complete file safely. - safeFile.CloseAndRemove() - return ObjectInfo{}, probe.NewError(e) - } - partFile.Close() // Close part file after successful copy. - } - // All parts concatenated, safely close the temp file. - safeFile.Close() - - // Stat to gather fresh stat info. - objSt, e := os.Stat(completeObjectFile) - if e != nil { - return ObjectInfo{}, probe.NewError(e) - } - - bucketPath := filepath.Join(fs.diskPath, bucket) - objectPath := filepath.Join(bucketPath, object) - if e = os.MkdirAll(filepath.Dir(objectPath), 0755); e != nil { - os.Remove(completeObjectFile) - return ObjectInfo{}, probe.NewError(e) - } - if e = os.Rename(completeObjectFile, objectPath); e != nil { - os.Remove(completeObjectFile) - return ObjectInfo{}, probe.NewError(e) - } - - fs.cleanupUploadID(bucket, object, uploadID) // TODO: handle and log the error - - contentType := "application/octet-stream" - if objectExt := filepath.Ext(objectPath); objectExt != "" { - if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok { - contentType = content.ContentType - } - } - - newObject := ObjectInfo{ - Bucket: bucket, - Name: object, - ModifiedTime: objSt.ModTime(), - Size: objSt.Size(), - ContentType: contentType, - MD5Sum: s3MD5, - } - - return newObject, nil -} - -func (fs *Filesystem) saveListMultipartObjectCh(params listMultipartObjectParams, ch <-chan multipartObjectInfo) { - fs.listMultipartObjectMapMutex.Lock() - defer fs.listMultipartObjectMapMutex.Unlock() - - channels := []<-chan multipartObjectInfo{ch} - if _, ok := fs.listMultipartObjectMap[params]; ok { - channels = append(fs.listMultipartObjectMap[params], ch) - } - - fs.listMultipartObjectMap[params] = channels -} - -func (fs *Filesystem) lookupListMultipartObjectCh(params listMultipartObjectParams) <-chan multipartObjectInfo { - fs.listMultipartObjectMapMutex.Lock() - defer fs.listMultipartObjectMapMutex.Unlock() - - if channels, ok := fs.listMultipartObjectMap[params]; ok { - var channel <-chan multipartObjectInfo - channel, channels = channels[0], channels[1:] - if len(channels) > 0 { - fs.listMultipartObjectMap[params] = channels - } else { - // do not store empty channel list - delete(fs.listMultipartObjectMap, params) - } - - return channel - } - - return nil -} - -// ListMultipartUploads - list incomplete multipart sessions for a given BucketMultipartResourcesMetadata -func (fs Filesystem) ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error) { - result := ListMultipartsInfo{} - - bucket, e := fs.checkBucketArg(bucket) - if e != nil { - return result, probe.NewError(e) - } - - if !IsValidObjectPrefix(objectPrefix) { - return result, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: objectPrefix}) - } - - prefixPath := filepath.FromSlash(objectPrefix) - - // Verify if delimiter is anything other than '/', which we do not support. - if delimiter != "" && delimiter != "/" { - return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported", delimiter)) - } - - if keyMarker != "" && !strings.HasPrefix(keyMarker, objectPrefix) { - return result, probe.NewError(fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", keyMarker, objectPrefix)) - } - - markerPath := filepath.FromSlash(keyMarker) - if uploadIDMarker != "" { - if strings.HasSuffix(markerPath, string(os.PathSeparator)) { - return result, probe.NewError(fmt.Errorf("Invalid combination of uploadID marker '%s' and marker '%s'", uploadIDMarker, keyMarker)) - } - id, e := uuid.Parse(uploadIDMarker) - if e != nil { - return result, probe.NewError(e) - } - if id.IsZero() { - return result, probe.NewError(fmt.Errorf("Invalid upload ID marker %s", uploadIDMarker)) - } - } - - // Return empty response if maxUploads is zero - if maxUploads == 0 { - return result, nil - } - - // set listObjectsLimit to maxUploads for out-of-range limit - if maxUploads < 0 || maxUploads > listObjectsLimit { - maxUploads = listObjectsLimit - } - - recursive := true - if delimiter == "/" { - recursive = false - } - - metaBucketDir := filepath.Join(fs.diskPath, minioMetaDir, bucket) - - // Lookup of if listMultipartObjectChannel is available for given - // parameters, else create a new one. - savedChannel := true - multipartObjectInfoCh := fs.lookupListMultipartObjectCh(listMultipartObjectParams{ - bucket: bucket, - delimiter: delimiter, - keyMarker: markerPath, - prefix: prefixPath, - uploadIDMarker: uploadIDMarker, - }) - - if multipartObjectInfoCh == nil { - multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, keyMarker, uploadIDMarker, recursive) - savedChannel = false - } - - var objInfo *multipartObjectInfo - nextKeyMarker := "" - nextUploadIDMarker := "" - for i := 0; i < maxUploads; { - // read the channel - if oi, ok := <-multipartObjectInfoCh; ok { - objInfo = &oi - } else { - // closed channel - if i == 0 { - // first read - if !savedChannel { - // its valid to have a closed new channel for first read - multipartObjectInfoCh = nil - break - } - - // invalid saved channel amd create new channel - multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, keyMarker, - uploadIDMarker, recursive) - } else { - // TODO: FIX: there is a chance of infinite loop if we get closed channel always - // the channel got closed due to timeout - // create a new channel - multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, nextKeyMarker, - nextUploadIDMarker, recursive) - } - - // make it as new channel - savedChannel = false - continue - } - - if objInfo.Err != nil { - if os.IsNotExist(objInfo.Err) { - return ListMultipartsInfo{}, nil - } - - return ListMultipartsInfo{}, probe.NewError(objInfo.Err) - } - - // backward compatibility check - if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") { - continue - } - - // Directories are listed only if recursive is false - if objInfo.IsDir { - result.CommonPrefixes = append(result.CommonPrefixes, objInfo.Name) - } else { - result.Uploads = append(result.Uploads, uploadMetadata{ - Object: objInfo.Name, - UploadID: objInfo.UploadID, - Initiated: objInfo.ModifiedTime, - }) - } - - nextKeyMarker = objInfo.Name - nextUploadIDMarker = objInfo.UploadID - i++ - - if objInfo.End { - // as we received last object, do not save this channel for later use - multipartObjectInfoCh = nil - break - } - } - - if multipartObjectInfoCh != nil { - // we haven't received last object - result.IsTruncated = true - result.NextKeyMarker = nextKeyMarker - result.NextUploadIDMarker = nextUploadIDMarker - - // save this channel for later use - fs.saveListMultipartObjectCh(listMultipartObjectParams{ - bucket: bucket, - delimiter: delimiter, - keyMarker: nextKeyMarker, - prefix: objectPrefix, - uploadIDMarker: nextUploadIDMarker, - }, multipartObjectInfoCh) - } - - return result, nil -} - -// ListObjectParts - list parts from incomplete multipart session for a given ObjectResourcesMetadata -func (fs Filesystem) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error) { - if bucketDirName, err := fs.checkMultipartArgs(bucket, object); err == nil { - bucket = bucketDirName - } else { - return ListPartsInfo{}, probe.NewError(err) - } - - if status, err := fs.isUploadIDExist(bucket, object, uploadID); err != nil { - //return probe.NewError(InternalError{Err: err}) - return ListPartsInfo{}, probe.NewError(err) - } else if !status { - return ListPartsInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID}) - } - - // return empty ListPartsInfo - if maxParts == 0 { - return ListPartsInfo{}, nil - } - - if maxParts < 0 || maxParts > 1000 { - maxParts = 1000 - } - - metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object) - uploadIDPrefix := uploadID + "." - - dirents, e := scandir(metaObjectDir, - func(dirent fsDirent) bool { - // Part file is a regular file and has to be started with 'UPLOADID.' - if !(dirent.IsRegular() && strings.HasPrefix(dirent.name, uploadIDPrefix)) { - return false - } - - // Valid part file has to be 'UPLOADID.PARTNUMBER.MD5SUM' - tokens := strings.Split(dirent.name, ".") - if len(tokens) != 3 { - return false - } - - if partNumber, err := strconv.Atoi(tokens[1]); err == nil { - if partNumber >= 1 && partNumber <= 10000 && partNumber > partNumberMarker { - return true - } - } - - return false - }, - true) - if e != nil { - return ListPartsInfo{}, probe.NewError(e) - } - - isTruncated := false - nextPartNumberMarker := 0 - - parts := []partInfo{} - for i := range dirents { - if i == maxParts { - isTruncated = true - break - } - - // In some OS modTime is empty and use os.Stat() to fill missing values - if dirents[i].modTime.IsZero() { - if fi, e := os.Stat(filepath.Join(metaObjectDir, dirents[i].name)); e == nil { - dirents[i].modTime = fi.ModTime() - dirents[i].size = fi.Size() - } else { - return ListPartsInfo{}, probe.NewError(e) - } - } - - tokens := strings.Split(dirents[i].name, ".") - partNumber, _ := strconv.Atoi(tokens[1]) - md5sum := tokens[2] - parts = append(parts, partInfo{ - PartNumber: partNumber, - LastModified: dirents[i].modTime, - ETag: md5sum, - Size: dirents[i].size, - }) - } - - if isTruncated { - nextPartNumberMarker = 0 - } - - return ListPartsInfo{ - Bucket: bucket, - Object: object, - UploadID: uploadID, - PartNumberMarker: partNumberMarker, - NextPartNumberMarker: nextPartNumberMarker, - MaxParts: maxParts, - IsTruncated: isTruncated, - Parts: parts, - }, nil -} diff --git a/fs-object.go b/fs-object.go deleted file mode 100644 index c10f91832..000000000 --- a/fs-object.go +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "bytes" - "crypto/md5" - "io" - "os" - "path/filepath" - "strings" - - "encoding/hex" - "runtime" - - "github.com/minio/minio/pkg/mimedb" - "github.com/minio/minio/pkg/probe" - "github.com/minio/minio/pkg/safe" -) - -/// Object Operations - -// GetObject - GET object -func (fs Filesystem) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) { - // Input validation. - bucket, e := fs.checkBucketArg(bucket) - if e != nil { - return nil, probe.NewError(e) - } - if !IsValidObjectName(object) { - return nil, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) - } - - objectPath := filepath.Join(fs.diskPath, bucket, object) - file, e := os.Open(objectPath) - if e != nil { - // If the object doesn't exist, the bucket might not exist either. Stat for - // the bucket and give a better error message if that is true. - if os.IsNotExist(e) { - _, e = os.Stat(filepath.Join(fs.diskPath, bucket)) - if os.IsNotExist(e) { - return nil, probe.NewError(BucketNotFound{Bucket: bucket}) - } - return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) - } - return nil, probe.NewError(e) - } - // Initiate a cached stat operation on the file handler. - st, e := file.Stat() - if e != nil { - return nil, probe.NewError(e) - } - // Object path is a directory prefix, return object not found error. - if st.IsDir() { - return nil, probe.NewError(ObjectExistsAsPrefix{ - Bucket: bucket, - Prefix: object, - }) - } - - // Seek to a starting offset. - _, e = file.Seek(startOffset, os.SEEK_SET) - if e != nil { - // When the "handle is invalid", the file might be a directory on Windows. - if runtime.GOOS == "windows" && strings.Contains(e.Error(), "handle is invalid") { - return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) - } - return nil, probe.NewError(e) - } - return file, nil -} - -// GetObjectInfo - get object info. -func (fs Filesystem) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) { - // Input validation. - bucket, e := fs.checkBucketArg(bucket) - if e != nil { - return ObjectInfo{}, probe.NewError(e) - } - - if !IsValidObjectName(object) { - return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) - } - - info, err := getObjectInfo(fs.diskPath, bucket, object) - if err != nil { - if os.IsNotExist(err.ToGoError()) { - return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) - } - return ObjectInfo{}, err.Trace(bucket, object) - } - if info.IsDir { - return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) - } - return info, nil -} - -// getObjectInfo - get object stat info. -func getObjectInfo(rootPath, bucket, object string) (ObjectInfo, *probe.Error) { - // Do not use filepath.Join() since filepath.Join strips off any - // object names with '/', use them as is in a static manner so - // that we can send a proper 'ObjectNotFound' reply back upon os.Stat(). - var objectPath string - // For windows use its special os.PathSeparator == "\\" - if runtime.GOOS == "windows" { - objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object - } else { - objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object - } - stat, e := os.Stat(objectPath) - if e != nil { - return ObjectInfo{}, probe.NewError(e) - } - contentType := "application/octet-stream" - if runtime.GOOS == "windows" { - object = filepath.ToSlash(object) - } - - if objectExt := filepath.Ext(object); objectExt != "" { - content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))] - if ok { - contentType = content.ContentType - } - } - objInfo := ObjectInfo{ - Bucket: bucket, - Name: object, - ModifiedTime: stat.ModTime(), - Size: stat.Size(), - ContentType: contentType, - IsDir: stat.Mode().IsDir(), - } - return objInfo, nil -} - -// isMD5SumEqual - returns error if md5sum mismatches, success its `nil` -func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool { - // Verify the md5sum. - if expectedMD5Sum != "" && actualMD5Sum != "" { - // Decode md5sum to bytes from their hexadecimal - // representations. - expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum) - if err != nil { - return false - } - actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum) - if err != nil { - return false - } - // Verify md5sum bytes are equal after successful decoding. - if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) { - return false - } - return true - } - return false -} - -// PutObject - create an object. -func (fs Filesystem) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error) { - // Check bucket name valid. - bucket, e := fs.checkBucketArg(bucket) - if e != nil { - return ObjectInfo{}, probe.NewError(e) - } - - bucketDir := filepath.Join(fs.diskPath, bucket) - - // Verify object path legal. - if !IsValidObjectName(object) { - return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) - } - - if e = checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil { - return ObjectInfo{}, probe.NewError(e) - } - - // Get object path. - objectPath := filepath.Join(bucketDir, object) - - // md5Hex representation. - var md5Hex string - if len(metadata) != 0 { - md5Hex = metadata["md5Sum"] - } - - // Write object. - safeFile, e := safe.CreateFileWithPrefix(objectPath, md5Hex+"$tmpobject") - if e != nil { - switch e := e.(type) { - case *os.PathError: - if e.Op == "mkdir" { - if strings.Contains(e.Error(), "not a directory") { - return ObjectInfo{}, probe.NewError(ObjectExistsAsPrefix{Bucket: bucket, Prefix: object}) - } - } - return ObjectInfo{}, probe.NewError(e) - default: - return ObjectInfo{}, probe.NewError(e) - } - } - - // Initialize md5 writer. - md5Writer := md5.New() - - // Instantiate a new multi writer. - multiWriter := io.MultiWriter(md5Writer, safeFile) - - // Instantiate checksum hashers and create a multiwriter. - if size > 0 { - if _, e = io.CopyN(multiWriter, data, size); e != nil { - safeFile.CloseAndRemove() - return ObjectInfo{}, probe.NewError(e) - } - } else { - if _, e = io.Copy(multiWriter, data); e != nil { - safeFile.CloseAndRemove() - return ObjectInfo{}, probe.NewError(e) - } - } - - newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) - if md5Hex != "" { - if newMD5Hex != md5Hex { - return ObjectInfo{}, probe.NewError(BadDigest{md5Hex, newMD5Hex}) - } - } - - // Set stat again to get the latest metadata. - st, e := os.Stat(safeFile.Name()) - if e != nil { - return ObjectInfo{}, probe.NewError(e) - } - - contentType := "application/octet-stream" - if objectExt := filepath.Ext(objectPath); objectExt != "" { - content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))] - if ok { - contentType = content.ContentType - } - } - newObject := ObjectInfo{ - Bucket: bucket, - Name: object, - ModifiedTime: st.ModTime(), - Size: st.Size(), - MD5Sum: newMD5Hex, - ContentType: contentType, - } - - // Safely close and atomically rename the file. - safeFile.Close() - - return newObject, nil -} - -// deleteObjectPath - delete object path if its empty. -func deleteObjectPath(basePath, deletePath, bucket, object string) *probe.Error { - if basePath == deletePath { - return nil - } - // Verify if the path exists. - pathSt, e := os.Stat(deletePath) - if e != nil { - if os.IsNotExist(e) { - return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) - } - return probe.NewError(e) - } - if pathSt.IsDir() { - // Verify if directory is empty. - empty, e := isDirEmpty(deletePath) - if e != nil { - return probe.NewError(e) - } - if !empty { - return nil - } - } - // Attempt to remove path. - if e := os.Remove(deletePath); e != nil { - return probe.NewError(e) - } - // Recursively go down the next path and delete again. - if err := deleteObjectPath(basePath, filepath.Dir(deletePath), bucket, object); err != nil { - return err.Trace(basePath, deletePath, bucket, object) - } - return nil -} - -// DeleteObject - delete object. -func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error { - // Check bucket name valid - bucket, e := fs.checkBucketArg(bucket) - if e != nil { - return probe.NewError(e) - } - - bucketDir := filepath.Join(fs.diskPath, bucket) - - // Verify object path legal - if !IsValidObjectName(object) { - return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) - } - - // Do not use filepath.Join() since filepath.Join strips off any - // object names with '/', use them as is in a static manner so - // that we can send a proper 'ObjectNotFound' reply back upon os.Stat(). - var objectPath string - if runtime.GOOS == "windows" { - objectPath = fs.diskPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object - } else { - objectPath = fs.diskPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object - } - // Delete object path if its empty. - err := deleteObjectPath(bucketDir, objectPath, bucket, object) - if err != nil { - if os.IsNotExist(err.ToGoError()) { - return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) - } - return err.Trace(bucketDir, objectPath, bucket, object) - } - return nil -} diff --git a/fs-utils.go b/fs-utils.go index d1df78dd7..bf3da0a3b 100644 --- a/fs-utils.go +++ b/fs-utils.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. + * Minio Cloud Storage, (C) 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,48 +18,83 @@ package main import ( "regexp" + "runtime" + "strings" "unicode/utf8" ) -// validBucket regexp. -var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`) +// validVolname regexp. +var validVolname = regexp.MustCompile(`^.{3,63}$`) -// IsValidBucketName verifies a bucket name in accordance with Amazon's -// requirements. It must be 3-63 characters long, can contain dashes -// and periods, but must begin and end with a lowercase letter or a number. -// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html -func IsValidBucketName(bucket string) bool { - if len(bucket) < 3 || len(bucket) > 63 { +// isValidVolname verifies a volname name in accordance with object +// layer requirements. +func isValidVolname(volname string) bool { + if !validVolname.MatchString(volname) { return false } - if bucket[0] == '.' || bucket[len(bucket)-1] == '.' { - return false + switch runtime.GOOS { + case "windows": + // Volname shouldn't have reserved characters on windows in it. + return !strings.ContainsAny(volname, "/\\:*?\"<>|") + default: + // Volname shouldn't have '/' in it. + return !strings.ContainsAny(volname, "/") } - return validBucket.MatchString(bucket) } -// IsValidObjectName verifies an object name in accordance with Amazon's -// requirements. It cannot exceed 1024 characters and must be a valid UTF8 -// string. -// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html -func IsValidObjectName(object string) bool { - if len(object) > 1024 || len(object) == 0 { +// Keeping this as lower bound value supporting Linux, Darwin and Windows operating systems. +const pathMax = 4096 + +// isValidPath verifies if a path name is in accordance with FS limitations. +func isValidPath(path string) bool { + // TODO: Make this FSType or Operating system specific. + if len(path) > pathMax || len(path) == 0 { return false } - if !utf8.ValidString(object) { + if !utf8.ValidString(path) { return false } return true } -// IsValidObjectPrefix verifies whether the prefix is a valid object name. -// Its valid to have a empty prefix. -func IsValidObjectPrefix(object string) bool { +// isValidPrefix verifies where the prefix is a valid path. +func isValidPrefix(prefix string) bool { // Prefix can be empty. - if object == "" { + if prefix == "" { return true } - // Verify if prefix is a valid object name. - return IsValidObjectName(object) - + // Verify if prefix is a valid path. + return isValidPath(prefix) +} + +// List of reserved words for files, includes old and new ones. +var reservedKeywords = []string{ + "$multiparts", + "$tmpobject", + "$tmpfile", + // Add new reserved words if any used in future. +} + +// hasReservedPrefix - returns true if name has a reserved keyword suffixed. +func hasReservedSuffix(name string) (isReserved bool) { + for _, reservedKey := range reservedKeywords { + if strings.HasSuffix(name, reservedKey) { + isReserved = true + break + } + isReserved = false + } + return isReserved +} + +// hasReservedPrefix - has reserved prefix. +func hasReservedPrefix(name string) (isReserved bool) { + for _, reservedKey := range reservedKeywords { + if strings.HasPrefix(name, reservedKey) { + isReserved = true + break + } + isReserved = false + } + return isReserved } diff --git a/fs.go b/fs.go index 66d832e3c..e1af593f4 100644 --- a/fs.go +++ b/fs.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. + * Minio Cloud Storage, (C) 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,96 +17,529 @@ package main import ( + "io" "os" "path/filepath" + "strings" "sync" + "syscall" "github.com/minio/minio/pkg/disk" - "github.com/minio/minio/pkg/probe" + "github.com/minio/minio/pkg/safe" ) -// listObjectParams - list object params used for list object map -type listObjectParams struct { +const ( + fsListLimit = 1000 +) + +// listParams - list object params used for list object map +type listParams struct { bucket string - delimiter string + recursive bool marker string prefix string } -// listMultipartObjectParams - list multipart object params used for list multipart object map -type listMultipartObjectParams struct { - bucket string - delimiter string - keyMarker string - prefix string - uploadIDMarker string +// fsStorage - implements StorageAPI interface. +type fsStorage struct { + diskPath string + diskInfo disk.Info + minFreeDisk int64 + listObjectMap map[listParams][]*treeWalker + listObjectMapMutex *sync.Mutex } -// Filesystem - local variables -type Filesystem struct { - diskPath string - minFreeDisk int64 - rwLock *sync.RWMutex - listObjectMap map[listObjectParams][]*treeWalker - listObjectMapMutex *sync.Mutex - listMultipartObjectMap map[listMultipartObjectParams][]<-chan multipartObjectInfo - listMultipartObjectMapMutex *sync.Mutex -} - -// newFS instantiate a new filesystem. -func newFS(diskPath string) (ObjectAPI, *probe.Error) { - fs := &Filesystem{ - rwLock: &sync.RWMutex{}, +// isDirEmpty - returns whether given directory is empty or not. +func isDirEmpty(dirname string) (status bool, err error) { + f, err := os.Open(dirname) + if err == nil { + defer f.Close() + if _, err = f.Readdirnames(1); err == io.EOF { + status = true + err = nil + } } - fs.diskPath = diskPath + return status, err +} - /// Defaults - // Minium free disk required for i/o operations to succeed. - fs.minFreeDisk = 5 +// isDirExist - returns whether given directory exists or not. +func isDirExist(dirname string) (bool, error) { + fi, e := os.Stat(dirname) + if e != nil { + if os.IsNotExist(e) { + return false, nil + } + return false, e + } + return fi.IsDir(), nil +} - // Initialize list object map. - fs.listObjectMap = make(map[listObjectParams][]*treeWalker) - fs.listObjectMapMutex = &sync.Mutex{} - - // Initialize list multipart map. - fs.listMultipartObjectMap = make(map[listMultipartObjectParams][]<-chan multipartObjectInfo) - fs.listMultipartObjectMapMutex = &sync.Mutex{} - - // Return here. +// Initialize a new storage disk. +func newFS(diskPath string) (StorageAPI, error) { + if diskPath == "" { + return nil, errInvalidArgument + } + st, e := os.Stat(diskPath) + if e != nil { + return nil, e + } + if !st.IsDir() { + return nil, syscall.ENOTDIR + } + diskInfo, e := disk.GetInfo(diskPath) + if e != nil { + return nil, e + } + fs := fsStorage{ + diskPath: diskPath, + diskInfo: diskInfo, + minFreeDisk: 5, // Minimum 5% disk should be free. + listObjectMap: make(map[listParams][]*treeWalker), + listObjectMapMutex: &sync.Mutex{}, + } return fs, nil } -func (fs Filesystem) checkBucketArg(bucket string) (string, error) { - if !IsValidBucketName(bucket) { - return "", BucketNameInvalid{Bucket: bucket} +// checkDiskFree verifies if disk path has sufficient minium free disk space. +func checkDiskFree(diskPath string, minFreeDisk int64) (err error) { + di, err := disk.GetInfo(diskPath) + if err != nil { + return err } - bucket = getActualBucketname(fs.diskPath, bucket) - if status, e := isDirExist(filepath.Join(fs.diskPath, bucket)); !status { - if e == nil { - return "", BucketNotFound{Bucket: bucket} - } else if os.IsNotExist(e) { - return "", BucketNotFound{Bucket: bucket} - } else { - return "", e - } - } - return bucket, nil -} -func checkDiskFree(diskPath string, minFreeDisk int64) error { - di, e := disk.GetInfo(diskPath) - if e != nil { - return e - } - // Remove 5% from total space for cumulative disk space used for journalling, inodes etc. + // Remove 5% from total space for cumulative disk + // space used for journalling, inodes etc. availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100 if int64(availableDiskSpace) <= minFreeDisk { - return RootPathFull{Path: diskPath} + return errDiskPathFull + } + + // Success. + return nil +} + +// removeDuplicateVols - remove duplicate volumes. +func removeDuplicateVols(vols []VolInfo) []VolInfo { + length := len(vols) - 1 + for i := 0; i < length; i++ { + for j := i + 1; j <= length; j++ { + if vols[i].Name == vols[j].Name { + // Pick the latest volume, if there is a duplicate. + if vols[i].Created.Sub(vols[j].Created) > 0 { + vols[i] = vols[length] + } else { + vols[j] = vols[length] + } + vols = vols[0:length] + length-- + j-- + } + } + } + return vols +} + +// gets all the unique directories from diskPath. +func getAllUniqueVols(dirPath string) ([]VolInfo, error) { + volumeFn := func(dirent fsDirent) bool { + // Return all directories. + return dirent.IsDir() + } + namesOnly := false // Returned dirent names are absolute. + dirents, err := scandir(dirPath, volumeFn, namesOnly) + if err != nil { + return nil, err + } + var volsInfo []VolInfo + for _, dirent := range dirents { + fi, err := os.Stat(dirent.name) + if err != nil { + return nil, err + } + volsInfo = append(volsInfo, VolInfo{ + Name: fi.Name(), + // As os.Stat() doesn't carry other than ModTime(), use + // ModTime() as CreatedTime. + Created: fi.ModTime(), + }) + } + volsInfo = removeDuplicateVols(volsInfo) + return volsInfo, nil +} + +// getVolumeDir - will convert incoming volume names to +// corresponding valid volume names on the backend in a platform +// compatible way for all operating systems. If volume is not found +// an error is generated. +func (s fsStorage) getVolumeDir(volume string) (string, error) { + if !isValidVolname(volume) { + return "", errInvalidArgument + } + volumeDir := filepath.Join(s.diskPath, volume) + _, err := os.Stat(volumeDir) + if err == nil { + return volumeDir, nil + } + if os.IsNotExist(err) { + var volsInfo []VolInfo + volsInfo, err = getAllUniqueVols(s.diskPath) + if err != nil { + return volumeDir, errVolumeNotFound + } + for _, vol := range volsInfo { + // Verify if lowercase version of the volume + // is equal to the incoming volume, then use the proper name. + if strings.ToLower(vol.Name) == volume { + volumeDir = filepath.Join(s.diskPath, vol.Name) + return volumeDir, nil + } + } + return volumeDir, errVolumeNotFound + } else if os.IsPermission(err) { + return volumeDir, errVolumeAccessDenied + } + return volumeDir, err +} + +// Make a volume entry. +func (s fsStorage) MakeVol(volume string) (err error) { + volumeDir, err := s.getVolumeDir(volume) + if err == nil { + // Volume already exists, return error. + return errVolumeExists + } + + // Validate if disk is free. + if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil { + return e + } + + // If volume not found create it. + if err == errVolumeNotFound { + // Make a volume entry. + return os.Mkdir(volumeDir, 0700) + } + + // For all other errors return here. + return err +} + +// ListVols - list volumes. +func (s fsStorage) ListVols() (volsInfo []VolInfo, err error) { + volsInfo, err = getAllUniqueVols(s.diskPath) + if err != nil { + return nil, err + } + for _, vol := range volsInfo { + // Volname on case sensitive fs backends can come in as + // capitalized, but object layer cannot consume it + // directly. Convert it as we see fit. + volName := strings.ToLower(vol.Name) + volInfo := VolInfo{ + Name: volName, + Created: vol.Created, + } + volsInfo = append(volsInfo, volInfo) + } + return volsInfo, nil +} + +// StatVol - get volume info. +func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) { + // Verify if volume is valid and it exists. + volumeDir, err := s.getVolumeDir(volume) + if err != nil { + return VolInfo{}, err + } + // Stat a volume entry. + var st os.FileInfo + st, err = os.Stat(volumeDir) + if err != nil { + if os.IsNotExist(err) { + return VolInfo{}, errVolumeNotFound + } + return VolInfo{}, err + } + // As os.Stat() doesn't carry other than ModTime(), use ModTime() + // as CreatedTime. + createdTime := st.ModTime() + return VolInfo{ + Name: volume, + Created: createdTime, + }, nil +} + +// DeleteVol - delete a volume. +func (s fsStorage) DeleteVol(volume string) error { + // Verify if volume is valid and it exists. + volumeDir, err := s.getVolumeDir(volume) + if err != nil { + return err + } + err = os.Remove(volumeDir) + if err != nil { + if os.IsNotExist(err) { + return errVolumeNotFound + } else if strings.Contains(err.Error(), "directory is not empty") { + // On windows the string is slightly different, handle it + // here. + return errVolumeNotEmpty + } else if strings.Contains(err.Error(), "directory not empty") { + // Hopefully for all other operating systems, this is + // assumed to be consistent. + return errVolumeNotEmpty + } + return err + } + return err +} + +// Save the goroutine reference in the map +func (s *fsStorage) saveTreeWalk(params listParams, walker *treeWalker) { + s.listObjectMapMutex.Lock() + defer s.listObjectMapMutex.Unlock() + + walkers, _ := s.listObjectMap[params] + walkers = append(walkers, walker) + + s.listObjectMap[params] = walkers +} + +// Lookup the goroutine reference from map +func (s *fsStorage) lookupTreeWalk(params listParams) *treeWalker { + s.listObjectMapMutex.Lock() + defer s.listObjectMapMutex.Unlock() + + if walkChs, ok := s.listObjectMap[params]; ok { + for i, walkCh := range walkChs { + if !walkCh.timedOut { + newWalkChs := walkChs[i+1:] + if len(newWalkChs) > 0 { + s.listObjectMap[params] = newWalkChs + } else { + delete(s.listObjectMap, params) + } + return walkCh + } + } + // As all channels are timed out, delete the map entry + delete(s.listObjectMap, params) } return nil } -// GetRootPath - get root path. -func (fs Filesystem) GetRootPath() string { - return fs.diskPath +// List operation. +func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, count int) ([]FileInfo, bool, error) { + // Verify if volume is valid and it exists. + volumeDir, err := s.getVolumeDir(volume) + if err != nil { + return nil, true, err + } + var fileInfos []FileInfo + + if marker != "" { + // Verify if marker has prefix. + if marker != "" && !strings.HasPrefix(marker, prefix) { + return nil, true, errInvalidArgument + } + } + + // Return empty response for a valid request when count is 0. + if count == 0 { + return nil, true, nil + } + + // Over flowing count - reset to fsListLimit. + if count < 0 || count > fsListLimit { + count = fsListLimit + } + + // Verify if prefix exists. + prefixDir := filepath.Dir(filepath.FromSlash(prefix)) + prefixRootDir := filepath.Join(volumeDir, prefixDir) + if status, err := isDirExist(prefixRootDir); !status { + if err == nil { + // Prefix does not exist, not an error just respond empty list response. + return nil, true, nil + } else if strings.Contains(err.Error(), "not a directory") { + // Prefix exists as a file. + return nil, true, nil + } + // Rest errors should be treated as failure. + return nil, true, err + } + + // Maximum 1000 files returned in a single call. + // Further calls will set right marker value to continue reading the rest of the files. + // popTreeWalker returns nil if the call to ListFiles is done for the first time. + // On further calls to ListFiles to retrive more files within the timeout period, + // popTreeWalker returns the channel from which rest of the objects can be retrieved. + walker := s.lookupTreeWalk(listParams{volume, recursive, marker, prefix}) + if walker == nil { + walker = startTreeWalk(s.diskPath, volume, filepath.FromSlash(prefix), filepath.FromSlash(marker), recursive) + } + nextMarker := "" + for i := 0; i < count; { + walkResult, ok := <-walker.ch + if !ok { + // Closed channel. + return fileInfos, true, nil + } + // For any walk error return right away. + if walkResult.err != nil { + return nil, true, walkResult.err + } + fileInfo := walkResult.fileInfo + fileInfo.Name = filepath.ToSlash(fileInfo.Name) + fileInfos = append(fileInfos, fileInfo) + // We have listed everything return. + if walkResult.end { + return fileInfos, true, nil + } + nextMarker = fileInfo.Name + i++ + } + s.saveTreeWalk(listParams{volume, recursive, nextMarker, prefix}, walker) + return fileInfos, false, nil +} + +// ReadFile - read a file at a given offset. +func (s fsStorage) ReadFile(volume string, path string, offset int64) (readCloser io.ReadCloser, err error) { + volumeDir, err := s.getVolumeDir(volume) + if err != nil { + return nil, err + } + + filePath := filepath.Join(volumeDir, filepath.FromSlash(path)) + file, err := os.Open(filePath) + if err != nil { + if os.IsNotExist(err) { + return nil, errFileNotFound + } else if os.IsPermission(err) { + return nil, errFileAccessDenied + } + return nil, err + } + st, err := file.Stat() + if err != nil { + return nil, err + } + // Verify if its not a regular file, since subsequent Seek is undefined. + if !st.Mode().IsRegular() { + return nil, errIsNotRegular + } + // Seek to requested offset. + _, err = file.Seek(offset, os.SEEK_SET) + if err != nil { + return nil, err + } + return file, nil +} + +// CreateFile - create a file at path. +func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) { + volumeDir, err := s.getVolumeDir(volume) + if err != nil { + return nil, err + } + if err := checkDiskFree(s.diskPath, s.minFreeDisk); err != nil { + return nil, err + } + filePath := filepath.Join(volumeDir, path) + // Verify if the file already exists and is not of regular type. + if st, err := os.Stat(filePath); err == nil { + if st.IsDir() { + return nil, errIsNotRegular + } + } + return safe.CreateFileWithPrefix(filePath, "$tmpfile") +} + +// StatFile - get file info. +func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) { + volumeDir, err := s.getVolumeDir(volume) + if err != nil { + return FileInfo{}, err + } + + filePath := filepath.Join(volumeDir, filepath.FromSlash(path)) + st, err := os.Stat(filePath) + if err != nil { + // File is really not found. + if os.IsNotExist(err) { + return FileInfo{}, errFileNotFound + } + // File path cannot be verified since one of the parents is a file. + if strings.Contains(err.Error(), "not a directory") { + return FileInfo{}, errIsNotRegular + } + // Return all errors here. + return FileInfo{}, err + } + // If its a directory its not a regular file. + if st.Mode().IsDir() { + return FileInfo{}, errIsNotRegular + } + file = FileInfo{ + Volume: volume, + Name: path, + ModTime: st.ModTime(), + Size: st.Size(), + Mode: st.Mode(), + } + return file, nil +} + +// deleteFile - delete file path if its empty. +func deleteFile(basePath, deletePath string) error { + if basePath == deletePath { + return nil + } + // Verify if the path exists. + pathSt, err := os.Stat(deletePath) + if err != nil { + if os.IsNotExist(err) { + return errFileNotFound + } else if os.IsPermission(err) { + return errFileAccessDenied + } + return err + } + if pathSt.IsDir() { + // Verify if directory is empty. + empty, err := isDirEmpty(deletePath) + if err != nil { + return err + } + if !empty { + return nil + } + } + // Attempt to remove path. + if err := os.Remove(deletePath); err != nil { + return err + } + // Recursively go down the next path and delete again. + if err := deleteFile(basePath, filepath.Dir(deletePath)); err != nil { + return err + } + return nil +} + +// DeleteFile - delete a file at path. +func (s fsStorage) DeleteFile(volume, path string) error { + volumeDir, err := s.getVolumeDir(volume) + if err != nil { + return err + } + + // Following code is needed so that we retain "/" suffix if any in + // path argument. + filePath := filepath.Join(volumeDir, filepath.FromSlash(path)) + if strings.HasSuffix(filepath.FromSlash(path), string(os.PathSeparator)) { + filePath = filePath + string(os.PathSeparator) + } + + // Delete file and delete parent directory as well if its empty. + return deleteFile(volumeDir, filePath) } diff --git a/fs_test.go b/fs_test.go deleted file mode 100644 index 811ceeb7a..000000000 --- a/fs_test.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "io/ioutil" - "os" - - . "gopkg.in/check.v1" -) - -func (s *MyAPISuite) TestAPISuite(c *C) { - var storageList []string - create := func() ObjectAPI { - path, e := ioutil.TempDir(os.TempDir(), "minio-") - c.Check(e, IsNil) - storageList = append(storageList, path) - store, err := newFS(path) - c.Check(err, IsNil) - return store - } - APITestSuite(c, create) - defer removeRoots(c, storageList) -} - -func removeRoots(c *C, roots []string) { - for _, root := range roots { - os.RemoveAll(root) - } -} diff --git a/generic-handlers.go b/generic-handlers.go index cc2a03bd3..daa071d34 100644 --- a/generic-handlers.go +++ b/generic-handlers.go @@ -58,10 +58,10 @@ func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Re-direction handled specifically for browsers. if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") { // '/' is redirected to 'locationPrefix/' - // '/rpc' is redirected to 'locationPrefix/rpc' + // '/webrpc' is redirected to 'locationPrefix/webrpc' // '/login' is redirected to 'locationPrefix/login' switch r.URL.Path { - case "/", "/rpc", "/login", "/favicon.ico": + case "/", "/webrpc", "/login", "/favicon.ico": location := h.locationPrefix + r.URL.Path // Redirect to new location. http.Redirect(w, r, location, http.StatusTemporaryRedirect) diff --git a/network-fs.go b/network-fs.go new file mode 100644 index 000000000..3872deaf0 --- /dev/null +++ b/network-fs.go @@ -0,0 +1,243 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/rpc" + "net/url" + urlpath "path" + "strconv" + "strings" + "time" +) + +type networkFS struct { + netScheme string + netAddr string + netPath string + rpcClient *rpc.Client + httpClient *http.Client +} + +const ( + storageRPCPath = reservedBucket + "/rpc/storage" +) + +// splits network path into its components Address and Path. +func splitNetPath(networkPath string) (netAddr, netPath string) { + index := strings.LastIndex(networkPath, ":") + netAddr = networkPath[:index] + netPath = networkPath[index+1:] + return netAddr, netPath +} + +// Converts rpc.ServerError to underlying error. This function is +// written so that the storageAPI errors are consistent across network +// disks as well. +func toStorageErr(err error) error { + switch err.Error() { + case errVolumeNotFound.Error(): + return errVolumeNotFound + case errVolumeExists.Error(): + return errVolumeExists + case errFileNotFound.Error(): + return errFileNotFound + case errIsNotRegular.Error(): + return errIsNotRegular + case errVolumeNotEmpty.Error(): + return errVolumeNotEmpty + case errFileAccessDenied.Error(): + return errFileAccessDenied + case errVolumeAccessDenied.Error(): + return errVolumeAccessDenied + } + return err +} + +// Initialize new network file system. +func newNetworkFS(networkPath string) (StorageAPI, error) { + // Input validation. + if networkPath == "" && strings.LastIndex(networkPath, ":") != -1 { + return nil, errInvalidArgument + } + + // TODO validate netAddr and netPath. + netAddr, netPath := splitNetPath(networkPath) + + // Dial minio rpc storage http path. + rpcClient, err := rpc.DialHTTPPath("tcp", netAddr, storageRPCPath) + if err != nil { + return nil, err + } + + // Initialize http client. + httpClient := &http.Client{ + // Setting a sensible time out of 6minutes to wait for + // response headers. Request is pro-actively cancelled + // after 6minutes if no response was received from server. + Timeout: 6 * time.Minute, + Transport: http.DefaultTransport, + } + + // Initialize network storage. + ndisk := &networkFS{ + netScheme: "http", // TODO: fix for ssl rpc support. + netAddr: netAddr, + netPath: netPath, + rpcClient: rpcClient, + httpClient: httpClient, + } + + // Returns successfully here. + return ndisk, nil +} + +// MakeVol - make a volume. +func (n networkFS) MakeVol(volume string) error { + reply := GenericReply{} + if err := n.rpcClient.Call("Storage.MakeVolHandler", volume, &reply); err != nil { + return toStorageErr(err) + } + return nil +} + +// ListVols - List all volumes. +func (n networkFS) ListVols() (vols []VolInfo, err error) { + ListVols := ListVolsReply{} + err = n.rpcClient.Call("Storage.ListVolsHandler", "", &ListVols) + if err != nil { + return nil, err + } + return ListVols.Vols, nil +} + +// StatVol - get current Stat volume info. +func (n networkFS) StatVol(volume string) (volInfo VolInfo, err error) { + if err = n.rpcClient.Call("Storage.StatVolHandler", volume, &volInfo); err != nil { + return VolInfo{}, toStorageErr(err) + } + return volInfo, nil +} + +// DeleteVol - Delete a volume. +func (n networkFS) DeleteVol(volume string) error { + reply := GenericReply{} + if err := n.rpcClient.Call("Storage.DeleteVolHandler", volume, &reply); err != nil { + return toStorageErr(err) + } + return nil +} + +// File operations. + +// CreateFile - create file. +func (n networkFS) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) { + writeURL := new(url.URL) + writeURL.Scheme = n.netScheme + writeURL.Host = n.netAddr + writeURL.Path = fmt.Sprintf("%s/upload/%s", storageRPCPath, urlpath.Join(volume, path)) + + contentType := "application/octet-stream" + readCloser, writeCloser := io.Pipe() + go func() { + resp, err := n.httpClient.Post(writeURL.String(), contentType, readCloser) + if err != nil { + readCloser.CloseWithError(err) + return + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + readCloser.CloseWithError(errFileNotFound) + return + } + readCloser.CloseWithError(errors.New("Invalid response.")) + return + } + // Close the reader. + readCloser.Close() + } + }() + return writeCloser, nil +} + +// StatFile - get latest Stat information for a file at path. +func (n networkFS) StatFile(volume, path string) (fileInfo FileInfo, err error) { + if err = n.rpcClient.Call("Storage.StatFileHandler", StatFileArgs{ + Vol: volume, + Path: path, + }, &fileInfo); err != nil { + return FileInfo{}, toStorageErr(err) + } + return fileInfo, nil +} + +// ReadFile - reads a file. +func (n networkFS) ReadFile(volume string, path string, offset int64) (reader io.ReadCloser, err error) { + readURL := new(url.URL) + readURL.Scheme = n.netScheme + readURL.Host = n.netAddr + readURL.Path = fmt.Sprintf("%s/download/%s", storageRPCPath, urlpath.Join(volume, path)) + readQuery := make(url.Values) + readQuery.Set("offset", strconv.FormatInt(offset, 10)) + readURL.RawQuery = readQuery.Encode() + resp, err := n.httpClient.Get(readURL.String()) + if err != nil { + return nil, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, errFileNotFound + } + return nil, errors.New("Invalid response") + } + } + return resp.Body, nil +} + +// ListFiles - List all files in a volume. +func (n networkFS) ListFiles(volume, prefix, marker string, recursive bool, count int) (files []FileInfo, eof bool, err error) { + listFilesReply := ListFilesReply{} + if err = n.rpcClient.Call("Storage.ListFilesHandler", ListFilesArgs{ + Vol: volume, + Prefix: prefix, + Marker: marker, + Recursive: recursive, + Count: count, + }, &listFilesReply); err != nil { + return nil, true, toStorageErr(err) + } + // Return successfully unmarshalled results. + return listFilesReply.Files, listFilesReply.EOF, nil +} + +// DeleteFile - Delete a file at path. +func (n networkFS) DeleteFile(volume, path string) (err error) { + reply := GenericReply{} + if err = n.rpcClient.Call("Storage.DeleteFileHandler", DeleteFileArgs{ + Vol: volume, + Path: path, + }, &reply); err != nil { + return toStorageErr(err) + } + return nil +} diff --git a/object-api-interface.go b/object-api-interface.go deleted file mode 100644 index 9c4b11681..000000000 --- a/object-api-interface.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "io" - - "github.com/minio/minio/pkg/probe" -) - -// ObjectAPI interface. -type ObjectAPI interface { - // Bucket resource API. - DeleteBucket(bucket string) *probe.Error - ListBuckets() ([]BucketInfo, *probe.Error) - MakeBucket(bucket string) *probe.Error - GetBucketInfo(bucket string) (BucketInfo, *probe.Error) - - // Bucket query API. - ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error) - ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error) - - // Object resource API. - GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) - GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) - PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error) - DeleteObject(bucket, object string) *probe.Error - - // Object query API. - NewMultipartUpload(bucket, object string) (string, *probe.Error) - PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) - ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error) - CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (ObjectInfo, *probe.Error) - AbortMultipartUpload(bucket, object, uploadID string) *probe.Error -} diff --git a/fs-bucket-listobjects_test.go b/object-api-listobjects_test.go similarity index 92% rename from fs-bucket-listobjects_test.go rename to object-api-listobjects_test.go index 8ebda79fa..bb8079a16 100644 --- a/fs-bucket-listobjects_test.go +++ b/object-api-listobjects_test.go @@ -24,29 +24,34 @@ import ( "strconv" "strings" "testing" + + "github.com/minio/minio/pkg/probe" ) func TestListObjects(t *testing.T) { - // Make a temporary directory to use as the fs. + // Make a temporary directory to use as the obj. directory, e := ioutil.TempDir("", "minio-list-object-test") if e != nil { t.Fatal(e) } defer os.RemoveAll(directory) - // Create the fs. - fs, err := newFS(directory) - if err != nil { - t.Fatal(err) + // Create the obj. + fs, e := newFS(directory) + if e != nil { + t.Fatal(e) } + + obj := newObjectLayer(fs) + var err *probe.Error // This bucket is used for testing ListObject operations. - err = fs.MakeBucket("test-bucket-list-object") + err = obj.MakeBucket("test-bucket-list-object") if err != nil { t.Fatal(err) } // Will not store any objects in this bucket, // Its to test ListObjects on an empty bucket. - err = fs.MakeBucket("empty-bucket") + err = obj.MakeBucket("empty-bucket") if err != nil { t.Fatal(err) } @@ -57,36 +62,36 @@ func TestListObjects(t *testing.T) { } defer os.Remove(tmpfile.Name()) // clean up - _, err = fs.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil) + _, err = obj.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil) if err != nil { t.Fatal(e) } - _, err = fs.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) + _, err = obj.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { t.Fatal(e) } - _, err = fs.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) + _, err = obj.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) if err != nil { t.Fatal(e) } for i := 0; i < 2; i++ { key := "newPrefix" + strconv.Itoa(i) - _, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { t.Fatal(err) } } - _, err = fs.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil) + _, err = obj.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil) if err != nil { t.Fatal(e) } for i := 0; i < 3; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { t.Fatal(err) } @@ -529,7 +534,7 @@ func TestListObjects(t *testing.T) { } for i, testCase := range testCases { - result, err := fs.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys) + result, err := obj.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Cause.Error()) } @@ -565,28 +570,31 @@ func TestListObjects(t *testing.T) { } func BenchmarkListObjects(b *testing.B) { - // Make a temporary directory to use as the fs. + // Make a temporary directory to use as the obj. directory, e := ioutil.TempDir("", "minio-list-benchmark") if e != nil { b.Fatal(e) } defer os.RemoveAll(directory) - // Create the fs. - fs, err := newFS(directory) - if err != nil { - b.Fatal(err) + // Create the obj. + fs, e := newFS(directory) + if e != nil { + b.Fatal(e) } + obj := newObjectLayer(fs) + var err *probe.Error + // Create a bucket. - err = fs.MakeBucket("ls-benchmark-bucket") + err = obj.MakeBucket("ls-benchmark-bucket") if err != nil { b.Fatal(err) } for i := 0; i < 20000; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = obj.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) if err != nil { b.Fatal(err) } @@ -596,7 +604,7 @@ func BenchmarkListObjects(b *testing.B) { // List the buckets over and over and over. for i := 0; i < b.N; i++ { - _, err = fs.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1) + _, err = obj.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1) if err != nil { b.Fatal(err) } diff --git a/object-api-multipart.go b/object-api-multipart.go new file mode 100644 index 000000000..6c6980b0f --- /dev/null +++ b/object-api-multipart.go @@ -0,0 +1,559 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "path" + "strconv" + "strings" + + "github.com/minio/minio/pkg/probe" + "github.com/skyrings/skyring-common/tools/uuid" +) + +const ( + minioMetaVolume = ".minio" +) + +// checkLeafDirectory - verifies if a given path is leaf directory if +// yes returns all the files inside it. +func (o objectAPI) checkLeafDirectory(prefixPath string) (isLeaf bool, fis []FileInfo) { + var allFileInfos []FileInfo + for { + fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, "", false, 1000) + if e != nil { + break + } + allFileInfos = append(allFileInfos, fileInfos...) + if eof { + break + } + } + for _, fileInfo := range allFileInfos { + if fileInfo.Mode.IsDir() { + isLeaf = false + return isLeaf, nil + } + fileName := path.Base(fileInfo.Name) + if !strings.Contains(fileName, ".") { + fis = append(fis, fileInfo) + } + } + isLeaf = true + return isLeaf, fis +} + +// ListMultipartUploads - list multipart uploads. +func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error) { + result := ListMultipartsInfo{} + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return ListMultipartsInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectPrefix(prefix) { + return ListMultipartsInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) + } + if _, e := o.storage.StatVol(minioMetaVolume); e != nil { + if e == errVolumeNotFound { + e = o.storage.MakeVol(minioMetaVolume) + if e != nil { + return ListMultipartsInfo{}, probe.NewError(e) + } + } + } + // Verify if delimiter is anything other than '/', which we do not support. + if delimiter != "" && delimiter != slashPathSeparator { + return ListMultipartsInfo{}, probe.NewError(UnsupportedDelimiter{ + Delimiter: delimiter, + }) + } + // Verify if marker has prefix. + if keyMarker != "" && !strings.HasPrefix(keyMarker, prefix) { + return ListMultipartsInfo{}, probe.NewError(InvalidMarkerPrefixCombination{ + Marker: keyMarker, + Prefix: prefix, + }) + } + if uploadIDMarker != "" { + if strings.HasSuffix(keyMarker, slashPathSeparator) { + return result, probe.NewError(InvalidUploadIDKeyCombination{ + UploadIDMarker: uploadIDMarker, + KeyMarker: keyMarker, + }) + } + id, e := uuid.Parse(uploadIDMarker) + if e != nil { + return result, probe.NewError(e) + } + if id.IsZero() { + return result, probe.NewError(MalformedUploadID{ + UploadID: uploadIDMarker, + }) + } + } + + recursive := true + if delimiter == slashPathSeparator { + recursive = false + } + result.IsTruncated = true + result.MaxUploads = maxUploads + newMaxUploads := 0 + // not using path.Join() as it strips off the trailing '/'. + // Also bucket should always be followed by '/' even if prefix is empty. + prefixPath := pathJoin(bucket, prefix) + if recursive { + keyMarkerPath := "" + if keyMarker != "" { + keyMarkerPath = path.Join(bucket, keyMarker, uploadIDMarker) + } + outerLoop: + for { + fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, keyMarkerPath, recursive, maxUploads-newMaxUploads) + if e != nil { + return ListMultipartsInfo{}, probe.NewError(e) + } + for _, fi := range fileInfos { + keyMarkerPath = fi.Name + // fi.Name will look like bucket/object/uploadID, extract object and uploadID. + uploadID := path.Base(fi.Name) + objectName := strings.TrimPrefix(path.Dir(fi.Name), bucket+slashPathSeparator) + if strings.Contains(uploadID, ".") { + // contains partnumber and md5sum info, skip this. + continue + } + result.Uploads = append(result.Uploads, uploadMetadata{ + Object: objectName, + UploadID: uploadID, + Initiated: fi.ModTime, + }) + result.NextKeyMarker = objectName + result.NextUploadIDMarker = uploadID + newMaxUploads++ + if newMaxUploads == maxUploads { + if eof { + result.IsTruncated = false + } + break outerLoop + } + } + if eof { + result.IsTruncated = false + break + } + } + if !result.IsTruncated { + result.NextKeyMarker = "" + result.NextUploadIDMarker = "" + } + return result, nil + } + + var fileInfos []FileInfo + // read all the "fileInfos" in the prefix + for { + marker := "" + fis, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, marker, recursive, 1000) + if e != nil { + return ListMultipartsInfo{}, probe.NewError(e) + } + for _, fi := range fis { + marker = fi.Name + if fi.Mode.IsDir() { + fileInfos = append(fileInfos, fi) + } + } + if eof { + break + } + } + // Create "uploads" slice from "fileInfos" slice. + var uploads []uploadMetadata + for _, fi := range fileInfos { + leaf, entries := o.checkLeafDirectory(fi.Name) + objectName := strings.TrimPrefix(fi.Name, bucket+slashPathSeparator) + if leaf { + for _, entry := range entries { + if strings.Contains(entry.Name, ".") { + continue + } + uploads = append(uploads, uploadMetadata{ + Object: strings.TrimSuffix(objectName, slashPathSeparator), + UploadID: path.Base(entry.Name), + Initiated: entry.ModTime, + }) + } + continue + } + uploads = append(uploads, uploadMetadata{ + Object: objectName, + }) + } + index := 0 + for i, upload := range uploads { + index = i + if upload.Object > keyMarker { + break + } + if uploads[index].Object == keyMarker && upload.UploadID > uploadIDMarker { + break + } + } + for ; index < len(uploads); index++ { + if (len(result.Uploads) + len(result.CommonPrefixes)) == maxUploads { + break + } + result.NextKeyMarker = uploads[index].Object + if strings.HasSuffix(uploads[index].Object, slashPathSeparator) { + // for a directory entry + result.CommonPrefixes = append(result.CommonPrefixes, uploads[index].Object) + continue + } + result.NextUploadIDMarker = uploads[index].UploadID + result.Uploads = append(result.Uploads, uploads[index]) + } + if index == len(uploads) { + result.IsTruncated = false + result.NextKeyMarker = "" + result.NextUploadIDMarker = "" + } + return result, nil +} + +func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return "", probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return "", probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + } + if _, e := o.storage.StatVol(minioMetaVolume); e != nil { + if e == errVolumeNotFound { + e = o.storage.MakeVol(minioMetaVolume) + if e != nil { + return "", probe.NewError(e) + } + } + } + for { + uuid, e := uuid.New() + if e != nil { + return "", probe.NewError(e) + } + uploadID := uuid.String() + uploadIDPath := path.Join(bucket, object, uploadID) + if _, e = o.storage.StatFile(minioMetaVolume, uploadIDPath); e != nil { + if e != errFileNotFound { + return "", probe.NewError(e) + } + // uploadIDPath doesn't exist, so create empty file to reserve the name + var w io.WriteCloser + if w, e = o.storage.CreateFile(minioMetaVolume, uploadIDPath); e == nil { + if e = w.Close(); e != nil { + return "", probe.NewError(e) + } + } else { + return "", probe.NewError(e) + } + return uploadID, nil + } + // uploadIDPath already exists. + // loop again to try with different uuid generated. + } +} + +// isUploadIDExists - verify if a given uploadID exists and is valid. +func (o objectAPI) isUploadIDExists(bucket, object, uploadID string) (bool, error) { + uploadIDPath := path.Join(bucket, object, uploadID) + st, e := o.storage.StatFile(minioMetaVolume, uploadIDPath) + if e != nil { + // Upload id does not exist. + if e == errFileNotFound { + return false, nil + } + return false, e + } + // Upload id exists and is a regular file. + return st.Mode.IsRegular(), nil +} + +func (o objectAPI) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return "", probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return "", probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + } + if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil { + return "", probe.NewError(e) + } else if !status { + return "", probe.NewError(InvalidUploadID{UploadID: uploadID}) + } + + partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partID, md5Hex) + fileWriter, e := o.storage.CreateFile(minioMetaVolume, path.Join(bucket, object, partSuffix)) + if e != nil { + if e == errVolumeNotFound { + return "", probe.NewError(BucketNotFound{ + Bucket: bucket, + }) + } else if e == errIsNotRegular { + return "", probe.NewError(ObjectExistsAsPrefix{ + Bucket: bucket, + Prefix: object, + }) + } + return "", probe.NewError(e) + } + + // Initialize md5 writer. + md5Writer := md5.New() + + // Instantiate a new multi writer. + multiWriter := io.MultiWriter(md5Writer, fileWriter) + + // Instantiate checksum hashers and create a multiwriter. + if size > 0 { + if _, e = io.CopyN(multiWriter, data, size); e != nil { + safeCloseAndRemove(fileWriter) + return "", probe.NewError(e) + } + } else { + if _, e = io.Copy(multiWriter, data); e != nil { + safeCloseAndRemove(fileWriter) + return "", probe.NewError(e) + } + } + + newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) + if md5Hex != "" { + if newMD5Hex != md5Hex { + safeCloseAndRemove(fileWriter) + return "", probe.NewError(BadDigest{md5Hex, newMD5Hex}) + } + } + e = fileWriter.Close() + if e != nil { + return "", probe.NewError(e) + } + return newMD5Hex, nil +} + +func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return ListPartsInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return ListPartsInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + } + // Create minio meta volume, if it doesn't exist yet. + if _, e := o.storage.StatVol(minioMetaVolume); e != nil { + if e == errVolumeNotFound { + e = o.storage.MakeVol(minioMetaVolume) + if e != nil { + return ListPartsInfo{}, probe.NewError(e) + } + } + } + if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil { + return ListPartsInfo{}, probe.NewError(e) + } else if !status { + return ListPartsInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID}) + } + result := ListPartsInfo{} + marker := "" + nextPartNumberMarker := 0 + uploadIDPath := path.Join(bucket, object, uploadID) + // Figure out the marker for the next subsequent calls, if the + // partNumberMarker is already set. + if partNumberMarker > 0 { + fileInfos, _, e := o.storage.ListFiles(minioMetaVolume, uploadIDPath+"."+strconv.Itoa(partNumberMarker)+".", "", false, 1) + if e != nil { + return result, probe.NewError(e) + } + if len(fileInfos) == 0 { + return result, probe.NewError(InvalidPart{}) + } + marker = fileInfos[0].Name + } + fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, uploadIDPath+".", marker, false, maxParts) + if e != nil { + return result, probe.NewError(InvalidPart{}) + } + for _, fileInfo := range fileInfos { + fileName := path.Base(fileInfo.Name) + splitResult := strings.Split(fileName, ".") + partNum, e := strconv.Atoi(splitResult[1]) + if e != nil { + return result, probe.NewError(e) + } + md5sum := splitResult[2] + result.Parts = append(result.Parts, partInfo{ + PartNumber: partNum, + LastModified: fileInfo.ModTime, + ETag: md5sum, + Size: fileInfo.Size, + }) + nextPartNumberMarker = partNum + } + result.Bucket = bucket + result.Object = object + result.UploadID = uploadID + result.PartNumberMarker = partNumberMarker + result.NextPartNumberMarker = nextPartNumberMarker + result.MaxParts = maxParts + result.IsTruncated = !eof + return result, nil +} + +// Create an s3 compatible MD5sum for complete multipart transaction. +func makeS3MD5(md5Strs ...string) (string, *probe.Error) { + var finalMD5Bytes []byte + for _, md5Str := range md5Strs { + md5Bytes, e := hex.DecodeString(md5Str) + if e != nil { + return "", probe.NewError(e) + } + finalMD5Bytes = append(finalMD5Bytes, md5Bytes...) + } + md5Hasher := md5.New() + md5Hasher.Write(finalMD5Bytes) + s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(md5Strs)) + return s3MD5, nil +} + +func (o objectAPI) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (string, *probe.Error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return "", probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return "", probe.NewError(ObjectNameInvalid{ + Bucket: bucket, + Object: object, + }) + } + if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil { + return "", probe.NewError(e) + } else if !status { + return "", probe.NewError(InvalidUploadID{UploadID: uploadID}) + } + fileWriter, e := o.storage.CreateFile(bucket, object) + if e != nil { + if e == errVolumeNotFound { + return "", probe.NewError(BucketNotFound{ + Bucket: bucket, + }) + } else if e == errIsNotRegular { + return "", probe.NewError(ObjectExistsAsPrefix{ + Bucket: bucket, + Prefix: object, + }) + } + return "", probe.NewError(e) + } + + var md5Sums []string + for _, part := range parts { + partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, part.PartNumber, part.ETag) + var fileReader io.ReadCloser + fileReader, e = o.storage.ReadFile(minioMetaVolume, path.Join(bucket, object, partSuffix), 0) + if e != nil { + if e == errFileNotFound { + return "", probe.NewError(InvalidPart{}) + } + return "", probe.NewError(e) + } + _, e = io.Copy(fileWriter, fileReader) + if e != nil { + return "", probe.NewError(e) + } + e = fileReader.Close() + if e != nil { + return "", probe.NewError(e) + } + md5Sums = append(md5Sums, part.ETag) + } + e = fileWriter.Close() + if e != nil { + return "", probe.NewError(e) + } + + // Save the s3 md5. + s3MD5, err := makeS3MD5(md5Sums...) + if err != nil { + return "", err.Trace(md5Sums...) + } + + // Cleanup all the parts. + o.removeMultipartUpload(bucket, object, uploadID) + + return s3MD5, nil +} + +func (o objectAPI) removeMultipartUpload(bucket, object, uploadID string) *probe.Error { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + } + marker := "" + for { + uploadIDPath := path.Join(bucket, object, uploadID) + fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, uploadIDPath, marker, false, 1000) + if e != nil { + return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) + } + for _, fileInfo := range fileInfos { + o.storage.DeleteFile(minioMetaVolume, fileInfo.Name) + marker = fileInfo.Name + } + if eof { + break + } + } + return nil +} + +func (o objectAPI) AbortMultipartUpload(bucket, object, uploadID string) *probe.Error { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + } + if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil { + return probe.NewError(e) + } else if !status { + return probe.NewError(InvalidUploadID{UploadID: uploadID}) + } + err := o.removeMultipartUpload(bucket, object, uploadID) + if err != nil { + return err.Trace(bucket, object, uploadID) + } + return nil +} diff --git a/object-api.go b/object-api.go new file mode 100644 index 000000000..bd2ad1ff2 --- /dev/null +++ b/object-api.go @@ -0,0 +1,336 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "crypto/md5" + "encoding/hex" + "errors" + "io" + "path/filepath" + "strings" + + "github.com/minio/minio/pkg/mimedb" + "github.com/minio/minio/pkg/probe" + "github.com/minio/minio/pkg/safe" +) + +type objectAPI struct { + storage StorageAPI +} + +func newObjectLayer(storage StorageAPI) *objectAPI { + return &objectAPI{storage} +} + +/// Bucket operations + +// MakeBucket - make a bucket. +func (o objectAPI) MakeBucket(bucket string) *probe.Error { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if e := o.storage.MakeVol(bucket); e != nil { + if e == errVolumeExists { + return probe.NewError(BucketExists{Bucket: bucket}) + } + return probe.NewError(e) + } + return nil +} + +// GetBucketInfo - get bucket info. +func (o objectAPI) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + vi, e := o.storage.StatVol(bucket) + if e != nil { + if e == errVolumeNotFound { + return BucketInfo{}, probe.NewError(BucketNotFound{Bucket: bucket}) + } + return BucketInfo{}, probe.NewError(e) + } + return BucketInfo{ + Name: bucket, + Created: vi.Created, + }, nil +} + +// ListBuckets - list buckets. +func (o objectAPI) ListBuckets() ([]BucketInfo, *probe.Error) { + var bucketInfos []BucketInfo + vols, e := o.storage.ListVols() + if e != nil { + return nil, probe.NewError(e) + } + for _, vol := range vols { + // StorageAPI can send volume names which are incompatible + // with buckets, handle it and skip them. + if !IsValidBucketName(vol.Name) { + continue + } + bucketInfos = append(bucketInfos, BucketInfo{ + Name: vol.Name, + Created: vol.Created, + }) + } + return bucketInfos, nil +} + +// DeleteBucket - delete a bucket. +func (o objectAPI) DeleteBucket(bucket string) *probe.Error { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if e := o.storage.DeleteVol(bucket); e != nil { + if e == errVolumeNotFound { + return probe.NewError(BucketNotFound{Bucket: bucket}) + } else if e == errVolumeNotEmpty { + return probe.NewError(BucketNotEmpty{Bucket: bucket}) + } + return probe.NewError(e) + } + return nil +} + +/// Object Operations + +// GetObject - get an object. +func (o objectAPI) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return nil, probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + // Verify if object is valid. + if !IsValidObjectName(object) { + return nil, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + } + r, e := o.storage.ReadFile(bucket, object, startOffset) + if e != nil { + if e == errVolumeNotFound { + return nil, probe.NewError(BucketNotFound{Bucket: bucket}) + } else if e == errFileNotFound { + return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) + } + return nil, probe.NewError(e) + } + return r, nil +} + +// GetObjectInfo - get object info. +func (o objectAPI) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + // Verify if object is valid. + if !IsValidObjectName(object) { + return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + } + fi, e := o.storage.StatFile(bucket, object) + if e != nil { + if e == errVolumeNotFound { + return ObjectInfo{}, probe.NewError(BucketNotFound{Bucket: bucket}) + } else if e == errFileNotFound || e == errIsNotRegular { + return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) + // Handle more lower level errors if needed. + } else { + return ObjectInfo{}, probe.NewError(e) + } + } + contentType := "application/octet-stream" + if objectExt := filepath.Ext(object); objectExt != "" { + content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))] + if ok { + contentType = content.ContentType + } + } + return ObjectInfo{ + Bucket: bucket, + Name: object, + ModTime: fi.ModTime, + Size: fi.Size, + IsDir: fi.Mode.IsDir(), + ContentType: contentType, + MD5Sum: "", // Read from metadata. + }, nil +} + +// safeCloseAndRemove - safely closes and removes underlying temporary +// file writer if possible. +func safeCloseAndRemove(writer io.WriteCloser) error { + // If writer is a safe file, Attempt to close and remove. + safeWriter, ok := writer.(*safe.File) + if ok { + return safeWriter.CloseAndRemove() + } + pipeWriter, ok := writer.(*io.PipeWriter) + if ok { + return pipeWriter.CloseWithError(errors.New("Close and error out.")) + } + return nil +} + +func (o objectAPI) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, *probe.Error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return "", probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return "", probe.NewError(ObjectNameInvalid{ + Bucket: bucket, + Object: object, + }) + } + fileWriter, e := o.storage.CreateFile(bucket, object) + if e != nil { + if e == errVolumeNotFound { + return "", probe.NewError(BucketNotFound{ + Bucket: bucket, + }) + } else if e == errIsNotRegular { + return "", probe.NewError(ObjectExistsAsPrefix{ + Bucket: bucket, + Prefix: object, + }) + } + return "", probe.NewError(e) + } + + // Initialize md5 writer. + md5Writer := md5.New() + + // Instantiate a new multi writer. + multiWriter := io.MultiWriter(md5Writer, fileWriter) + + // Instantiate checksum hashers and create a multiwriter. + if size > 0 { + if _, e = io.CopyN(multiWriter, data, size); e != nil { + safeCloseAndRemove(fileWriter) + return "", probe.NewError(e) + } + } else { + if _, e = io.Copy(multiWriter, data); e != nil { + safeCloseAndRemove(fileWriter) + return "", probe.NewError(e) + } + } + + newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) + // md5Hex representation. + var md5Hex string + if len(metadata) != 0 { + md5Hex = metadata["md5Sum"] + } + if md5Hex != "" { + if newMD5Hex != md5Hex { + safeCloseAndRemove(fileWriter) + return "", probe.NewError(BadDigest{md5Hex, newMD5Hex}) + } + } + e = fileWriter.Close() + if e != nil { + return "", probe.NewError(e) + } + + // Return md5sum. + return newMD5Hex, nil +} + +func (o objectAPI) DeleteObject(bucket, object string) *probe.Error { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectName(object) { + return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) + } + if e := o.storage.DeleteFile(bucket, object); e != nil { + if e == errVolumeNotFound { + return probe.NewError(BucketNotFound{Bucket: bucket}) + } + if e == errFileNotFound { + return probe.NewError(ObjectNotFound{ + Bucket: bucket, + Object: object, + }) + } + return probe.NewError(e) + } + return nil +} + +func (o objectAPI) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error) { + // Verify if bucket is valid. + if !IsValidBucketName(bucket) { + return ListObjectsInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) + } + if !IsValidObjectPrefix(prefix) { + return ListObjectsInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix}) + } + // Verify if delimiter is anything other than '/', which we do not support. + if delimiter != "" && delimiter != slashPathSeparator { + return ListObjectsInfo{}, probe.NewError(UnsupportedDelimiter{ + Delimiter: delimiter, + }) + } + // Verify if marker has prefix. + if marker != "" { + if !strings.HasPrefix(marker, prefix) { + return ListObjectsInfo{}, probe.NewError(InvalidMarkerPrefixCombination{ + Marker: marker, + Prefix: prefix, + }) + } + } + recursive := true + if delimiter == slashPathSeparator { + recursive = false + } + fileInfos, eof, e := o.storage.ListFiles(bucket, prefix, marker, recursive, maxKeys) + if e != nil { + if e == errVolumeNotFound { + return ListObjectsInfo{}, probe.NewError(BucketNotFound{Bucket: bucket}) + } + return ListObjectsInfo{}, probe.NewError(e) + } + if maxKeys == 0 { + return ListObjectsInfo{}, nil + } + result := ListObjectsInfo{IsTruncated: !eof} + for _, fileInfo := range fileInfos { + // With delimiter set we fill in NextMarker and Prefixes. + if delimiter == slashPathSeparator { + result.NextMarker = fileInfo.Name + if fileInfo.Mode.IsDir() { + result.Prefixes = append(result.Prefixes, fileInfo.Name) + continue + } + } + result.Objects = append(result.Objects, ObjectInfo{ + Name: fileInfo.Name, + ModTime: fileInfo.ModTime, + Size: fileInfo.Size, + IsDir: fileInfo.Mode.IsDir(), + }) + } + return result, nil +} diff --git a/fs-object_test.go b/object-api_test.go similarity index 54% rename from fs-object_test.go rename to object-api_test.go index d9958c294..82a473e89 100644 --- a/fs-object_test.go +++ b/object-api_test.go @@ -20,14 +20,13 @@ import ( "bytes" "crypto/md5" "encoding/hex" - "fmt" "io" "io/ioutil" "os" - "path/filepath" "strconv" - "strings" "testing" + + "github.com/minio/minio/pkg/probe" ) // Testing GetObjectInfo(). @@ -38,17 +37,21 @@ func TestGetObjectInfo(t *testing.T) { } defer os.RemoveAll(directory) - // Create the fs. - fs, err := newFS(directory) - if err != nil { - t.Fatal(err) + // Create the obj. + fs, e := newFS(directory) + if e != nil { + t.Fatal(e) } + + obj := newObjectLayer(fs) + var err *probe.Error + // This bucket is used for testing getObjectInfo operations. - err = fs.MakeBucket("test-getobjectinfo") + err = obj.MakeBucket("test-getobjectinfo") if err != nil { t.Fatal(err) } - _, err = fs.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) + _, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) if err != nil { t.Fatal(err) } @@ -76,8 +79,8 @@ func TestGetObjectInfo(t *testing.T) { {"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false}, {"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false}, // Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9). - {"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false}, - {"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false}, + {"abcdefgh", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "abcdefgh", Object: ""}, false}, + {"ijklmnop", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "ijklmnop", Object: ""}, false}, // Test cases with non-existing object name with existing bucket (Test number 10-12). {"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false}, {"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false}, @@ -88,7 +91,7 @@ func TestGetObjectInfo(t *testing.T) { {"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true}, } for i, testCase := range testCases { - result, err := fs.GetObjectInfo(testCase.bucketName, testCase.objectName) + result, err := obj.GetObjectInfo(testCase.bucketName, testCase.objectName) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Cause.Error()) } @@ -120,107 +123,25 @@ func TestGetObjectInfo(t *testing.T) { } } -// Testing getObjectInfo(). -func TestGetObjectInfoCore(t *testing.T) { - directory, e := ioutil.TempDir("", "minio-get-objinfo-test") - if e != nil { - t.Fatal(e) - } - defer os.RemoveAll(directory) - - // Create the fs. - fs, err := newFS(directory) - if err != nil { - t.Fatal(err) - } - // This bucket is used for testing getObjectInfo operations. - err = fs.MakeBucket("test-getobjinfo") - if err != nil { - t.Fatal(err) - } - _, err = fs.PutObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) - if err != nil { - t.Fatal(err) - } - resultCases := []ObjectInfo{ - // ObjectInfo - 1. - // ObjectName object name set to a existing directory in the test case. - {Bucket: "test-getobjinfo", Name: "Asia", Size: 0, ContentType: "application/octet-stream", IsDir: true}, - // ObjectInfo -2. - // ObjectName set to a existing object in the test case. - {Bucket: "test-getobjinfo", Name: "Asia/asiapics.jpg", Size: int64(len("asiapics")), ContentType: "image/jpeg", IsDir: false}, - // ObjectInfo-3. - // Object name set to a non-existing object in the test case. - {Bucket: "test-getobjinfo", Name: "Africa", Size: 0, ContentType: "image/jpeg", IsDir: false}, - } - testCases := []struct { - bucketName string - objectName string - - // Expected output of getObjectInfo. - result ObjectInfo - err error - - // Flag indicating whether the test is expected to pass or not. - shouldPass bool - }{ - // Testcase with object name set to a existing directory ( Test number 1). - {"test-getobjinfo", "Asia", resultCases[0], nil, true}, - // ObjectName set to a existing object ( Test number 2). - {"test-getobjinfo", "Asia/asiapics.jpg", resultCases[1], nil, true}, - // Object name set to a non-existing object. (Test number 3). - {"test-getobjinfo", "Africa", resultCases[2], fmt.Errorf("%s", filepath.FromSlash("test-getobjinfo/Africa")), false}, - } - rootPath := fs.(*Filesystem).GetRootPath() - for i, testCase := range testCases { - result, err := getObjectInfo(rootPath, testCase.bucketName, testCase.objectName) - if err != nil && testCase.shouldPass { - t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Cause.Error()) - } - if err == nil && !testCase.shouldPass { - t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) - } - // Failed as expected, but does it fail for the expected reason. - if err != nil && !testCase.shouldPass { - if !strings.Contains(err.Cause.Error(), testCase.err.Error()) { - t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error()) - } - } - - // Test passes as expected, but the output values are verified for correctness here. - if err == nil && testCase.shouldPass { - if testCase.result.Bucket != result.Bucket { - t.Fatalf("Test %d: Expected Bucket name to be '%s', but found '%s' instead", i+1, testCase.result.Bucket, result.Bucket) - } - if testCase.result.Name != result.Name { - t.Errorf("Test %d: Expected Object name to be %s, but instead found it to be %s", i+1, testCase.result.Name, result.Name) - } - if testCase.result.ContentType != result.ContentType { - t.Errorf("Test %d: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, testCase.result.ContentType, result.ContentType) - } - if testCase.result.IsDir != result.IsDir { - t.Errorf("Test %d: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, testCase.result.IsDir, result.IsDir) - } - } - } -} - func BenchmarkGetObject(b *testing.B) { - // Make a temporary directory to use as the fs. + // Make a temporary directory to use as the obj. directory, e := ioutil.TempDir("", "minio-benchmark-getobject") if e != nil { b.Fatal(e) } defer os.RemoveAll(directory) - // Create the fs. - fs, err := newFS(directory) - if err != nil { - b.Fatal(err) + // Create the obj. + fs, e := newFS(directory) + if e != nil { + b.Fatal(e) } + obj := newObjectLayer(fs) + var err *probe.Error + // Make a bucket and put in a few objects. - err = fs.MakeBucket("bucket") + err = obj.MakeBucket("bucket") if err != nil { b.Fatal(err) } @@ -231,7 +152,7 @@ func BenchmarkGetObject(b *testing.B) { metadata := make(map[string]string) for i := 0; i < 10; i++ { metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) - _, err = fs.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata) + _, err = obj.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata) if err != nil { b.Fatal(err) } @@ -240,7 +161,7 @@ func BenchmarkGetObject(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { var buffer = new(bytes.Buffer) - r, err := fs.GetObject("bucket", "object"+strconv.Itoa(i%10), 0) + r, err := obj.GetObject("bucket", "object"+strconv.Itoa(i%10), 0) if err != nil { b.Error(err) } diff --git a/object-datatypes.go b/object-datatypes.go index 0bce3b0c4..55faf2bb8 100644 --- a/object-datatypes.go +++ b/object-datatypes.go @@ -26,14 +26,13 @@ type BucketInfo struct { // ObjectInfo - object info. type ObjectInfo struct { - Bucket string - Name string - ModifiedTime time.Time - ContentType string - MD5Sum string - Size int64 - IsDir bool - Err error + Bucket string + Name string + ModTime time.Time + ContentType string + MD5Sum string + Size int64 + IsDir bool } // ListPartsInfo - various types of object resources. diff --git a/fs-errors.go b/object-errors.go similarity index 86% rename from fs-errors.go rename to object-errors.go index 03d7ed88e..5c4a8d015 100644 --- a/fs-errors.go +++ b/object-errors.go @@ -134,6 +134,34 @@ func (e BadDigest) Error() string { return "Bad digest expected " + e.ExpectedMD5 + " is not valid with what we calculated " + e.CalculatedMD5 } +// UnsupportedDelimiter - unsupported delimiter. +type UnsupportedDelimiter struct { + Delimiter string +} + +func (e UnsupportedDelimiter) Error() string { + return fmt.Sprintf("delimiter '%s' is not supported. Only '/' is supported", e.Delimiter) +} + +// InvalidUploadIDKeyCombination - invalid upload id and key marker +// combination. +type InvalidUploadIDKeyCombination struct { + UploadIDMarker, KeyMarker string +} + +func (e InvalidUploadIDKeyCombination) Error() string { + return fmt.Sprintf("Invalid combination of uploadID marker '%s' and marker '%s'", e.UploadIDMarker, e.KeyMarker) +} + +// InvalidMarkerPrefixCombination - invalid marker and prefix combination. +type InvalidMarkerPrefixCombination struct { + Marker, Prefix string +} + +func (e InvalidMarkerPrefixCombination) Error() string { + return fmt.Sprintf("Invalid combination of marker '%s' and prefix '%s'", e.Marker, e.Prefix) +} + // InternalError - generic internal error type InternalError struct{} @@ -262,7 +290,16 @@ func (e InvalidRange) Error() string { /// Multipart related errors -// InvalidUploadID invalid upload id +// MalformedUploadID malformed upload id. +type MalformedUploadID struct { + UploadID string +} + +func (e MalformedUploadID) Error() string { + return "Malformed upload id " + e.UploadID +} + +// InvalidUploadID invalid upload id. type InvalidUploadID struct { UploadID string } diff --git a/object-handlers.go b/object-handlers.go index 7409dffc5..3424db5b0 100644 --- a/object-handlers.go +++ b/object-handlers.go @@ -74,7 +74,7 @@ func errAllowableObjectNotFound(bucket string, r *http.Request) APIErrorCode { // ---------- // This implementation of the GET operation retrieves object. To use GET, // you must have READ access to the object. -func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) { var object, bucket string vars := mux.Vars(r) bucket = vars["bucket"] @@ -97,7 +97,7 @@ func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Requ return } } - + // Fetch object stat info. objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object) if err != nil { switch err.ToGoError().(type) { @@ -117,7 +117,7 @@ func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Requ } // Verify 'If-Modified-Since' and 'If-Unmodified-Since'. - lastModified := objInfo.ModifiedTime + lastModified := objInfo.ModTime if checkLastModified(w, r, lastModified) { return } @@ -137,8 +137,15 @@ func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Requ startOffset := hrange.start readCloser, err := api.ObjectAPI.GetObject(bucket, object, startOffset) if err != nil { - errorIf(err.Trace(), "GetObject failed.", nil) - writeErrorResponse(w, r, ErrInternalError, r.URL.Path) + switch err.ToGoError().(type) { + case BucketNotFound: + writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) + case ObjectNotFound: + writeErrorResponse(w, r, errAllowableObjectNotFound(bucket, r), r.URL.Path) + default: + errorIf(err.Trace(), "GetObject failed.", nil) + writeErrorResponse(w, r, ErrInternalError, r.URL.Path) + } return } defer readCloser.Close() // Close after this handler returns. @@ -261,7 +268,7 @@ func checkETag(w http.ResponseWriter, r *http.Request) bool { // HeadObjectHandler - HEAD Object // ----------- // The HEAD operation retrieves metadata from an object without returning the object itself. -func (api objectStorageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { var object, bucket string vars := mux.Vars(r) bucket = vars["bucket"] @@ -304,7 +311,7 @@ func (api objectStorageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Req } // Verify 'If-Modified-Since' and 'If-Unmodified-Since'. - lastModified := objInfo.ModifiedTime + lastModified := objInfo.ModTime if checkLastModified(w, r, lastModified) { return } @@ -325,7 +332,7 @@ func (api objectStorageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Req // ---------- // This implementation of the PUT operation adds an object to a bucket // while reading the object from another source. -func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -399,7 +406,7 @@ func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Req // Verify x-amz-copy-source-if-modified-since and // x-amz-copy-source-if-unmodified-since. - lastModified := objInfo.ModifiedTime + lastModified := objInfo.ModTime if checkCopySourceLastModified(w, r, lastModified) { return } @@ -450,9 +457,8 @@ func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Req metadata["md5Sum"] = hex.EncodeToString(md5Bytes) // Create the object. - objInfo, err = api.ObjectAPI.PutObject(bucket, object, size, readCloser, metadata) + md5Sum, err := api.ObjectAPI.PutObject(bucket, object, size, readCloser, metadata) if err != nil { - errorIf(err.Trace(), "PutObject failed.", nil) switch err.ToGoError().(type) { case RootPathFull: writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) @@ -467,11 +473,20 @@ func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Req case ObjectExistsAsPrefix: writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path) default: + errorIf(err.Trace(), "PutObject failed.", nil) writeErrorResponse(w, r, ErrInternalError, r.URL.Path) } return } - response := generateCopyObjectResponse(objInfo.MD5Sum, objInfo.ModifiedTime) + + objInfo, err = api.ObjectAPI.GetObjectInfo(bucket, object) + if err != nil { + errorIf(err.Trace(), "GetObjectInfo failed.", nil) + writeErrorResponse(w, r, ErrInternalError, r.URL.Path) + return + } + + response := generateCopyObjectResponse(md5Sum, objInfo.ModTime) encodedSuccessResponse := encodeResponse(response) // write headers setCommonHeaders(w) @@ -576,7 +591,7 @@ func checkCopySourceETag(w http.ResponseWriter, r *http.Request) bool { // PutObjectHandler - PUT Object // ---------- // This implementation of the PUT operation adds an object to a bucket. -func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) { // If the matching failed, it means that the X-Amz-Copy-Source was // wrong, fail right here. if _, ok := r.Header["X-Amz-Copy-Source"]; ok { @@ -606,7 +621,7 @@ func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Requ return } - var objInfo ObjectInfo + var md5Sum string switch getRequestAuthType(r) { default: // For all unknown auth types return error. @@ -619,7 +634,7 @@ func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Requ return } // Create anonymous object. - objInfo, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, nil) + md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, nil) case authTypePresigned, authTypeSigned: // Initialize a pipe for data pipe line. reader, writer := io.Pipe() @@ -658,7 +673,7 @@ func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Requ // Make sure we hex encode here. metadata["md5"] = hex.EncodeToString(md5Bytes) // Create object. - objInfo, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata) + md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata) } if err != nil { errorIf(err.Trace(), "PutObject failed.", nil) @@ -686,16 +701,16 @@ func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Requ } return } - if objInfo.MD5Sum != "" { - w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"") + if md5Sum != "" { + w.Header().Set("ETag", "\""+md5Sum+"\"") } writeSuccessResponse(w, nil) } -/// Multipart objectStorageAPI +/// Multipart objectAPIHandlers // NewMultipartUploadHandler - New multipart upload -func (api objectStorageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { var object, bucket string vars := mux.Vars(r) bucket = vars["bucket"] @@ -748,7 +763,7 @@ func (api objectStorageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r * } // PutObjectPartHandler - Upload part -func (api objectStorageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -861,7 +876,7 @@ func (api objectStorageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http. } // AbortMultipartUploadHandler - Abort multipart upload -func (api objectStorageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -908,7 +923,7 @@ func (api objectStorageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r } // ListObjectPartsHandler - List object parts -func (api objectStorageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -958,6 +973,8 @@ func (api objectStorageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *htt writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) case InvalidUploadID: writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) + case InvalidPart: + writeErrorResponse(w, r, ErrInvalidPart, r.URL.Path) default: writeErrorResponse(w, r, ErrInternalError, r.URL.Path) } @@ -972,7 +989,7 @@ func (api objectStorageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *htt } // CompleteMultipartUploadHandler - Complete multipart upload -func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] @@ -980,7 +997,7 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter // Get upload id. uploadID, _, _, _ := getObjectResources(r.URL.Query()) - var objInfo ObjectInfo + var md5Sum string var err *probe.Error switch getRequestAuthType(r) { default: @@ -1007,6 +1024,7 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter } complMultipartUpload := &completeMultipartUpload{} if e = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); e != nil { + errorIf(probe.NewError(e), "XML Unmarshal failed", nil) writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path) return } @@ -1015,10 +1033,14 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter return } // Complete parts. - completeParts := complMultipartUpload.Parts - + var completeParts []completePart + for _, part := range complMultipartUpload.Parts { + part.ETag = strings.TrimPrefix(part.ETag, "\"") + part.ETag = strings.TrimSuffix(part.ETag, "\"") + completeParts = append(completeParts, part) + } // Complete multipart upload. - objInfo, err = api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts) + md5Sum, err = api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts) if err != nil { errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil) switch err.ToGoError().(type) { @@ -1046,7 +1068,7 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter // Get object location. location := getLocation(r) // Generate complete multipart response. - response := generateCompleteMultpartUploadResponse(bucket, object, location, objInfo.MD5Sum) + response := generateCompleteMultpartUploadResponse(bucket, object, location, md5Sum) encodedSuccessResponse := encodeResponse(response) // Write headers. setCommonHeaders(w) @@ -1054,10 +1076,10 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter writeSuccessResponse(w, encodedSuccessResponse) } -/// Delete objectStorageAPI +/// Delete objectAPIHandlers // DeleteObjectHandler - delete an object -func (api objectStorageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { +func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] diff --git a/object-interface.go b/object-interface.go deleted file mode 100644 index 06ab7d0f9..000000000 --- a/object-interface.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/object-utils.go b/object-utils.go new file mode 100644 index 000000000..1c31becb2 --- /dev/null +++ b/object-utils.go @@ -0,0 +1,95 @@ +/* + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "regexp" + "strings" + "unicode/utf8" +) + +const ( + slashPathSeparator = "/" +) + +// validBucket regexp. +var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`) + +// IsValidBucketName verifies a bucket name in accordance with Amazon's +// requirements. It must be 3-63 characters long, can contain dashes +// and periods, but must begin and end with a lowercase letter or a number. +// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html +func IsValidBucketName(bucket string) bool { + if len(bucket) < 3 || len(bucket) > 63 { + return false + } + if bucket[0] == '.' || bucket[len(bucket)-1] == '.' { + return false + } + return validBucket.MatchString(bucket) +} + +// IsValidObjectName verifies an object name in accordance with Amazon's +// requirements. It cannot exceed 1024 characters and must be a valid UTF8 +// string. +// +// See: +// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html +// +// You should avoid the following characters in a key name because of +// significant special handling for consistency across all +// applications. +// +// Rejects strings with following characters. +// +// - Backslash ("\") +// - Left curly brace ("{") +// - Caret ("^") +// - Right curly brace ("}") +// - Grave accent / back tick ("`") +// - Right square bracket ("]") +// - Left square bracket ("[") +// - Tilde ("~") +// - 'Greater Than' symbol (">") +// - 'Less Than' symbol ("<") +// - Vertical bar / pipe ("|") +func IsValidObjectName(object string) bool { + if len(object) > 1024 || len(object) == 0 { + return false + } + if !utf8.ValidString(object) { + return false + } + // Reject unsupported characters in object name. + return !strings.ContainsAny(object, "`^*{}[]|\\\"'") +} + +// IsValidObjectPrefix verifies whether the prefix is a valid object name. +// Its valid to have a empty prefix. +func IsValidObjectPrefix(object string) bool { + // Prefix can be empty or "/". + if object == "" || object == "/" { + return true + } + // Verify if prefix is a valid object name. + return IsValidObjectName(object) + +} + +func pathJoin(path1 string, path2 string) string { + return strings.TrimSuffix(path1, slashPathSeparator) + slashPathSeparator + path2 +} diff --git a/fs-utils_test.go b/object-utils_test.go similarity index 100% rename from fs-utils_test.go rename to object-utils_test.go diff --git a/fs_api_suite_test.go b/object_api_suite_test.go similarity index 63% rename from fs_api_suite_test.go rename to object_api_suite_test.go index 757d43846..859533a7f 100644 --- a/fs_api_suite_test.go +++ b/object_api_suite_test.go @@ -28,7 +28,7 @@ import ( ) // APITestSuite - collection of API tests -func APITestSuite(c *check.C, create func() ObjectAPI) { +func APITestSuite(c *check.C, create func() *objectAPI) { testMakeBucket(c, create) testMultipleObjectCreation(c, create) testPaging(c, create) @@ -46,17 +46,17 @@ func APITestSuite(c *check.C, create func() ObjectAPI) { testMultipartObjectAbort(c, create) } -func testMakeBucket(c *check.C, create func() ObjectAPI) { - fs := create() - err := fs.MakeBucket("bucket") +func testMakeBucket(c *check.C, create func() *objectAPI) { + obj := create() + err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) } -func testMultipartObjectCreation(c *check.C, create func() ObjectAPI) { - fs := create() - err := fs.MakeBucket("bucket") +func testMultipartObjectCreation(c *check.C, create func() *objectAPI) { + obj := create() + err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) - uploadID, err := fs.NewMultipartUpload("bucket", "key") + uploadID, err := obj.NewMultipartUpload("bucket", "key") c.Assert(err, check.IsNil) completedParts := completeMultipartUpload{} @@ -72,21 +72,21 @@ func testMultipartObjectCreation(c *check.C, create func() ObjectAPI) { expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) var calculatedMD5sum string - calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex) + calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex) c.Assert(err, check.IsNil) c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) completedParts.Parts = append(completedParts.Parts, completePart{PartNumber: i, ETag: calculatedMD5sum}) } - objInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts) + md5Sum, err := obj.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts) c.Assert(err, check.IsNil) - c.Assert(objInfo.MD5Sum, check.Equals, "3605d84b1c43b1a664aa7c0d5082d271-10") + c.Assert(md5Sum, check.Equals, "3605d84b1c43b1a664aa7c0d5082d271-10") } -func testMultipartObjectAbort(c *check.C, create func() ObjectAPI) { - fs := create() - err := fs.MakeBucket("bucket") +func testMultipartObjectAbort(c *check.C, create func() *objectAPI) { + obj := create() + err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) - uploadID, err := fs.NewMultipartUpload("bucket", "key") + uploadID, err := obj.NewMultipartUpload("bucket", "key") c.Assert(err, check.IsNil) parts := make(map[int]string) @@ -104,19 +104,19 @@ func testMultipartObjectAbort(c *check.C, create func() ObjectAPI) { metadata["md5"] = expectedMD5Sumhex var calculatedMD5sum string - calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex) + calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex) c.Assert(err, check.IsNil) c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) parts[i] = expectedMD5Sumhex } - err = fs.AbortMultipartUpload("bucket", "key", uploadID) + err = obj.AbortMultipartUpload("bucket", "key", uploadID) c.Assert(err, check.IsNil) } -func testMultipleObjectCreation(c *check.C, create func() ObjectAPI) { +func testMultipleObjectCreation(c *check.C, create func() *objectAPI) { objects := make(map[string][]byte) - fs := create() - err := fs.MakeBucket("bucket") + obj := create() + err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) for i := 0; i < 10; i++ { randomPerm := rand.Perm(10) @@ -133,40 +133,40 @@ func testMultipleObjectCreation(c *check.C, create func() ObjectAPI) { objects[key] = []byte(randomString) metadata := make(map[string]string) metadata["md5Sum"] = expectedMD5Sumhex - objInfo, err := fs.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata) + md5Sum, err := obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata) c.Assert(err, check.IsNil) - c.Assert(objInfo.MD5Sum, check.Equals, expectedMD5Sumhex) + c.Assert(md5Sum, check.Equals, expectedMD5Sumhex) } for key, value := range objects { var byteBuffer bytes.Buffer - r, err := fs.GetObject("bucket", key, 0) + r, err := obj.GetObject("bucket", key, 0) c.Assert(err, check.IsNil) _, e := io.Copy(&byteBuffer, r) c.Assert(e, check.IsNil) c.Assert(byteBuffer.Bytes(), check.DeepEquals, value) c.Assert(r.Close(), check.IsNil) - objInfo, err := fs.GetObjectInfo("bucket", key) + objInfo, err := obj.GetObjectInfo("bucket", key) c.Assert(err, check.IsNil) c.Assert(objInfo.Size, check.Equals, int64(len(value))) r.Close() } } -func testPaging(c *check.C, create func() ObjectAPI) { - fs := create() - fs.MakeBucket("bucket") - result, err := fs.ListObjects("bucket", "", "", "", 0) +func testPaging(c *check.C, create func() *objectAPI) { + obj := create() + obj.MakeBucket("bucket") + result, err := obj.ListObjects("bucket", "", "", "", 0) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, 0) c.Assert(result.IsTruncated, check.Equals, false) // check before paging occurs for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = obj.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) - result, err = fs.ListObjects("bucket", "", "", "", 5) + result, err = obj.ListObjects("bucket", "", "", "", 5) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, i+1) c.Assert(result.IsTruncated, check.Equals, false) @@ -174,27 +174,27 @@ func testPaging(c *check.C, create func() ObjectAPI) { // check after paging occurs pages work for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) - _, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) + _, err = obj.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) c.Assert(err, check.IsNil) - result, err = fs.ListObjects("bucket", "obj", "", "", 5) + result, err = obj.ListObjects("bucket", "obj", "", "", 5) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, 5) c.Assert(result.IsTruncated, check.Equals, true) } // check paging with prefix at end returns less objects { - _, err = fs.PutObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) + _, err = obj.PutObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) - _, err = fs.PutObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) + _, err = obj.PutObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) - result, err = fs.ListObjects("bucket", "new", "", "", 5) + result, err = obj.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, 2) } // check ordering of pages { - result, err = fs.ListObjects("bucket", "", "", "", 1000) + result, err = obj.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "newPrefix") c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2") @@ -205,11 +205,11 @@ func testPaging(c *check.C, create func() ObjectAPI) { // check delimited results with delimiter and prefix { - _, err = fs.PutObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) + _, err = obj.PutObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) c.Assert(err, check.IsNil) - _, err = fs.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) + _, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) c.Assert(err, check.IsNil) - result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10) + result, err = obj.ListObjects("bucket", "this/is/", "", "/", 10) c.Assert(err, check.IsNil) c.Assert(len(result.Objects), check.Equals, 1) c.Assert(result.Prefixes[0], check.Equals, "this/is/also/") @@ -217,7 +217,7 @@ func testPaging(c *check.C, create func() ObjectAPI) { // check delimited results with delimiter without prefix { - result, err = fs.ListObjects("bucket", "", "", "/", 1000) + result, err = obj.ListObjects("bucket", "", "", "/", 1000) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "newPrefix") c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2") @@ -229,7 +229,7 @@ func testPaging(c *check.C, create func() ObjectAPI) { // check results with Marker { - result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3) + result, err = obj.ListObjects("bucket", "", "newPrefix", "", 3) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "newPrefix2") c.Assert(result.Objects[1].Name, check.Equals, "obj0") @@ -237,7 +237,7 @@ func testPaging(c *check.C, create func() ObjectAPI) { } // check ordering of results with prefix { - result, err = fs.ListObjects("bucket", "obj", "", "", 1000) + result, err = obj.ListObjects("bucket", "obj", "", "", 1000) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "obj0") c.Assert(result.Objects[1].Name, check.Equals, "obj1") @@ -247,27 +247,27 @@ func testPaging(c *check.C, create func() ObjectAPI) { } // check ordering of results with prefix and no paging { - result, err = fs.ListObjects("bucket", "new", "", "", 5) + result, err = obj.ListObjects("bucket", "new", "", "", 5) c.Assert(err, check.IsNil) c.Assert(result.Objects[0].Name, check.Equals, "newPrefix") c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2") } } -func testObjectOverwriteWorks(c *check.C, create func() ObjectAPI) { - fs := create() - err := fs.MakeBucket("bucket") +func testObjectOverwriteWorks(c *check.C, create func() *objectAPI) { + obj := create() + err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) - _, err = fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) + _, err = obj.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.IsNil) // c.Assert(md5Sum1hex, check.Equals, objInfo.MD5Sum) - _, err = fs.PutObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil) + _, err = obj.PutObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil) c.Assert(err, check.IsNil) var bytesBuffer bytes.Buffer - r, err := fs.GetObject("bucket", "object", 0) + r, err := obj.GetObject("bucket", "object", 0) c.Assert(err, check.IsNil) _, e := io.Copy(&bytesBuffer, r) c.Assert(e, check.IsNil) @@ -275,30 +275,30 @@ func testObjectOverwriteWorks(c *check.C, create func() ObjectAPI) { c.Assert(r.Close(), check.IsNil) } -func testNonExistantBucketOperations(c *check.C, create func() ObjectAPI) { - fs := create() - _, err := fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) +func testNonExistantBucketOperations(c *check.C, create func() *objectAPI) { + obj := create() + _, err := obj.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.Not(check.IsNil)) } -func testBucketRecreateFails(c *check.C, create func() ObjectAPI) { - fs := create() - err := fs.MakeBucket("string") +func testBucketRecreateFails(c *check.C, create func() *objectAPI) { + obj := create() + err := obj.MakeBucket("string") c.Assert(err, check.IsNil) - err = fs.MakeBucket("string") + err = obj.MakeBucket("string") c.Assert(err, check.Not(check.IsNil)) } -func testPutObjectInSubdir(c *check.C, create func() ObjectAPI) { - fs := create() - err := fs.MakeBucket("bucket") +func testPutObjectInSubdir(c *check.C, create func() *objectAPI) { + obj := create() + err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) - _, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) + _, err = obj.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) c.Assert(err, check.IsNil) var bytesBuffer bytes.Buffer - r, err := fs.GetObject("bucket", "dir1/dir2/object", 0) + r, err := obj.GetObject("bucket", "dir1/dir2/object", 0) c.Assert(err, check.IsNil) n, e := io.Copy(&bytesBuffer, r) c.Assert(e, check.IsNil) @@ -307,49 +307,49 @@ func testPutObjectInSubdir(c *check.C, create func() ObjectAPI) { c.Assert(r.Close(), check.IsNil) } -func testListBuckets(c *check.C, create func() ObjectAPI) { - fs := create() +func testListBuckets(c *check.C, create func() *objectAPI) { + obj := create() // test empty list - buckets, err := fs.ListBuckets() + buckets, err := obj.ListBuckets() c.Assert(err, check.IsNil) c.Assert(len(buckets), check.Equals, 0) // add one and test exists - err = fs.MakeBucket("bucket1") + err = obj.MakeBucket("bucket1") c.Assert(err, check.IsNil) - buckets, err = fs.ListBuckets() + buckets, err = obj.ListBuckets() c.Assert(len(buckets), check.Equals, 1) c.Assert(err, check.IsNil) // add two and test exists - err = fs.MakeBucket("bucket2") + err = obj.MakeBucket("bucket2") c.Assert(err, check.IsNil) - buckets, err = fs.ListBuckets() + buckets, err = obj.ListBuckets() c.Assert(len(buckets), check.Equals, 2) c.Assert(err, check.IsNil) // add three and test exists + prefix - err = fs.MakeBucket("bucket22") + err = obj.MakeBucket("bucket22") - buckets, err = fs.ListBuckets() + buckets, err = obj.ListBuckets() c.Assert(len(buckets), check.Equals, 3) c.Assert(err, check.IsNil) } -func testListBucketsOrder(c *check.C, create func() ObjectAPI) { +func testListBucketsOrder(c *check.C, create func() *objectAPI) { // if implementation contains a map, order of map keys will vary. // this ensures they return in the same order each time for i := 0; i < 10; i++ { - fs := create() + obj := create() // add one and test exists - err := fs.MakeBucket("bucket1") + err := obj.MakeBucket("bucket1") c.Assert(err, check.IsNil) - err = fs.MakeBucket("bucket2") + err = obj.MakeBucket("bucket2") c.Assert(err, check.IsNil) - buckets, err := fs.ListBuckets() + buckets, err := obj.ListBuckets() c.Assert(err, check.IsNil) c.Assert(len(buckets), check.Equals, 2) c.Assert(buckets[0].Name, check.Equals, "bucket1") @@ -357,20 +357,20 @@ func testListBucketsOrder(c *check.C, create func() ObjectAPI) { } } -func testListObjectsTestsForNonExistantBucket(c *check.C, create func() ObjectAPI) { - fs := create() - result, err := fs.ListObjects("bucket", "", "", "", 1000) +func testListObjectsTestsForNonExistantBucket(c *check.C, create func() *objectAPI) { + obj := create() + result, err := obj.ListObjects("bucket", "", "", "", 1000) c.Assert(err, check.Not(check.IsNil)) c.Assert(result.IsTruncated, check.Equals, false) c.Assert(len(result.Objects), check.Equals, 0) } -func testNonExistantObjectInBucket(c *check.C, create func() ObjectAPI) { - fs := create() - err := fs.MakeBucket("bucket") +func testNonExistantObjectInBucket(c *check.C, create func() *objectAPI) { + obj := create() + err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) - _, err = fs.GetObject("bucket", "dir1", 0) + _, err = obj.GetObject("bucket", "dir1", 0) c.Assert(err, check.Not(check.IsNil)) switch err := err.ToGoError().(type) { case ObjectNotFound: @@ -380,44 +380,44 @@ func testNonExistantObjectInBucket(c *check.C, create func() ObjectAPI) { } } -func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() ObjectAPI) { - fs := create() - err := fs.MakeBucket("bucket") +func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() *objectAPI) { + obj := create() + err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) - _, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) + _, err = obj.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) c.Assert(err, check.IsNil) - _, err = fs.GetObject("bucket", "dir1", 0) + _, err = obj.GetObject("bucket", "dir1", 0) switch err := err.ToGoError().(type) { - case ObjectExistsAsPrefix: + case ObjectNotFound: c.Assert(err.Bucket, check.Equals, "bucket") - c.Assert(err.Prefix, check.Equals, "dir1") + c.Assert(err.Object, check.Equals, "dir1") default: // force a failure with a line number - c.Assert(err.Error(), check.Equals, "Object exists on : bucket as prefix dir1") + c.Assert(err, check.Equals, "ObjectNotFound") } - _, err = fs.GetObject("bucket", "dir1/", 0) + _, err = obj.GetObject("bucket", "dir1/", 0) switch err := err.ToGoError().(type) { - case ObjectExistsAsPrefix: + case ObjectNotFound: c.Assert(err.Bucket, check.Equals, "bucket") - c.Assert(err.Prefix, check.Equals, "dir1/") + c.Assert(err.Object, check.Equals, "dir1/") default: // force a failure with a line number - c.Assert(err.Error(), check.Equals, "Object exists on : bucket as prefix dir1") + c.Assert(err, check.Equals, "ObjectNotFound") } } -func testDefaultContentType(c *check.C, create func() ObjectAPI) { - fs := create() - err := fs.MakeBucket("bucket") +func testDefaultContentType(c *check.C, create func() *objectAPI) { + obj := create() + err := obj.MakeBucket("bucket") c.Assert(err, check.IsNil) // Test empty - _, err = fs.PutObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil) + _, err = obj.PutObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil) c.Assert(err, check.IsNil) - objInfo, err := fs.GetObjectInfo("bucket", "one") + objInfo, err := obj.GetObjectInfo("bucket", "one") c.Assert(err, check.IsNil) c.Assert(objInfo.ContentType, check.Equals, "application/octet-stream") } diff --git a/routers.go b/routers.go index cc363424e..be0716153 100644 --- a/routers.go +++ b/routers.go @@ -18,19 +18,39 @@ package main import ( "net/http" + "os" router "github.com/gorilla/mux" + "github.com/minio/minio/pkg/probe" ) // configureServer handler returns final handler for the http server. -func configureServerHandler(objectAPI ObjectAPI) http.Handler { +func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler { + var storageHandlers StorageAPI + if len(srvCmdConfig.exportPaths) == 1 { + // Verify if export path is a local file system path. + st, e := os.Stat(srvCmdConfig.exportPaths[0]) + if e == nil && st.Mode().IsDir() { + // Initialize storage API. + storageHandlers, e = newFS(srvCmdConfig.exportPaths[0]) + fatalIf(probe.NewError(e), "Initializing fs failed.", nil) + } else { + // Initialize storage API. + storageHandlers, e = newNetworkFS(srvCmdConfig.exportPaths[0]) + fatalIf(probe.NewError(e), "Initializing network fs failed.", nil) + } + } // else if - XL part. + + // Initialize object layer. + objectAPI := newObjectLayer(storageHandlers) + // Initialize API. - api := objectStorageAPI{ + apiHandlers := objectAPIHandlers{ ObjectAPI: objectAPI, } // Initialize Web. - web := &webAPI{ + webHandlers := &webAPIHandlers{ ObjectAPI: objectAPI, } @@ -38,8 +58,9 @@ func configureServerHandler(objectAPI ObjectAPI) http.Handler { mux := router.NewRouter() // Register all routers. - registerWebRouter(mux, web) - registerAPIRouter(mux, api) + registerStorageRPCRouter(mux, storageHandlers) + registerWebRouter(mux, webHandlers) + registerAPIRouter(mux, apiHandlers) // Add new routers here. // List of some generic handlers which are applied for all diff --git a/server-main.go b/server-main.go index 360bd888f..4dba2a0f8 100644 --- a/server-main.go +++ b/server-main.go @@ -69,12 +69,17 @@ EXAMPLES: `, } +type serverCmdConfig struct { + serverAddr string + exportPaths []string +} + // configureServer configure a new server instance -func configureServer(serverAddr string, objectAPI ObjectAPI) *http.Server { +func configureServer(srvCmdConfig serverCmdConfig) *http.Server { // Minio server config apiServer := &http.Server{ - Addr: serverAddr, - Handler: configureServerHandler(objectAPI), + Addr: srvCmdConfig.serverAddr, + Handler: configureServerHandler(srvCmdConfig), MaxHeaderBytes: 1 << 20, } @@ -148,7 +153,7 @@ func initServerConfig(c *cli.Context) { // Check server arguments. func checkServerSyntax(c *cli.Context) { - if c.Args().First() == "help" { + if !c.Args().Present() && c.Args().First() == "help" { cli.ShowCommandHelpAndExit(c, "server", 1) } if len(c.Args()) > 2 { @@ -255,25 +260,17 @@ func serverMain(c *cli.Context) { } } - // Check configured ports. + // Check if requested port is available. checkPortAvailability(getPort(net.JoinHostPort(host, port))) - var objectAPI ObjectAPI - var err *probe.Error - - // Set backend FS type. - fsPath := strings.TrimSpace(c.Args().Get(0)) - if fsPath != "" { - // Last argument is always a file system path, verify if it exists and is accessible. - _, e := os.Stat(fsPath) - fatalIf(probe.NewError(e), "Unable to validate the path", nil) - // Initialize filesystem storage layer. - objectAPI, err = newFS(fsPath) - fatalIf(err.Trace(fsPath), "Initializing filesystem failed.", nil) - } + // Save all command line args as export paths. + exportPaths := c.Args() // Configure server. - apiServer := configureServer(serverAddress, objectAPI) + apiServer := configureServer(serverCmdConfig{ + serverAddr: serverAddress, + exportPaths: exportPaths, + }) // Credential. cred := serverConfig.GetCredential() @@ -304,6 +301,6 @@ func serverMain(c *cli.Context) { } // Start server. - err = minhttp.ListenAndServe(apiServer) + err := minhttp.ListenAndServe(apiServer) errorIf(err.Trace(), "Failed to start the minio server.", nil) } diff --git a/server_fs_test.go b/server_test.go similarity index 99% rename from server_fs_test.go rename to server_test.go index f94c03fea..2f5d9c044 100644 --- a/server_fs_test.go +++ b/server_test.go @@ -96,10 +96,10 @@ func (s *MyAPISuite) SetUpSuite(c *C) { // Save config. c.Assert(serverConfig.Save(), IsNil) - fs, err := newFS(fsroot) - c.Assert(err, IsNil) - - apiServer := configureServer(addr, fs) + apiServer := configureServer(serverCmdConfig{ + serverAddr: addr, + exportPaths: []string{fsroot}, + }) testAPIFSCacheServer = httptest.NewServer(apiServer.Handler) } @@ -1172,6 +1172,7 @@ func (s *MyAPISuite) TestValidateObjectMultipartUploadID(c *C) { c.Assert(err, IsNil) response, err = client.Do(request) + c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) decoder := xml.NewDecoder(response.Body) diff --git a/storage-api-interface.go b/storage-api-interface.go index 8c54760db..eeb0d4070 100644 --- a/storage-api-interface.go +++ b/storage-api-interface.go @@ -27,7 +27,7 @@ type StorageAPI interface { DeleteVol(volume string) (err error) // File operations. - ListFiles(volume, prefix, marker string, recursive bool, count int) (files []FileInfo, isEOF bool, err error) + ListFiles(volume, prefix, marker string, recursive bool, count int) (files []FileInfo, eof bool, err error) ReadFile(volume string, path string, offset int64) (readCloser io.ReadCloser, err error) CreateFile(volume string, path string) (writeCloser io.WriteCloser, err error) StatFile(volume string, path string) (file FileInfo, err error) diff --git a/storage-common.go b/storage-common.go deleted file mode 100644 index 945f35f2e..000000000 --- a/storage-common.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "io" - "os" -) - -// isDirEmpty - returns whether given directory is empty or not. -func isDirEmpty(dirname string) (status bool, err error) { - f, err := os.Open(dirname) - if err == nil { - defer f.Close() - if _, err = f.Readdirnames(1); err == io.EOF { - status = true - err = nil - } - } - return status, err -} diff --git a/storage-datatypes.go b/storage-datatypes.go index da8c8a5f3..4c0f9ac77 100644 --- a/storage-datatypes.go +++ b/storage-datatypes.go @@ -1,19 +1,3 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package main import ( @@ -21,17 +5,17 @@ import ( "time" ) +// VolInfo - volume info +type VolInfo struct { + Name string + Created time.Time +} + // FileInfo - file stat information. type FileInfo struct { Volume string Name string ModTime time.Time Size int64 - Type os.FileMode -} - -// VolInfo - volume info -type VolInfo struct { - Name string - Created time.Time + Mode os.FileMode } diff --git a/storage-errors.go b/storage-errors.go new file mode 100644 index 000000000..0e78ffdab --- /dev/null +++ b/storage-errors.go @@ -0,0 +1,45 @@ +/* + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import "errors" + +// errDiskPathFull - cannot create volume or files when disk is full. +var errDiskPathFull = errors.New("Disk path full.") + +// errFileNotFound - cannot find the file. +var errFileNotFound = errors.New("File not found.") + +// errVolumeExists - cannot create same volume again. +var errVolumeExists = errors.New("Volume already exists.") + +// errIsNotRegular - not a regular file type. +var errIsNotRegular = errors.New("Not a regular file type.") + +// errVolumeNotFound - cannot find the volume. +var errVolumeNotFound = errors.New("Volume not found.") + +// errVolumeNotEmpty - volume not empty. +var errVolumeNotEmpty = errors.New("Volume is not empty.") + +// errVolumeAccessDenied - cannot access volume, insufficient +// permissions. +var errVolumeAccessDenied = errors.New("Volume access denied.") + +// errVolumeAccessDenied - cannot access file, insufficient +// permissions. +var errFileAccessDenied = errors.New("File access denied.") diff --git a/storage-local.go b/storage-local.go deleted file mode 100644 index a92fed617..000000000 --- a/storage-local.go +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "errors" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "syscall" - - "github.com/minio/minio/pkg/disk" - "github.com/minio/minio/pkg/safe" -) - -// ErrDiskPathFull - cannot create volume or files when disk is full. -var ErrDiskPathFull = errors.New("Disk path full.") - -// ErrVolumeExists - cannot create same volume again. -var ErrVolumeExists = errors.New("Volume already exists.") - -// ErrIsNotRegular - is not a regular file type. -var ErrIsNotRegular = errors.New("Not a regular file type.") - -// localStorage implements StorageAPI on top of provided diskPath. -type localStorage struct { - diskPath string - fsInfo disk.Info - minFreeDisk int64 -} - -// Initialize a new local storage. -func newLocalStorage(diskPath string) (StorageAPI, error) { - if diskPath == "" { - return nil, errInvalidArgument - } - st, e := os.Stat(diskPath) - if e != nil { - return nil, e - } - if !st.IsDir() { - return nil, syscall.ENOTDIR - } - - info, e := disk.GetInfo(diskPath) - if e != nil { - return nil, e - } - disk := localStorage{ - diskPath: diskPath, - fsInfo: info, - minFreeDisk: 5, // Minimum 5% disk should be free. - } - return disk, nil -} - -// Make a volume entry. -func (s localStorage) MakeVol(volume string) error { - if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil { - return e - } - volumeDir := getVolumeDir(s.diskPath, volume) - if _, e := os.Stat(volumeDir); e == nil { - return ErrVolumeExists - } - - // Make a volume entry. - if e := os.Mkdir(volumeDir, 0700); e != nil { - return e - } - return nil -} - -// removeDuplicateVols - remove duplicate volumes. -func removeDuplicateVols(vols []VolInfo) []VolInfo { - length := len(vols) - 1 - for i := 0; i < length; i++ { - for j := i + 1; j <= length; j++ { - if vols[i].Name == vols[j].Name { - // Pick the latest volume from a duplicate entry. - if vols[i].Created.Sub(vols[j].Created) > 0 { - vols[i] = vols[length] - } else { - vols[j] = vols[length] - } - vols = vols[0:length] - length-- - j-- - } - } - } - return vols -} - -// ListVols - list volumes. -func (s localStorage) ListVols() ([]VolInfo, error) { - files, e := ioutil.ReadDir(s.diskPath) - if e != nil { - return nil, e - } - var volsInfo []VolInfo - for _, file := range files { - if !file.IsDir() { - // If not directory, ignore all file types. - continue - } - volInfo := VolInfo{ - Name: file.Name(), - Created: file.ModTime(), - } - volsInfo = append(volsInfo, volInfo) - } - // Remove duplicated volume entries. - volsInfo = removeDuplicateVols(volsInfo) - return volsInfo, nil -} - -// getVolumeDir - will convert incoming volume names to -// corresponding valid volume names on the backend in a platform -// compatible way for all operating systems. -func getVolumeDir(diskPath, volume string) string { - volumes, e := ioutil.ReadDir(diskPath) - if e != nil { - return volume - } - for _, vol := range volumes { - // Verify if lowercase version of the volume - // is equal to the incoming volume, then use the proper name. - if strings.ToLower(vol.Name()) == volume { - return filepath.Join(diskPath, vol.Name()) - } - } - return filepath.Join(diskPath, volume) -} - -// StatVol - get volume info. -func (s localStorage) StatVol(volume string) (VolInfo, error) { - volumeDir := getVolumeDir(s.diskPath, volume) - // Stat a volume entry. - st, e := os.Stat(volumeDir) - if e != nil { - return VolInfo{}, e - } - volInfo := VolInfo{} - volInfo.Name = st.Name() - volInfo.Created = st.ModTime() - return volInfo, nil -} - -// DeleteVol - delete a volume. -func (s localStorage) DeleteVol(volume string) error { - return os.Remove(getVolumeDir(s.diskPath, volume)) -} - -/// File operations. - -// ListFiles - list files are prefix and marker. -func (s localStorage) ListFiles(volume, prefix, marker string, recursive bool, count int) (files []FileInfo, isEOF bool, err error) { - // TODO - return files, true, nil -} - -// ReadFile - read a file at a given offset. -func (s localStorage) ReadFile(volume string, path string, offset int64) (io.ReadCloser, error) { - filePath := filepath.Join(getVolumeDir(s.diskPath, volume), path) - file, e := os.Open(filePath) - if e != nil { - return nil, e - } - st, e := file.Stat() - if e != nil { - return nil, e - } - // Verify if its not a regular file, since subsequent Seek is undefined. - if !st.Mode().IsRegular() { - return nil, ErrIsNotRegular - } - _, e = file.Seek(offset, os.SEEK_SET) - if e != nil { - return nil, e - } - return file, nil -} - -// CreateFile - create a file at path. -func (s localStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) { - if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil { - return nil, e - } - filePath := filepath.Join(getVolumeDir(s.diskPath, volume), path) - // Creates a safe file. - return safe.CreateFileWithPrefix(filePath, "$tmpfile") -} - -// StatFile - get file info. -func (s localStorage) StatFile(volume, path string) (file FileInfo, err error) { - filePath := filepath.Join(getVolumeDir(s.diskPath, volume), path) - st, e := os.Stat(filePath) - if e != nil { - return FileInfo{}, e - } - file = FileInfo{ - Volume: volume, - Name: st.Name(), - ModTime: st.ModTime(), - Size: st.Size(), - Type: st.Mode(), - } - return file, nil -} - -// deleteFile - delete file path if its empty. -func deleteFile(basePath, deletePath, volume, path string) error { - if basePath == deletePath { - return nil - } - // Verify if the path exists. - pathSt, e := os.Stat(deletePath) - if e != nil { - return e - } - if pathSt.IsDir() { - // Verify if directory is empty. - empty, e := isDirEmpty(deletePath) - if e != nil { - return e - } - if !empty { - return nil - } - } - // Attempt to remove path. - if e := os.Remove(deletePath); e != nil { - return e - } - // Recursively go down the next path and delete again. - if e := deleteFile(basePath, filepath.Dir(deletePath), volume, path); e != nil { - return e - } - return nil -} - -// DeleteFile - delete a file at path. -func (s localStorage) DeleteFile(volume, path string) error { - volumeDir := getVolumeDir(s.diskPath, volume) - - // Following code is needed so that we retain "/" suffix if any - // in path argument. Do not use filepath.Join() since it would - // strip off any suffixes. - filePath := s.diskPath + string(os.PathSeparator) + volume + string(os.PathSeparator) + path - - // Convert to platform friendly paths. - filePath = filepath.FromSlash(filePath) - - // Delete file and delete parent directory as well if its empty. - return deleteFile(volumeDir, filePath, volume, path) -} diff --git a/storage-rpc-datatypes.go b/storage-rpc-datatypes.go new file mode 100644 index 000000000..e0fc197e5 --- /dev/null +++ b/storage-rpc-datatypes.go @@ -0,0 +1,55 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +// GenericReply generic rpc reply. +type GenericReply struct{} + +// GenericArgs generic rpc args. +type GenericArgs struct{} + +// ListVolsReply list vols rpc reply. +type ListVolsReply struct { + Vols []VolInfo +} + +// ListFilesArgs list file args. +type ListFilesArgs struct { + Vol string + Prefix string + Marker string + Recursive bool + Count int +} + +// ListFilesReply list file reply. +type ListFilesReply struct { + Files []FileInfo + EOF bool +} + +// StatFileArgs stat file args. +type StatFileArgs struct { + Vol string + Path string +} + +// DeleteFileArgs delete file args. +type DeleteFileArgs struct { + Vol string + Path string +} diff --git a/storage-rpc-server.go b/storage-rpc-server.go new file mode 100644 index 000000000..1dce8337a --- /dev/null +++ b/storage-rpc-server.go @@ -0,0 +1,149 @@ +package main + +import ( + "io" + "net/http" + "net/rpc" + "strconv" + + router "github.com/gorilla/mux" +) + +// Storage server implements rpc primitives to facilitate exporting a +// disk over a network. +type storageServer struct { + storage StorageAPI +} + +/// Volume operations handlers + +// MakeVolHandler - make vol handler is rpc wrapper for MakeVol operation. +func (s *storageServer) MakeVolHandler(arg *string, reply *GenericReply) error { + return s.storage.MakeVol(*arg) +} + +// ListVolsHandler - list vols handler is rpc wrapper for ListVols operation. +func (s *storageServer) ListVolsHandler(arg *string, reply *ListVolsReply) error { + vols, err := s.storage.ListVols() + if err != nil { + return err + } + reply.Vols = vols + return nil +} + +// StatVolHandler - stat vol handler is a rpc wrapper for StatVol operation. +func (s *storageServer) StatVolHandler(arg *string, reply *VolInfo) error { + volInfo, err := s.storage.StatVol(*arg) + if err != nil { + return err + } + *reply = volInfo + return nil +} + +// DeleteVolHandler - delete vol handler is a rpc wrapper for +// DeleteVol operation. +func (s *storageServer) DeleteVolHandler(arg *string, reply *GenericReply) error { + return s.storage.DeleteVol(*arg) +} + +/// File operations + +// ListFilesHandler - list files handler. +func (s *storageServer) ListFilesHandler(arg *ListFilesArgs, reply *ListFilesReply) error { + files, eof, err := s.storage.ListFiles(arg.Vol, arg.Prefix, arg.Marker, arg.Recursive, arg.Count) + if err != nil { + return err + } + + // Fill reply structure. + reply.Files = files + reply.EOF = eof + + // Return success. + return nil +} + +// StatFileHandler - stat file handler is rpc wrapper to stat file. +func (s *storageServer) StatFileHandler(arg *StatFileArgs, reply *FileInfo) error { + fileInfo, err := s.storage.StatFile(arg.Vol, arg.Path) + if err != nil { + return err + } + *reply = fileInfo + return nil +} + +// DeleteFileHandler - delete file handler is rpc wrapper to delete file. +func (s *storageServer) DeleteFileHandler(arg *DeleteFileArgs, reply *GenericReply) error { + return s.storage.DeleteFile(arg.Vol, arg.Path) +} + +// registerStorageRPCRouter - register storage rpc router. +func registerStorageRPCRouter(mux *router.Router, storageAPI StorageAPI) { + stServer := &storageServer{ + storage: storageAPI, + } + storageRPCServer := rpc.NewServer() + storageRPCServer.RegisterName("Storage", stServer) + storageRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter() + // Add minio storage routes. + storageRouter.Path("/rpc/storage").Handler(storageRPCServer) + // StreamUpload - stream upload handler. + storageRouter.Methods("POST").Path("/rpc/storage/upload/{volume}/{path:.+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := router.Vars(r) + volume := vars["volume"] + path := vars["path"] + writeCloser, err := stServer.storage.CreateFile(volume, path) + if err != nil { + httpErr := http.StatusInternalServerError + if err == errVolumeNotFound { + httpErr = http.StatusNotFound + } else if err == errIsNotRegular { + httpErr = http.StatusConflict + } + http.Error(w, err.Error(), httpErr) + return + } + reader := r.Body + if _, err = io.Copy(writeCloser, reader); err != nil { + safeCloseAndRemove(writeCloser) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + writeCloser.Close() + reader.Close() + }) + // StreamDownloadHandler - stream download handler. + storageRouter.Methods("GET").Path("/rpc/storage/download/{volume}/{path:.+}").Queries("offset", "{offset:.*}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + vars := router.Vars(r) + volume := vars["volume"] + path := vars["path"] + offset, err := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + readCloser, err := stServer.storage.ReadFile(volume, path, offset) + if err != nil { + httpErr := http.StatusBadRequest + if err == errVolumeNotFound { + httpErr = http.StatusNotFound + } else if err == errFileNotFound { + httpErr = http.StatusNotFound + } + http.Error(w, err.Error(), httpErr) + return + } + + // Copy reader to writer. + io.Copy(w, readCloser) + + // Flush out any remaining buffers to client. + w.(http.Flusher).Flush() + + // Close the reader. + readCloser.Close() + }) +} diff --git a/web-handlers.go b/web-handlers.go index fb75955fd..d8d87ae6d 100644 --- a/web-handlers.go +++ b/web-handlers.go @@ -71,7 +71,7 @@ type ServerInfoRep struct { } // ServerInfo - get server info. -func (web *webAPI) ServerInfo(r *http.Request, args *WebGenericArgs, reply *ServerInfoRep) error { +func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, reply *ServerInfoRep) error { if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } @@ -106,16 +106,17 @@ type DiskInfoRep struct { } // DiskInfo - get disk statistics. -func (web *webAPI) DiskInfo(r *http.Request, args *WebGenericArgs, reply *DiskInfoRep) error { - if !isJWTReqAuthenticated(r) { - return &json2.Error{Message: "Unauthorized request"} - } - info, e := disk.GetInfo(web.ObjectAPI.(*Filesystem).GetRootPath()) - if e != nil { - return &json2.Error{Message: e.Error()} - } - reply.DiskInfo = info - reply.UIVersion = miniobrowser.UIVersion +func (web *webAPIHandlers) DiskInfo(r *http.Request, args *WebGenericArgs, reply *DiskInfoRep) error { + // FIXME: bring in StatFS in StorageAPI interface and uncomment the below lines. + // if !isJWTReqAuthenticated(r) { + // return &json2.Error{Message: "Unauthorized request"} + // } + // info, e := disk.GetInfo(web.ObjectAPI.(*Filesystem).GetRootPath()) + // if e != nil { + // return &json2.Error{Message: e.Error()} + // } + // reply.DiskInfo = info + // reply.UIVersion = miniobrowser.UIVersion return nil } @@ -125,7 +126,7 @@ type MakeBucketArgs struct { } // MakeBucket - make a bucket. -func (web *webAPI) MakeBucket(r *http.Request, args *MakeBucketArgs, reply *WebGenericRep) error { +func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, reply *WebGenericRep) error { if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } @@ -152,7 +153,7 @@ type WebBucketInfo struct { } // ListBuckets - list buckets api. -func (web *webAPI) ListBuckets(r *http.Request, args *WebGenericArgs, reply *ListBucketsRep) error { +func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, reply *ListBucketsRep) error { if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } @@ -198,7 +199,7 @@ type WebObjectInfo struct { } // ListObjects - list objects api. -func (web *webAPI) ListObjects(r *http.Request, args *ListObjectsArgs, reply *ListObjectsRep) error { +func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, reply *ListObjectsRep) error { marker := "" if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} @@ -212,7 +213,7 @@ func (web *webAPI) ListObjects(r *http.Request, args *ListObjectsArgs, reply *Li for _, obj := range lo.Objects { reply.Objects = append(reply.Objects, WebObjectInfo{ Key: obj.Name, - LastModified: obj.ModifiedTime, + LastModified: obj.ModTime, Size: obj.Size, }) } @@ -237,7 +238,7 @@ type RemoveObjectArgs struct { } // RemoveObject - removes an object. -func (web *webAPI) RemoveObject(r *http.Request, args *RemoveObjectArgs, reply *WebGenericRep) error { +func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs, reply *WebGenericRep) error { if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } @@ -262,7 +263,7 @@ type LoginRep struct { } // Login - user login handler. -func (web *webAPI) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error { +func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error { jwt := initJWT() if jwt.Authenticate(args.Username, args.Password) { token, err := jwt.GenerateToken(args.Username) @@ -283,7 +284,7 @@ type GenerateAuthReply struct { UIVersion string `json:"uiVersion"` } -func (web webAPI) GenerateAuth(r *http.Request, args *WebGenericArgs, reply *GenerateAuthReply) error { +func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, reply *GenerateAuthReply) error { if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } @@ -307,7 +308,7 @@ type SetAuthReply struct { } // SetAuth - Set accessKey and secretKey credentials. -func (web *webAPI) SetAuth(r *http.Request, args *SetAuthArgs, reply *SetAuthReply) error { +func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *SetAuthReply) error { if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } @@ -344,7 +345,7 @@ type GetAuthReply struct { } // GetAuth - return accessKey and secretKey credentials. -func (web *webAPI) GetAuth(r *http.Request, args *WebGenericArgs, reply *GetAuthReply) error { +func (web *webAPIHandlers) GetAuth(r *http.Request, args *WebGenericArgs, reply *GetAuthReply) error { if !isJWTReqAuthenticated(r) { return &json2.Error{Message: "Unauthorized request"} } @@ -356,7 +357,7 @@ func (web *webAPI) GetAuth(r *http.Request, args *WebGenericArgs, reply *GetAuth } // Upload - file upload handler. -func (web *webAPI) Upload(w http.ResponseWriter, r *http.Request) { +func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { if !isJWTReqAuthenticated(r) { writeWebErrorResponse(w, errInvalidToken) return @@ -370,7 +371,7 @@ func (web *webAPI) Upload(w http.ResponseWriter, r *http.Request) { } // Download - file download handler. -func (web *webAPI) Download(w http.ResponseWriter, r *http.Request) { +func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] diff --git a/web-router.go b/web-router.go index 33674a8ad..698b6bd82 100644 --- a/web-router.go +++ b/web-router.go @@ -29,8 +29,8 @@ import ( ) // webAPI container for Web API. -type webAPI struct { - ObjectAPI ObjectAPI +type webAPIHandlers struct { + ObjectAPI *objectAPI } // indexHandler - Handler to serve index.html @@ -58,7 +58,7 @@ func assetFS() *assetfs.AssetFS { const specialAssets = "loader.css|logo.svg|firefox.png|safari.png|chrome.png|favicon.ico" // registerWebRouter - registers web router for serving minio browser. -func registerWebRouter(mux *router.Router, web *webAPI) { +func registerWebRouter(mux *router.Router, web *webAPIHandlers) { // Initialize a new json2 codec. codec := json2.NewCodec() @@ -74,7 +74,7 @@ func registerWebRouter(mux *router.Router, web *webAPI) { // RPC handler at URI - /minio/webrpc webBrowserRouter.Methods("POST").Path("/webrpc").Handler(webRPC) webBrowserRouter.Methods("PUT").Path("/upload/{bucket}/{object:.+}").HandlerFunc(web.Upload) - webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "").HandlerFunc(web.Download) + webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "{token:.*}").HandlerFunc(web.Download) // Add compression for assets. compressedAssets := handlers.CompressHandler(http.StripPrefix(reservedBucket, http.FileServer(assetFS())))