MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-11 15:03:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

fs: Break fs package to top-level and introduce ObjectAPI interface.

Browse Source 
ObjectAPI interface brings in changes needed for XL ObjectAPI layer.

The new interface for any ObjectAPI layer is as below

```
// ObjectAPI interface.
type ObjectAPI interface {
        // Bucket resource API.
        DeleteBucket(bucket string) *probe.Error
        ListBuckets() ([]BucketInfo, *probe.Error)
        MakeBucket(bucket string) *probe.Error
        GetBucketInfo(bucket string) (BucketInfo, *probe.Error)

        // Bucket query API.
        ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error)
        ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error)

        // Object resource API.
        GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error)
        GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error)
        PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error)
        DeleteObject(bucket, object string) *probe.Error

        // Object query API.
        NewMultipartUpload(bucket, object string) (string, *probe.Error)
        PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error)
        ListObjectParts(bucket, object string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error)
        CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (ObjectInfo, *probe.Error)
        AbortMultipartUpload(bucket, object, uploadID string) *probe.Error
}
```
This commit is contained in:
Harshavardhana 2016-03-30 16:15:28 -07:00 committed by Harshavardhana
parent 272c5165aa
commit efc80343e3
32 changed files with 613 additions and 581 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api-headers.go
View File

@ -23,8 +23,6 @@ import (
"net/http" "net/http"
"runtime" "runtime"
"strconv" "strconv"
"github.com/minio/minio/pkg/fs"
) )
//// helpers //// helpers
@ -60,7 +58,7 @@ func encodeResponse(response interface{}) []byte {
} }
// Write object header // Write object header
func setObjectHeaders(w http.ResponseWriter, objectInfo fs.ObjectInfo, contentRange *httpRange) { func setObjectHeaders(w http.ResponseWriter, objectInfo ObjectInfo, contentRange *httpRange) {
// set common headers // set common headers
setCommonHeaders(w) setCommonHeaders(w)

11
api-resources.go
View File

@ -19,8 +19,6 @@ package main
import ( import (
"net/url" "net/url"
"strconv" "strconv"
"github.com/minio/minio/pkg/fs"
) )
// parse bucket url queries // parse bucket url queries
@ -34,7 +32,7 @@ func getBucketResources(values url.Values) (prefix, marker, delimiter string, ma
} }
// part bucket url queries for ?uploads // part bucket url queries for ?uploads
func getBucketMultipartResources(values url.Values) (v fs.BucketMultipartResourcesMetadata) { func getBucketMultipartResources(values url.Values) (v BucketMultipartResourcesMetadata) {
v.Prefix = values.Get("prefix") v.Prefix = values.Get("prefix")
v.KeyMarker = values.Get("key-marker") v.KeyMarker = values.Get("key-marker")
v.MaxUploads, _ = strconv.Atoi(values.Get("max-uploads")) v.MaxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
@ -45,10 +43,15 @@ func getBucketMultipartResources(values url.Values) (v fs.BucketMultipartResourc
} }
// parse object url queries // parse object url queries
func getObjectResources(values url.Values) (v fs.ObjectResourcesMetadata) { func getObjectResources(values url.Values) (v ObjectResourcesMetadata) {
v.UploadID = values.Get("uploadId") v.UploadID = values.Get("uploadId")
v.PartNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker")) v.PartNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
v.MaxParts, _ = strconv.Atoi(values.Get("max-parts")) v.MaxParts, _ = strconv.Atoi(values.Get("max-parts"))
v.EncodingType = values.Get("encoding-type") v.EncodingType = values.Get("encoding-type")
return return
} }
// get upload id.
func getUploadID(values url.Values) (uploadID string) {
return getObjectResources(values).UploadID
}

10
api-response.go
View File

@ -20,8 +20,6 @@ import (
"encoding/xml" "encoding/xml"
"net/http" "net/http"
"time" "time"
"github.com/minio/minio/pkg/fs"
) )
const ( const (
@ -225,7 +223,7 @@ func getLocation(r *http.Request) string {
// //
// output: // output:
// populated struct that can be serialized to match xml and json api spec output // populated struct that can be serialized to match xml and json api spec output
func generateListBucketsResponse(buckets []fs.BucketInfo) ListBucketsResponse { func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
var listbuckets []Bucket var listbuckets []Bucket
var data = ListBucketsResponse{} var data = ListBucketsResponse{}
var owner = Owner{} var owner = Owner{}
@ -247,7 +245,7 @@ func generateListBucketsResponse(buckets []fs.BucketInfo) ListBucketsResponse {
} }
// generates an ListObjects response for the said bucket with other enumerated options. // generates an ListObjects response for the said bucket with other enumerated options.
func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp fs.ListObjectsResult) ListObjectsResponse { func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsResult) ListObjectsResponse {
var contents []Object var contents []Object
var prefixes []CommonPrefix var prefixes []CommonPrefix
var owner = Owner{} var owner = Owner{}
@ -319,7 +317,7 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string)
} }
// generateListPartsResult // generateListPartsResult
func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPartsResponse { func generateListPartsResponse(objectMetadata ObjectResourcesMetadata) ListPartsResponse {
// TODO - support EncodingType in xml decoding // TODO - support EncodingType in xml decoding
listPartsResponse := ListPartsResponse{} listPartsResponse := ListPartsResponse{}
listPartsResponse.Bucket = objectMetadata.Bucket listPartsResponse.Bucket = objectMetadata.Bucket
@ -349,7 +347,7 @@ func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPa
} }
// generateListMultipartUploadsResponse // generateListMultipartUploadsResponse
func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse { func generateListMultipartUploadsResponse(bucket string, metadata BucketMultipartResourcesMetadata) ListMultipartUploadsResponse {
listMultipartUploadsResponse := ListMultipartUploadsResponse{} listMultipartUploadsResponse := ListMultipartUploadsResponse{}
listMultipartUploadsResponse.Bucket = bucket listMultipartUploadsResponse.Bucket = bucket
listMultipartUploadsResponse.Delimiter = metadata.Delimiter listMultipartUploadsResponse.Delimiter = metadata.Delimiter

14
api-router.go
View File

@ -16,19 +16,15 @@
package main package main
import ( import router "github.com/gorilla/mux"
router "github.com/gorilla/mux"
"github.com/minio/minio/pkg/fs"
)
// storageAPI container for S3 compatible API. // objectStorageAPI container for S3 compatible API.
type storageAPI struct { type objectStorageAPI struct {
// Filesystem instance. ObjectAPI ObjectAPI
Filesystem fs.Filesystem
} }
// registerAPIRouter - registers S3 compatible APIs. // registerAPIRouter - registers S3 compatible APIs.
func registerAPIRouter(mux *router.Router, api storageAPI) { func registerAPIRouter(mux *router.Router, api objectStorageAPI) {
// API Router // API Router
apiRouter := mux.NewRoute().PathPrefix("/").Subrouter() apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()

87
bucket-handlers.go
View File

@ -27,7 +27,6 @@ import (
"strings" "strings"
mux "github.com/gorilla/mux" mux "github.com/gorilla/mux"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/probe"
) )
@ -38,9 +37,9 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error
if err != nil { if err != nil {
errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil) errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNotFound: case BucketNotFound:
return ErrNoSuchBucket return ErrNoSuchBucket
case fs.BucketNameInvalid: case BucketNameInvalid:
return ErrInvalidBucketName return ErrInvalidBucketName
default: default:
// For any other error just return AccessDenied. // For any other error just return AccessDenied.
@ -73,7 +72,7 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error
// GetBucketLocationHandler - GET Bucket location. // GetBucketLocationHandler - GET Bucket location.
// ------------------------- // -------------------------
// This operation returns bucket location. // This operation returns bucket location.
func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -95,13 +94,13 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re
} }
} }
_, err := api.Filesystem.GetBucketInfo(bucket) _, err := api.ObjectAPI.GetBucketInfo(bucket)
if err != nil { if err != nil {
errorIf(err.Trace(), "GetBucketInfo failed.", nil) errorIf(err.Trace(), "GetBucketInfo failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -130,7 +129,7 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re
// completed or aborted. This operation returns at most 1,000 multipart // completed or aborted. This operation returns at most 1,000 multipart
// uploads in the response. // uploads in the response.
// //
func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -161,11 +160,11 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http
resources.MaxUploads = maxObjectList resources.MaxUploads = maxObjectList
} }
resources, err := api.Filesystem.ListMultipartUploads(bucket, resources) resources, err := api.ObjectAPI.ListMultipartUploads(bucket, resources)
if err != nil { if err != nil {
errorIf(err.Trace(), "ListMultipartUploads failed.", nil) errorIf(err.Trace(), "ListMultipartUploads failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -187,7 +186,7 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http
// of the objects in a bucket. You can use the request parameters as selection // of the objects in a bucket. You can use the request parameters as selection
// criteria to return a subset of the objects in a bucket. // criteria to return a subset of the objects in a bucket.
// //
func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -219,7 +218,7 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request)
maxkeys = maxObjectList maxkeys = maxObjectList
} }
listResp, err := api.Filesystem.ListObjects(bucket, prefix, marker, delimiter, maxkeys) listResp, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
if err == nil { if err == nil {
// generate response // generate response
response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listResp) response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listResp)
@ -231,13 +230,13 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request)
return return
} }
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.ObjectNotFound: case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.ObjectNameInvalid: case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
default: default:
errorIf(err.Trace(), "ListObjects failed.", nil) errorIf(err.Trace(), "ListObjects failed.", nil)
@ -249,7 +248,7 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request)
// ----------- // -----------
// This implementation of the GET operation returns a list of all buckets // This implementation of the GET operation returns a list of all buckets
// owned by the authenticated sender of the request. // owned by the authenticated sender of the request.
func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
// List buckets does not support bucket policies. // List buckets does not support bucket policies.
switch getRequestAuthType(r) { switch getRequestAuthType(r) {
default: default:
@ -263,7 +262,7 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request)
} }
} }
buckets, err := api.Filesystem.ListBuckets() buckets, err := api.ObjectAPI.ListBuckets()
if err == nil { if err == nil {
// generate response // generate response
response := generateListBucketsResponse(buckets) response := generateListBucketsResponse(buckets)
@ -279,7 +278,7 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request)
} }
// DeleteMultipleObjectsHandler - deletes multiple objects. // DeleteMultipleObjectsHandler - deletes multiple objects.
func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -337,7 +336,7 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
var deletedObjects []ObjectIdentifier var deletedObjects []ObjectIdentifier
// Loop through all the objects and delete them sequentially. // Loop through all the objects and delete them sequentially.
for _, object := range deleteObjects.Objects { for _, object := range deleteObjects.Objects {
err := api.Filesystem.DeleteObject(bucket, object.ObjectName) err := api.ObjectAPI.DeleteObject(bucket, object.ObjectName)
if err == nil { if err == nil {
deletedObjects = append(deletedObjects, ObjectIdentifier{ deletedObjects = append(deletedObjects, ObjectIdentifier{
ObjectName: object.ObjectName, ObjectName: object.ObjectName,
@ -345,25 +344,25 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
} else { } else {
errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil) errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
deleteErrors = append(deleteErrors, DeleteError{ deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[ErrInvalidBucketName].Code, Code: errorCodeResponse[ErrInvalidBucketName].Code,
Message: errorCodeResponse[ErrInvalidBucketName].Description, Message: errorCodeResponse[ErrInvalidBucketName].Description,
Key: object.ObjectName, Key: object.ObjectName,
}) })
case fs.BucketNotFound: case BucketNotFound:
deleteErrors = append(deleteErrors, DeleteError{ deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[ErrNoSuchBucket].Code, Code: errorCodeResponse[ErrNoSuchBucket].Code,
Message: errorCodeResponse[ErrNoSuchBucket].Description, Message: errorCodeResponse[ErrNoSuchBucket].Description,
Key: object.ObjectName, Key: object.ObjectName,
}) })
case fs.ObjectNotFound: case ObjectNotFound:
deleteErrors = append(deleteErrors, DeleteError{ deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[ErrNoSuchKey].Code, Code: errorCodeResponse[ErrNoSuchKey].Code,
Message: errorCodeResponse[ErrNoSuchKey].Description, Message: errorCodeResponse[ErrNoSuchKey].Description,
Key: object.ObjectName, Key: object.ObjectName,
}) })
case fs.ObjectNameInvalid: case ObjectNameInvalid:
deleteErrors = append(deleteErrors, DeleteError{ deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[ErrNoSuchKey].Code, Code: errorCodeResponse[ErrNoSuchKey].Code,
Message: errorCodeResponse[ErrNoSuchKey].Description, Message: errorCodeResponse[ErrNoSuchKey].Description,
@ -390,7 +389,7 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
// PutBucketHandler - PUT Bucket // PutBucketHandler - PUT Bucket
// ---------- // ----------
// This implementation of the PUT operation creates a new bucket for authenticated request // This implementation of the PUT operation creates a new bucket for authenticated request
func (api storageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -408,13 +407,13 @@ func (api storageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
} }
// Make bucket. // Make bucket.
err := api.Filesystem.MakeBucket(bucket) err := api.ObjectAPI.MakeBucket(bucket)
if err != nil { if err != nil {
errorIf(err.Trace(), "MakeBucket failed.", nil) errorIf(err.Trace(), "MakeBucket failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketExists: case BucketExists:
writeErrorResponse(w, r, ErrBucketAlreadyExists, r.URL.Path) writeErrorResponse(w, r, ErrBucketAlreadyExists, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -455,7 +454,7 @@ func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]stri
// ---------- // ----------
// This implementation of the POST operation handles object creation with a specified // This implementation of the POST operation handles object creation with a specified
// signature policy in multipart/form-data // signature policy in multipart/form-data
func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
// Here the parameter is the size of the form data that should // Here the parameter is the size of the form data that should
// be loaded in memory, the remaining being put in temporary files. // be loaded in memory, the remaining being put in temporary files.
reader, e := r.MultipartReader() reader, e := r.MultipartReader()
@ -485,19 +484,19 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req
writeErrorResponse(w, r, apiErr, r.URL.Path) writeErrorResponse(w, r, apiErr, r.URL.Path)
return return
} }
objectInfo, err := api.Filesystem.CreateObject(bucket, object, -1, fileBody, nil) objectInfo, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, nil)
if err != nil { if err != nil {
errorIf(err.Trace(), "CreateObject failed.", nil) errorIf(err.Trace(), "PutObject failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.RootPathFull: case RootPathFull:
writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BadDigest: case BadDigest:
writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
case fs.IncompleteBody: case IncompleteBody:
writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -516,7 +515,7 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req
// The operation returns a 200 OK if the bucket exists and you // The operation returns a 200 OK if the bucket exists and you
// have permission to access it. Otherwise, the operation might // have permission to access it. Otherwise, the operation might
// return responses such as 404 Not Found and 403 Forbidden. // return responses such as 404 Not Found and 403 Forbidden.
func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -532,13 +531,13 @@ func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request)
} }
} }
_, err := api.Filesystem.GetBucketInfo(bucket) _, err := api.ObjectAPI.GetBucketInfo(bucket)
if err != nil { if err != nil {
errorIf(err.Trace(), "GetBucketInfo failed.", nil) errorIf(err.Trace(), "GetBucketInfo failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -549,7 +548,7 @@ func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request)
} }
// DeleteBucketHandler - Delete bucket // DeleteBucketHandler - Delete bucket
func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -565,13 +564,13 @@ func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request
} }
} }
err := api.Filesystem.DeleteBucket(bucket) err := api.ObjectAPI.DeleteBucket(bucket)
if err != nil { if err != nil {
errorIf(err.Trace(), "DeleteBucket failed.", nil) errorIf(err.Trace(), "DeleteBucket failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.BucketNotEmpty: case BucketNotEmpty:
writeErrorResponse(w, r, ErrBucketNotEmpty, r.URL.Path) writeErrorResponse(w, r, ErrBucketNotEmpty, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)

17
bucket-policy-handlers.go
View File

@ -25,7 +25,6 @@ import (
"strings" "strings"
mux "github.com/gorilla/mux" mux "github.com/gorilla/mux"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/probe"
) )
@ -128,7 +127,7 @@ func bucketPolicyConditionMatch(conditions map[string]string, statement policySt
// ----------------- // -----------------
// This implementation of the PUT operation uses the policy // This implementation of the PUT operation uses the policy
// subresource to add to or replace a policy on a bucket // subresource to add to or replace a policy on a bucket
func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -188,7 +187,7 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ
if err != nil { if err != nil {
errorIf(err.Trace(bucket, string(bucketPolicyBuf)), "SaveBucketPolicy failed.", nil) errorIf(err.Trace(bucket, string(bucketPolicyBuf)), "SaveBucketPolicy failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -202,7 +201,7 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ
// ----------------- // -----------------
// This implementation of the DELETE operation uses the policy // This implementation of the DELETE operation uses the policy
// subresource to add to remove a policy on a bucket. // subresource to add to remove a policy on a bucket.
func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -223,9 +222,9 @@ func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.R
if err != nil { if err != nil {
errorIf(err.Trace(bucket), "DeleteBucketPolicy failed.", nil) errorIf(err.Trace(bucket), "DeleteBucketPolicy failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketPolicyNotFound: case BucketPolicyNotFound:
writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -239,7 +238,7 @@ func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.R
// ----------------- // -----------------
// This operation uses the policy // This operation uses the policy
// subresource to return the policy of a specified bucket. // subresource to return the policy of a specified bucket.
func (api storageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
@ -260,9 +259,9 @@ func (api storageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Requ
if err != nil { if err != nil {
errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil) errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketPolicyNotFound: case BucketPolicyNotFound:
writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)

17
bucket-policy.go
View File

@ -21,7 +21,6 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/probe"
) )
@ -70,8 +69,8 @@ func createBucketConfigPath(bucket string) *probe.Error {
// readBucketPolicy - read bucket policy. // readBucketPolicy - read bucket policy.
func readBucketPolicy(bucket string) ([]byte, *probe.Error) { func readBucketPolicy(bucket string) ([]byte, *probe.Error) {
// Verify bucket is valid. // Verify bucket is valid.
if !fs.IsValidBucketName(bucket) { if !IsValidBucketName(bucket) {
return nil, probe.NewError(fs.BucketNameInvalid{Bucket: bucket}) return nil, probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
bucketConfigPath, err := getBucketConfigPath(bucket) bucketConfigPath, err := getBucketConfigPath(bucket)
@ -83,7 +82,7 @@ func readBucketPolicy(bucket string) ([]byte, *probe.Error) {
bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
if _, e := os.Stat(bucketPolicyFile); e != nil { if _, e := os.Stat(bucketPolicyFile); e != nil {
if os.IsNotExist(e) { if os.IsNotExist(e) {
return nil, probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket}) return nil, probe.NewError(BucketPolicyNotFound{Bucket: bucket})
} }
return nil, probe.NewError(e) return nil, probe.NewError(e)
} }
@ -98,8 +97,8 @@ func readBucketPolicy(bucket string) ([]byte, *probe.Error) {
// removeBucketPolicy - remove bucket policy. // removeBucketPolicy - remove bucket policy.
func removeBucketPolicy(bucket string) *probe.Error { func removeBucketPolicy(bucket string) *probe.Error {
// Verify bucket is valid. // Verify bucket is valid.
if !fs.IsValidBucketName(bucket) { if !IsValidBucketName(bucket) {
return probe.NewError(fs.BucketNameInvalid{Bucket: bucket}) return probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
bucketConfigPath, err := getBucketConfigPath(bucket) bucketConfigPath, err := getBucketConfigPath(bucket)
@ -111,7 +110,7 @@ func removeBucketPolicy(bucket string) *probe.Error {
bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json") bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
if _, e := os.Stat(bucketPolicyFile); e != nil { if _, e := os.Stat(bucketPolicyFile); e != nil {
if os.IsNotExist(e) { if os.IsNotExist(e) {
return probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket}) return probe.NewError(BucketPolicyNotFound{Bucket: bucket})
} }
return probe.NewError(e) return probe.NewError(e)
} }
@ -121,8 +120,8 @@ func removeBucketPolicy(bucket string) *probe.Error {
// writeBucketPolicy - save bucket policy. // writeBucketPolicy - save bucket policy.
func writeBucketPolicy(bucket string, accessPolicyBytes []byte) *probe.Error { func writeBucketPolicy(bucket string, accessPolicyBytes []byte) *probe.Error {
// Verify if bucket path legal // Verify if bucket path legal
if !fs.IsValidBucketName(bucket) { if !IsValidBucketName(bucket) {
return probe.NewError(fs.BucketNameInvalid{Bucket: bucket}) return probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
// Create bucket config path. // Create bucket config path.

2
pkg/fs/fs-backend-metadata.go → fs-backend-metadata.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/probe"

4
pkg/fs/fs-bucket-listobjects.go → fs-bucket-listobjects.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"fmt" "fmt"
@ -38,7 +38,7 @@ func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKe
return result, probe.NewError(BucketNameInvalid{Bucket: bucket}) return result, probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket) // Get the right bucket name.
bucketDir := filepath.Join(fs.path, bucket) bucketDir := filepath.Join(fs.path, bucket)
// Verify if bucket exists. // Verify if bucket exists.
if status, err := isDirExist(bucketDir); !status { if status, err := isDirExist(bucketDir); !status {

32
pkg/fs/fs-bucket-listobjects_test.go → fs-bucket-listobjects_test.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"bytes" "bytes"
@ -27,15 +27,15 @@ import (
) )
func TestListObjects(t *testing.T) { func TestListObjects(t *testing.T) {
// Make a temporary directory to use as the filesystem. // Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-list-object-test") directory, e := ioutil.TempDir("", "minio-list-object-test")
if e != nil { if e != nil {
t.Fatal(e) t.Fatal(e)
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
fs, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -57,36 +57,36 @@ func TestListObjects(t *testing.T) {
} }
defer os.Remove(tmpfile.Name()) // clean up defer os.Remove(tmpfile.Name()) // clean up
_, err = fs.CreateObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil) _, err = fs.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil)
if err != nil { if err != nil {
t.Fatal(e) t.Fatal(e)
} }
_, err = fs.CreateObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) _, err = fs.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
if err != nil { if err != nil {
t.Fatal(e) t.Fatal(e)
} }
_, err = fs.CreateObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil) _, err = fs.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
if err != nil { if err != nil {
t.Fatal(e) t.Fatal(e)
} }
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
key := "newPrefix" + strconv.Itoa(i) key := "newPrefix" + strconv.Itoa(i)
_, err = fs.CreateObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) _, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
_, err = fs.CreateObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil) _, err = fs.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil)
if err != nil { if err != nil {
t.Fatal(e) t.Fatal(e)
} }
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
key := "obj" + strconv.Itoa(i) key := "obj" + strconv.Itoa(i)
_, err = fs.CreateObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil) _, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -570,28 +570,28 @@ func TestListObjects(t *testing.T) {
} }
func BenchmarkListObjects(b *testing.B) { func BenchmarkListObjects(b *testing.B) {
// Make a temporary directory to use as the filesystem. // Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-list-benchmark") directory, e := ioutil.TempDir("", "minio-list-benchmark")
if e != nil { if e != nil {
b.Fatal(e) b.Fatal(e)
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
filesystem, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
// Create a bucket. // Create a bucket.
err = filesystem.MakeBucket("ls-benchmark-bucket") err = fs.MakeBucket("ls-benchmark-bucket")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
for i := 0; i < 20000; i++ { for i := 0; i < 20000; i++ {
key := "obj" + strconv.Itoa(i) key := "obj" + strconv.Itoa(i)
_, err = filesystem.CreateObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) _, err = fs.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -601,7 +601,7 @@ func BenchmarkListObjects(b *testing.B) {
// List the buckets over and over and over. // List the buckets over and over and over.
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, err = filesystem.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1) _, err = fs.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }

28
pkg/fs/fs-bucket.go → fs-bucket.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"io/ioutil" "io/ioutil"
@ -35,7 +35,7 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
if !IsValidBucketName(bucket) { if !IsValidBucketName(bucket) {
return probe.NewError(BucketNameInvalid{Bucket: bucket}) return probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketDir := filepath.Join(fs.path, bucket) bucketDir := filepath.Join(fs.path, bucket)
if e := os.Remove(bucketDir); e != nil { if e := os.Remove(bucketDir); e != nil {
// Error if there was no bucket in the first place. // Error if there was no bucket in the first place.
@ -129,7 +129,7 @@ func (fs Filesystem) MakeBucket(bucket string) *probe.Error {
return probe.NewError(BucketNameInvalid{Bucket: bucket}) return probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketDir := filepath.Join(fs.path, bucket) bucketDir := filepath.Join(fs.path, bucket)
if _, e := os.Stat(bucketDir); e == nil { if _, e := os.Stat(bucketDir); e == nil {
return probe.NewError(BucketExists{Bucket: bucket}) return probe.NewError(BucketExists{Bucket: bucket})
@ -142,19 +142,23 @@ func (fs Filesystem) MakeBucket(bucket string) *probe.Error {
return nil return nil
} }
// denormalizeBucket - will convert incoming bucket names to // getActualBucketname - will convert incoming bucket names to
// corresponding valid bucketnames on the backend in a platform // corresponding actual bucketnames on the backend in a platform
// compatible way for all operating systems. // compatible way for all operating systems.
func (fs Filesystem) denormalizeBucket(bucket string) string { func getActualBucketname(fsPath, bucket string) string {
buckets, e := ioutil.ReadDir(fs.path) fd, e := os.Open(fsPath)
if e != nil {
return bucket
}
buckets, e := fd.Readdirnames(-1)
if e != nil { if e != nil {
return bucket return bucket
} }
for _, b := range buckets { for _, b := range buckets {
// Verify if lowercase version of the bucket is equal to the // Verify if lowercase version of the bucket is equal
// incoming bucket, then use the proper name. // to the incoming bucket, then use the proper name.
if strings.ToLower(b.Name()) == bucket { if strings.ToLower(b) == bucket {
return b.Name() return b
} }
} }
return bucket return bucket
@ -165,7 +169,7 @@ func (fs Filesystem) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) {
if !IsValidBucketName(bucket) { if !IsValidBucketName(bucket) {
return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
// Get bucket path. // Get bucket path.
bucketDir := filepath.Join(fs.path, bucket) bucketDir := filepath.Join(fs.path, bucket)
fi, e := os.Stat(bucketDir) fi, e := os.Stat(bucketDir)

60
pkg/fs/fs-bucket_test.go → fs-bucket_test.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"io/ioutil" "io/ioutil"
@ -28,22 +28,22 @@ import (
// But also includes test cases for which the function should fail. // But also includes test cases for which the function should fail.
// For those cases for which it fails, its also asserted whether the function fails as expected. // For those cases for which it fails, its also asserted whether the function fails as expected.
func TestGetBucketInfo(t *testing.T) { func TestGetBucketInfo(t *testing.T) {
// Make a temporary directory to use as the filesystem. // Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-metadata-test") directory, e := ioutil.TempDir("", "minio-metadata-test")
if e != nil { if e != nil {
t.Fatal(e) t.Fatal(e)
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
filesystem, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Creating few buckets. // Creating few buckets.
for i := 0; i < 4; i++ { for i := 0; i < 4; i++ {
err = filesystem.MakeBucket("meta-test-bucket." + strconv.Itoa(i)) err = fs.MakeBucket("meta-test-bucket." + strconv.Itoa(i))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -70,7 +70,7 @@ func TestGetBucketInfo(t *testing.T) {
} }
for i, testCase := range testCases { for i, testCase := range testCases {
// The err returned is of type *probe.Error. // The err returned is of type *probe.Error.
bucketInfo, err := filesystem.GetBucketInfo(testCase.bucketName) bucketInfo, err := fs.GetBucketInfo(testCase.bucketName)
if err != nil && testCase.shouldPass { if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error()) t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
@ -96,29 +96,29 @@ func TestGetBucketInfo(t *testing.T) {
} }
func TestListBuckets(t *testing.T) { func TestListBuckets(t *testing.T) {
// Make a temporary directory to use as the filesystem. // Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark") directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil { if e != nil {
t.Fatal(e) t.Fatal(e)
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
filesystem, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Create a few buckets. // Create a few buckets.
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
err = filesystem.MakeBucket("testbucket." + strconv.Itoa(i)) err = fs.MakeBucket("testbucket." + strconv.Itoa(i))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
// List, and ensure that they are all there. // List, and ensure that they are all there.
metadatas, err := filesystem.ListBuckets() metadatas, err := fs.ListBuckets()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -136,43 +136,43 @@ func TestListBuckets(t *testing.T) {
} }
func TestDeleteBucket(t *testing.T) { func TestDeleteBucket(t *testing.T) {
// Make a temporary directory to use as the filesystem. // Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark") directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil { if e != nil {
t.Fatal(e) t.Fatal(e)
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
filesystem, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Deleting a bucket that doesn't exist should error. // Deleting a bucket that doesn't exist should error.
err = filesystem.DeleteBucket("bucket") err = fs.DeleteBucket("bucket")
if !strings.Contains(err.Cause.Error(), "Bucket not found:") { if !strings.Contains(err.Cause.Error(), "Bucket not found:") {
t.Fail() t.Fail()
} }
} }
func BenchmarkListBuckets(b *testing.B) { func BenchmarkListBuckets(b *testing.B) {
// Make a temporary directory to use as the filesystem. // Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark") directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil { if e != nil {
b.Fatal(e) b.Fatal(e)
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
filesystem, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
// Create a few buckets. // Create a few buckets.
for i := 0; i < 20; i++ { for i := 0; i < 20; i++ {
err = filesystem.MakeBucket("bucket." + strconv.Itoa(i)) err = fs.MakeBucket("bucket." + strconv.Itoa(i))
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -182,7 +182,7 @@ func BenchmarkListBuckets(b *testing.B) {
// List the buckets over and over and over. // List the buckets over and over and over.
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, err = filesystem.ListBuckets() _, err = fs.ListBuckets()
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -190,15 +190,15 @@ func BenchmarkListBuckets(b *testing.B) {
} }
func BenchmarkDeleteBucket(b *testing.B) { func BenchmarkDeleteBucket(b *testing.B) {
// Make a temporary directory to use as the filesystem. // Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark") directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil { if e != nil {
b.Fatal(e) b.Fatal(e)
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
filesystem, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -210,14 +210,14 @@ func BenchmarkDeleteBucket(b *testing.B) {
b.StopTimer() b.StopTimer()
// Create and delete the bucket over and over. // Create and delete the bucket over and over.
err = filesystem.MakeBucket("bucket") err = fs.MakeBucket("bucket")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
b.StartTimer() b.StartTimer()
err = filesystem.DeleteBucket("bucket") err = fs.DeleteBucket("bucket")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -225,21 +225,21 @@ func BenchmarkDeleteBucket(b *testing.B) {
} }
func BenchmarkGetBucketInfo(b *testing.B) { func BenchmarkGetBucketInfo(b *testing.B) {
// Make a temporary directory to use as the filesystem. // Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark") directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil { if e != nil {
b.Fatal(e) b.Fatal(e)
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
filesystem, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
// Put up a bucket with some metadata. // Put up a bucket with some metadata.
err = filesystem.MakeBucket("bucket") err = fs.MakeBucket("bucket")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -248,7 +248,7 @@ func BenchmarkGetBucketInfo(b *testing.B) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
// Retrieve the metadata! // Retrieve the metadata!
_, err := filesystem.GetBucketInfo("bucket") _, err := fs.GetBucketInfo("bucket")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }

4
pkg/fs/fs-datatypes.go → fs-datatypes.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import "time" import "time"
@ -87,5 +87,5 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part
// CompleteMultipartUpload container for completing multipart upload // CompleteMultipartUpload container for completing multipart upload
type CompleteMultipartUpload struct { type CompleteMultipartUpload struct {
Part []CompletePart Parts []CompletePart `xml:"Part"`
} }

2
pkg/fs/dir.go → fs-dir.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"io" "io"

2
pkg/fs/fs-errors.go → fs-errors.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import "fmt" import "fmt"

42
pkg/fs/fs-multipart.go → fs-multipart.go
View File

@ -14,13 +14,12 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"crypto/md5" "crypto/md5"
"encoding/base64" "encoding/base64"
"encoding/hex" "encoding/hex"
"encoding/xml"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -63,7 +62,7 @@ func (fs Filesystem) ListMultipartUploads(bucket string, resources BucketMultipa
if !IsValidBucketName(bucket) { if !IsValidBucketName(bucket) {
return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketPath := filepath.Join(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket)
if _, e := os.Stat(bucketPath); e != nil { if _, e := os.Stat(bucketPath); e != nil {
// Check bucket exists. // Check bucket exists.
@ -244,7 +243,7 @@ func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.E
return "", probe.NewError(ObjectNameInvalid{Object: object}) return "", probe.NewError(ObjectNameInvalid{Object: object})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketPath := filepath.Join(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket)
if _, e = os.Stat(bucketPath); e != nil { if _, e = os.Stat(bucketPath); e != nil {
// Check bucket exists. // Check bucket exists.
@ -318,8 +317,8 @@ func (a partNumber) Len() int { return len(a) }
func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber } func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
// CreateObjectPart - create a part in a multipart session // PutObjectPart - create a part in a multipart session
func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Bytes []byte) (string, *probe.Error) { func (fs Filesystem) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) {
di, err := disk.GetInfo(fs.path) di, err := disk.GetInfo(fs.path)
if err != nil { if err != nil {
return "", probe.NewError(err) return "", probe.NewError(err)
@ -352,7 +351,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID in
return "", probe.NewError(InvalidUploadID{UploadID: uploadID}) return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketPath := filepath.Join(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket)
if _, e := os.Stat(bucketPath); e != nil { if _, e := os.Stat(bucketPath); e != nil {
// Check bucket exists. // Check bucket exists.
@ -362,12 +361,6 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID in
return "", probe.NewError(e) return "", probe.NewError(e)
} }
// md5Hex representation.
var md5Hex string
if len(md5Bytes) != 0 {
md5Hex = hex.EncodeToString(md5Bytes)
}
objectPath := filepath.Join(bucketPath, object) objectPath := filepath.Join(bucketPath, object)
partPathPrefix := objectPath + uploadID partPathPrefix := objectPath + uploadID
partPath := partPathPrefix + md5Hex + fmt.Sprintf("$%d-$multiparts", partID) partPath := partPathPrefix + md5Hex + fmt.Sprintf("$%d-$multiparts", partID)
@ -390,7 +383,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID in
// Finalize new md5. // Finalize new md5.
newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
if len(md5Bytes) != 0 { if md5Hex != "" {
if newMD5Hex != md5Hex { if newMD5Hex != md5Hex {
return "", probe.NewError(BadDigest{md5Hex, newMD5Hex}) return "", probe.NewError(BadDigest{md5Hex, newMD5Hex})
} }
@ -438,7 +431,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID in
} }
// CompleteMultipartUpload - complete a multipart upload and persist the data // CompleteMultipartUpload - complete a multipart upload and persist the data
func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploadID string, completeMultipartBytes []byte) (ObjectInfo, *probe.Error) { func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (ObjectInfo, *probe.Error) {
// Check bucket name is valid. // Check bucket name is valid.
if !IsValidBucketName(bucket) { if !IsValidBucketName(bucket) {
return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
@ -454,7 +447,7 @@ func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploa
return ObjectInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID}) return ObjectInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketPath := filepath.Join(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket)
if _, e := os.Stat(bucketPath); e != nil { if _, e := os.Stat(bucketPath); e != nil {
// Check bucket exists. // Check bucket exists.
@ -470,19 +463,6 @@ func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploa
return ObjectInfo{}, probe.NewError(e) return ObjectInfo{}, probe.NewError(e)
} }
completeMultipartUpload := &CompleteMultipartUpload{}
if e = xml.Unmarshal(completeMultipartBytes, completeMultipartUpload); e != nil {
objectWriter.CloseAndPurge()
return ObjectInfo{}, probe.NewError(MalformedXML{})
}
if !sort.IsSorted(completedParts(completeMultipartUpload.Part)) {
objectWriter.CloseAndPurge()
return ObjectInfo{}, probe.NewError(InvalidPartOrder{})
}
// Save parts for verification.
parts := completeMultipartUpload.Part
// Critical region requiring read lock. // Critical region requiring read lock.
fs.rwLock.RLock() fs.rwLock.RLock()
savedParts := fs.multiparts.ActiveSession[uploadID].Parts savedParts := fs.multiparts.ActiveSession[uploadID].Parts
@ -582,7 +562,7 @@ func (fs Filesystem) ListObjectParts(bucket, object string, resources ObjectReso
startPartNumber = objectResourcesMetadata.PartNumberMarker startPartNumber = objectResourcesMetadata.PartNumberMarker
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketPath := filepath.Join(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket)
if _, e := os.Stat(bucketPath); e != nil { if _, e := os.Stat(bucketPath); e != nil {
// Check bucket exists. // Check bucket exists.
@ -631,7 +611,7 @@ func (fs Filesystem) AbortMultipartUpload(bucket, object, uploadID string) *prob
return probe.NewError(InvalidUploadID{UploadID: uploadID}) return probe.NewError(InvalidUploadID{UploadID: uploadID})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketPath := filepath.Join(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket)
if _, e := os.Stat(bucketPath); e != nil { if _, e := os.Stat(bucketPath); e != nil {
// Check bucket exists. // Check bucket exists.

76
pkg/fs/fs-object.go → fs-object.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"bytes" "bytes"
@ -36,21 +36,17 @@ import (
/// Object Operations /// Object Operations
// GetObject - GET object // GetObject - GET object
func (fs Filesystem) GetObject(w io.Writer, bucket, object string, start, length int64) (int64, *probe.Error) { func (fs Filesystem) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) {
// Critical region requiring read lock.
fs.rwLock.RLock()
defer fs.rwLock.RUnlock()
// Input validation. // Input validation.
if !IsValidBucketName(bucket) { if !IsValidBucketName(bucket) {
return 0, probe.NewError(BucketNameInvalid{Bucket: bucket}) return nil, probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
if !IsValidObjectName(object) { if !IsValidObjectName(object) {
return 0, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object}) return nil, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
} }
// normalize buckets. // normalize buckets.
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
objectPath := filepath.Join(fs.path, bucket, object) objectPath := filepath.Join(fs.path, bucket, object)
file, e := os.Open(objectPath) file, e := os.Open(objectPath)
@ -60,45 +56,35 @@ func (fs Filesystem) GetObject(w io.Writer, bucket, object string, start, length
if os.IsNotExist(e) { if os.IsNotExist(e) {
_, e = os.Stat(filepath.Join(fs.path, bucket)) _, e = os.Stat(filepath.Join(fs.path, bucket))
if os.IsNotExist(e) { if os.IsNotExist(e) {
return 0, probe.NewError(BucketNotFound{Bucket: bucket}) return nil, probe.NewError(BucketNotFound{Bucket: bucket})
} }
return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
} }
return 0, probe.NewError(e) return nil, probe.NewError(e)
}
// Initiate a cached stat operation on the file handler.
st, e := file.Stat()
if e != nil {
return nil, probe.NewError(e)
}
// Object path is a directory prefix, return object not found error.
if st.IsDir() {
return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
} }
defer file.Close()
_, e = file.Seek(start, os.SEEK_SET) // Seet to a starting offset.
_, e = file.Seek(startOffset, os.SEEK_SET)
if e != nil { if e != nil {
// When the "handle is invalid", the file might be a directory on Windows. // When the "handle is invalid", the file might be a directory on Windows.
if runtime.GOOS == "windows" && strings.Contains(e.Error(), "handle is invalid") { if runtime.GOOS == "windows" && strings.Contains(e.Error(), "handle is invalid") {
return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object}) return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
} }
return nil, probe.NewError(e)
return 0, probe.NewError(e)
} }
var count int64 // Return successfully seeked file handler.
// Copy over the whole file if the length is non-positive. return file, nil
if length > 0 {
count, e = io.CopyN(w, file, length)
} else {
count, e = io.Copy(w, file)
}
if e != nil {
// This call will fail if the object is a directory. Stat the file to see if
// this is true, if so, return an ObjectNotFound error.
stat, e := os.Stat(objectPath)
if e == nil && stat.IsDir() {
return count, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}
return count, probe.NewError(e)
}
return count, nil
} }
// GetObjectInfo - get object info. // GetObjectInfo - get object info.
@ -113,7 +99,7 @@ func (fs Filesystem) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Er
} }
// Normalize buckets. // Normalize buckets.
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketPath := filepath.Join(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket)
if _, e := os.Stat(bucketPath); e != nil { if _, e := os.Stat(bucketPath); e != nil {
if os.IsNotExist(e) { if os.IsNotExist(e) {
@ -196,8 +182,8 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
return false return false
} }
// CreateObject - create an object. // PutObject - create an object.
func (fs Filesystem) CreateObject(bucket string, object string, size int64, data io.Reader, md5Bytes []byte) (ObjectInfo, *probe.Error) { func (fs Filesystem) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error) {
di, e := disk.GetInfo(fs.path) di, e := disk.GetInfo(fs.path)
if e != nil { if e != nil {
return ObjectInfo{}, probe.NewError(e) return ObjectInfo{}, probe.NewError(e)
@ -215,7 +201,7 @@ func (fs Filesystem) CreateObject(bucket string, object string, size int64, data
return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket}) return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketPath := filepath.Join(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket)
if _, e = os.Stat(bucketPath); e != nil { if _, e = os.Stat(bucketPath); e != nil {
if os.IsNotExist(e) { if os.IsNotExist(e) {
@ -234,8 +220,8 @@ func (fs Filesystem) CreateObject(bucket string, object string, size int64, data
// md5Hex representation. // md5Hex representation.
var md5Hex string var md5Hex string
if len(md5Bytes) != 0 { if len(metadata) != 0 {
md5Hex = hex.EncodeToString(md5Bytes) md5Hex = metadata["md5Sum"]
} }
// Write object. // Write object.
@ -275,7 +261,7 @@ func (fs Filesystem) CreateObject(bucket string, object string, size int64, data
} }
newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
if len(md5Bytes) != 0 { if md5Hex != "" {
if newMD5Hex != md5Hex { if newMD5Hex != md5Hex {
return ObjectInfo{}, probe.NewError(BadDigest{md5Hex, newMD5Hex}) return ObjectInfo{}, probe.NewError(BadDigest{md5Hex, newMD5Hex})
} }
@ -346,7 +332,7 @@ func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error {
return probe.NewError(BucketNameInvalid{Bucket: bucket}) return probe.NewError(BucketNameInvalid{Bucket: bucket})
} }
bucket = fs.denormalizeBucket(bucket) bucket = getActualBucketname(fs.path, bucket)
bucketPath := filepath.Join(fs.path, bucket) bucketPath := filepath.Join(fs.path, bucket)
// Check bucket exists // Check bucket exists
if _, e := os.Stat(bucketPath); e != nil { if _, e := os.Stat(bucketPath); e != nil {

74
pkg/fs/fs-object_test.go → fs-object_test.go
View File

@ -14,12 +14,14 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"bytes" "bytes"
"crypto/md5" "crypto/md5"
"encoding/hex"
"fmt" "fmt"
"io"
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
@ -36,8 +38,8 @@ func TestGetObjectInfo(t *testing.T) {
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
fs, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -46,7 +48,7 @@ func TestGetObjectInfo(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
_, err = fs.CreateObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) _, err = fs.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -56,7 +58,6 @@ func TestGetObjectInfo(t *testing.T) {
{Bucket: "test-getobjectinfo", Name: "Asia/asiapics.jpg", ContentType: "image/jpeg", IsDir: false}, {Bucket: "test-getobjectinfo", Name: "Asia/asiapics.jpg", ContentType: "image/jpeg", IsDir: false},
} }
testCases := []struct { testCases := []struct {
rootPath string
bucketName string bucketName string
objectName string objectName string
@ -67,24 +68,24 @@ func TestGetObjectInfo(t *testing.T) {
shouldPass bool shouldPass bool
}{ }{
// Test cases with invalid bucket names ( Test number 1-4 ). // Test cases with invalid bucket names ( Test number 1-4 ).
{fs.path, ".test", "", ObjectInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, {".test", "", ObjectInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
{fs.path, "Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, {"Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
{fs.path, "---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false}, {"---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false},
{fs.path, "ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, {"ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
// Test cases with valid but non-existing bucket names (Test number 5-7). // Test cases with valid but non-existing bucket names (Test number 5-7).
{fs.path, "abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false}, {"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false},
{fs.path, "ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false}, {"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false},
// Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9). // Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9).
{fs.path, "abcdefgh", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "abcdefgh", Object: ""}, false}, {"abcdefgh", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "abcdefgh", Object: ""}, false},
{fs.path, "ijklmnop", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "ijklmnop", Object: ""}, false}, {"ijklmnop", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "ijklmnop", Object: ""}, false},
// Test cases with non-existing object name with existing bucket (Test number 10-12). // Test cases with non-existing object name with existing bucket (Test number 10-12).
{fs.path, "test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false}, {"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false},
{fs.path, "test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false}, {"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false},
{fs.path, "test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false}, {"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false},
// Test case with existing bucket but object name set to a directory (Test number 13). // Test case with existing bucket but object name set to a directory (Test number 13).
{fs.path, "test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false}, {"test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false},
// Valid case with existing object (Test number 14). // Valid case with existing object (Test number 14).
{fs.path, "test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true}, {"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true},
} }
for i, testCase := range testCases { for i, testCase := range testCases {
result, err := fs.GetObjectInfo(testCase.bucketName, testCase.objectName) result, err := fs.GetObjectInfo(testCase.bucketName, testCase.objectName)
@ -127,8 +128,8 @@ func TestGetObjectInfoCore(t *testing.T) {
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
fs, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -137,7 +138,7 @@ func TestGetObjectInfoCore(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
_, err = fs.CreateObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil) _, err = fs.PutObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -153,7 +154,6 @@ func TestGetObjectInfoCore(t *testing.T) {
{Bucket: "test-getobjinfo", Name: "Africa", Size: 0, ContentType: "image/jpeg", IsDir: false}, {Bucket: "test-getobjinfo", Name: "Africa", Size: 0, ContentType: "image/jpeg", IsDir: false},
} }
testCases := []struct { testCases := []struct {
rootPath string
bucketName string bucketName string
objectName string objectName string
@ -165,14 +165,15 @@ func TestGetObjectInfoCore(t *testing.T) {
shouldPass bool shouldPass bool
}{ }{
// Testcase with object name set to a existing directory ( Test number 1). // Testcase with object name set to a existing directory ( Test number 1).
{fs.path, "test-getobjinfo", "Asia", resultCases[0], nil, true}, {"test-getobjinfo", "Asia", resultCases[0], nil, true},
// ObjectName set to a existing object ( Test number 2). // ObjectName set to a existing object ( Test number 2).
{fs.path, "test-getobjinfo", "Asia/asiapics.jpg", resultCases[1], nil, true}, {"test-getobjinfo", "Asia/asiapics.jpg", resultCases[1], nil, true},
// Object name set to a non-existing object. (Test number 3). // Object name set to a non-existing object. (Test number 3).
{fs.path, "test-getobjinfo", "Africa", resultCases[2], fmt.Errorf("%s", filepath.FromSlash("test-getobjinfo/Africa")), false}, {"test-getobjinfo", "Africa", resultCases[2], fmt.Errorf("%s", filepath.FromSlash("test-getobjinfo/Africa")), false},
} }
rootPath := fs.(*Filesystem).GetRootPath()
for i, testCase := range testCases { for i, testCase := range testCases {
result, err := getObjectInfo(testCase.rootPath, testCase.bucketName, testCase.objectName) result, err := getObjectInfo(rootPath, testCase.bucketName, testCase.objectName)
if err != nil && testCase.shouldPass { if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error()) t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
} }
@ -205,21 +206,21 @@ func TestGetObjectInfoCore(t *testing.T) {
} }
func BenchmarkGetObject(b *testing.B) { func BenchmarkGetObject(b *testing.B) {
// Make a temporary directory to use as the filesystem. // Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark-getobject") directory, e := ioutil.TempDir("", "minio-benchmark-getobject")
if e != nil { if e != nil {
b.Fatal(e) b.Fatal(e)
} }
defer os.RemoveAll(directory) defer os.RemoveAll(directory)
// Create the filesystem. // Create the fs.
filesystem, err := New(directory) fs, err := newFS(directory)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
// Make a bucket and put in a few objects. // Make a bucket and put in a few objects.
err = filesystem.MakeBucket("bucket") err = fs.MakeBucket("bucket")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -227,23 +228,30 @@ func BenchmarkGetObject(b *testing.B) {
text := "Jack and Jill went up the hill / To fetch a pail of water." text := "Jack and Jill went up the hill / To fetch a pail of water."
hasher := md5.New() hasher := md5.New()
hasher.Write([]byte(text)) hasher.Write([]byte(text))
metadata := make(map[string]string)
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
_, err = filesystem.CreateObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), hasher.Sum(nil)) metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
_, err = fs.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
} }
var w bytes.Buffer
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
n, err := filesystem.GetObject(&w, "bucket", "object"+strconv.Itoa(i%10), 0, 0) var w bytes.Buffer
r, err := fs.GetObject("bucket", "object"+strconv.Itoa(i%10), 0)
if err != nil { if err != nil {
b.Error(err) b.Error(err)
} }
n, e := io.Copy(&w, r)
if e != nil {
b.Error(e)
}
if n != int64(len(text)) { if n != int64(len(text)) {
b.Errorf("GetObject returned incorrect length %d (should be %d)\n", n, int64(len(text))) b.Errorf("GetObject returned incorrect length %d (should be %d)\n", n, int64(len(text)))
} }
r.Close()
} }
} }

2
pkg/fs/fs-utils.go → fs-utils.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"regexp" "regexp"

2
pkg/fs/fs-utils_test.go → fs-utils_test.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"testing" "testing"

10
pkg/fs/fs.go → fs.go
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"os" "os"
@ -95,8 +95,8 @@ type Multiparts struct {
ActiveSession map[string]*MultipartSession `json:"activeSessions"` ActiveSession map[string]*MultipartSession `json:"activeSessions"`
} }
// New instantiate a new donut // newFS instantiate a new filesystem.
func New(rootPath string) (Filesystem, *probe.Error) { func newFS(rootPath string) (ObjectAPI, *probe.Error) {
setFSMultipartsMetadataPath(filepath.Join(rootPath, "$multiparts-session.json")) setFSMultipartsMetadataPath(filepath.Join(rootPath, "$multiparts-session.json"))
var err *probe.Error var err *probe.Error
@ -117,7 +117,7 @@ func New(rootPath string) (Filesystem, *probe.Error) {
} }
} }
fs := Filesystem{ fs := &Filesystem{
rwLock: &sync.RWMutex{}, rwLock: &sync.RWMutex{},
} }
fs.path = rootPath fs.path = rootPath
@ -125,7 +125,7 @@ func New(rootPath string) (Filesystem, *probe.Error) {
/// Defaults /// Defaults
// minium free disk required for i/o operations to succeed. // Minium free disk required for i/o operations to succeed.
fs.minFreeDisk = 5 fs.minFreeDisk = 5
fs.listObjectMap = make(map[ListObjectParams][]ObjectInfoChannel) fs.listObjectMap = make(map[ListObjectParams][]ObjectInfoChannel)

107
pkg/fs/api_suite_test.go → fs_api_suite_test.go
View File

@ -14,13 +14,13 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"bytes" "bytes"
"crypto/md5" "crypto/md5"
"encoding/hex" "encoding/hex"
"encoding/xml" "io"
"math/rand" "math/rand"
"strconv" "strconv"
@ -28,7 +28,7 @@ import (
) )
// APITestSuite - collection of API tests // APITestSuite - collection of API tests
func APITestSuite(c *check.C, create func() Filesystem) { func APITestSuite(c *check.C, create func() ObjectAPI) {
testMakeBucket(c, create) testMakeBucket(c, create)
testMultipleObjectCreation(c, create) testMultipleObjectCreation(c, create)
testPaging(c, create) testPaging(c, create)
@ -46,13 +46,13 @@ func APITestSuite(c *check.C, create func() Filesystem) {
testMultipartObjectAbort(c, create) testMultipartObjectAbort(c, create)
} }
func testMakeBucket(c *check.C, create func() Filesystem) { func testMakeBucket(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
err := fs.MakeBucket("bucket") err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
} }
func testMultipartObjectCreation(c *check.C, create func() Filesystem) { func testMultipartObjectCreation(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
err := fs.MakeBucket("bucket") err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
@ -73,19 +73,17 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
var calculatedMD5sum string var calculatedMD5sum string
calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedMD5sum}) completedParts.Parts = append(completedParts.Parts, CompletePart{PartNumber: i, ETag: calculatedMD5sum})
} }
completedPartsBytes, e := xml.Marshal(completedParts) objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts)
c.Assert(e, check.IsNil)
objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedPartsBytes)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(objectInfo.MD5Sum, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10") c.Assert(objectInfo.MD5Sum, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10")
} }
func testMultipartObjectAbort(c *check.C, create func() Filesystem) { func testMultipartObjectAbort(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
err := fs.MakeBucket("bucket") err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
@ -105,7 +103,7 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil)) expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
var calculatedMD5sum string var calculatedMD5sum string
calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex) c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
parts[i] = expectedMD5Sumhex parts[i] = expectedMD5Sumhex
@ -114,7 +112,7 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
} }
func testMultipleObjectCreation(c *check.C, create func() Filesystem) { func testMultipleObjectCreation(c *check.C, create func() ObjectAPI) {
objects := make(map[string][]byte) objects := make(map[string][]byte)
fs := create() fs := create()
err := fs.MakeBucket("bucket") err := fs.MakeBucket("bucket")
@ -133,24 +131,28 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
key := "obj" + strconv.Itoa(i) key := "obj" + strconv.Itoa(i)
objects[key] = []byte(randomString) objects[key] = []byte(randomString)
var objectInfo ObjectInfo var objectInfo ObjectInfo
objectInfo, err = fs.CreateObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil)) metadata := make(map[string]string)
metadata["md5Sum"] = expectedMD5Sumhex
objectInfo, err = fs.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(objectInfo.MD5Sum, check.Equals, expectedMD5Sumhex) c.Assert(objectInfo.MD5Sum, check.Equals, expectedMD5Sumhex)
} }
for key, value := range objects { for key, value := range objects {
var byteBuffer bytes.Buffer var byteBuffer bytes.Buffer
_, err := fs.GetObject(&byteBuffer, "bucket", key, 0, 0) r, err := fs.GetObject("bucket", key, 0)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
io.Copy(&byteBuffer, r)
c.Assert(byteBuffer.Bytes(), check.DeepEquals, value) c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
metadata, err := fs.GetObjectInfo("bucket", key) metadata, err := fs.GetObjectInfo("bucket", key)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(metadata.Size, check.Equals, int64(len(value))) c.Assert(metadata.Size, check.Equals, int64(len(value)))
r.Close()
} }
} }
func testPaging(c *check.C, create func() Filesystem) { func testPaging(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
fs.MakeBucket("bucket") fs.MakeBucket("bucket")
result, err := fs.ListObjects("bucket", "", "", "", 0) result, err := fs.ListObjects("bucket", "", "", "", 0)
@ -160,7 +162,7 @@ func testPaging(c *check.C, create func() Filesystem) {
// check before paging occurs // check before paging occurs
for i := 0; i < 5; i++ { for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i) key := "obj" + strconv.Itoa(i)
_, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) _, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "", "", "", 5) result, err = fs.ListObjects("bucket", "", "", "", 5)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
@ -170,7 +172,7 @@ func testPaging(c *check.C, create func() Filesystem) {
// check after paging occurs pages work // check after paging occurs pages work
for i := 6; i <= 10; i++ { for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i) key := "obj" + strconv.Itoa(i)
_, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil) _, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "obj", "", "", 5) result, err = fs.ListObjects("bucket", "obj", "", "", 5)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
@ -179,9 +181,9 @@ func testPaging(c *check.C, create func() Filesystem) {
} }
// check paging with prefix at end returns less objects // check paging with prefix at end returns less objects
{ {
_, err = fs.CreateObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) _, err = fs.PutObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) _, err = fs.PutObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "new", "", "", 5) result, err = fs.ListObjects("bucket", "new", "", "", 5)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
@ -201,9 +203,9 @@ func testPaging(c *check.C, create func() Filesystem) {
// check delimited results with delimiter and prefix // check delimited results with delimiter and prefix
{ {
_, err = fs.CreateObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil) _, err = fs.PutObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil) _, err = fs.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10) result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
@ -250,32 +252,33 @@ func testPaging(c *check.C, create func() Filesystem) {
} }
} }
func testObjectOverwriteWorks(c *check.C, create func() Filesystem) { func testObjectOverwriteWorks(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
err := fs.MakeBucket("bucket") err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) _, err = fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
// c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum) // c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum)
_, err = fs.CreateObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil) _, err = fs.PutObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
var bytesBuffer bytes.Buffer var bytesBuffer bytes.Buffer
length, err := fs.GetObject(&bytesBuffer, "bucket", "object", 0, 0) r, err := fs.GetObject("bucket", "object", 0)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(length, check.Equals, int64(len("three"))) io.Copy(&bytesBuffer, r)
c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three") c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three")
r.Close()
} }
func testNonExistantBucketOperations(c *check.C, create func() Filesystem) { func testNonExistantBucketOperations(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
_, err := fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil) _, err := fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
c.Assert(err, check.Not(check.IsNil)) c.Assert(err, check.Not(check.IsNil))
} }
func testBucketRecreateFails(c *check.C, create func() Filesystem) { func testBucketRecreateFails(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
err := fs.MakeBucket("string") err := fs.MakeBucket("string")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
@ -283,22 +286,23 @@ func testBucketRecreateFails(c *check.C, create func() Filesystem) {
c.Assert(err, check.Not(check.IsNil)) c.Assert(err, check.Not(check.IsNil))
} }
func testPutObjectInSubdir(c *check.C, create func() Filesystem) { func testPutObjectInSubdir(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
err := fs.MakeBucket("bucket") err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) _, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
var bytesBuffer bytes.Buffer var bytesBuffer bytes.Buffer
length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0) r, err := fs.GetObject("bucket", "dir1/dir2/object", 0)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
io.Copy(&bytesBuffer, r)
c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world")) c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length) r.Close()
} }
func testListBuckets(c *check.C, create func() Filesystem) { func testListBuckets(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
// test empty list // test empty list
@ -330,7 +334,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
} }
func testListBucketsOrder(c *check.C, create func() Filesystem) { func testListBucketsOrder(c *check.C, create func() ObjectAPI) {
// if implementation contains a map, order of map keys will vary. // if implementation contains a map, order of map keys will vary.
// this ensures they return in the same order each time // this ensures they return in the same order each time
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
@ -348,7 +352,7 @@ func testListBucketsOrder(c *check.C, create func() Filesystem) {
} }
} }
func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) { func testListObjectsTestsForNonExistantBucket(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
result, err := fs.ListObjects("bucket", "", "", "", 1000) result, err := fs.ListObjects("bucket", "", "", "", 1000)
c.Assert(err, check.Not(check.IsNil)) c.Assert(err, check.Not(check.IsNil))
@ -356,16 +360,13 @@ func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesyst
c.Assert(len(result.Objects), check.Equals, 0) c.Assert(len(result.Objects), check.Equals, 0)
} }
func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) { func testNonExistantObjectInBucket(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
err := fs.MakeBucket("bucket") err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer _, err = fs.GetObject("bucket", "dir1", 0)
length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
c.Assert(length, check.Equals, int64(0))
c.Assert(err, check.Not(check.IsNil)) c.Assert(err, check.Not(check.IsNil))
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
switch err := err.ToGoError().(type) { switch err := err.ToGoError().(type) {
case ObjectNotFound: case ObjectNotFound:
c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1") c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
@ -374,17 +375,15 @@ func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
} }
} }
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) { func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
err := fs.MakeBucket("bucket") err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil) _, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer _, err = fs.GetObject("bucket", "dir1", 0)
length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
c.Assert(length, check.Equals, int64(0))
switch err := err.ToGoError().(type) { switch err := err.ToGoError().(type) {
case ObjectNotFound: case ObjectNotFound:
c.Assert(err.Bucket, check.Equals, "bucket") c.Assert(err.Bucket, check.Equals, "bucket")
@ -393,11 +392,8 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem)
// force a failure with a line number // force a failure with a line number
c.Assert(err, check.Equals, "ObjectNotFound") c.Assert(err, check.Equals, "ObjectNotFound")
} }
c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
var byteBuffer2 bytes.Buffer _, err = fs.GetObject("bucket", "dir1/", 0)
length, err = fs.GetObject(&byteBuffer, "bucket", "dir1/", 0, 0)
c.Assert(length, check.Equals, int64(0))
switch err := err.ToGoError().(type) { switch err := err.ToGoError().(type) {
case ObjectNotFound: case ObjectNotFound:
c.Assert(err.Bucket, check.Equals, "bucket") c.Assert(err.Bucket, check.Equals, "bucket")
@ -406,17 +402,16 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem)
// force a failure with a line number // force a failure with a line number
c.Assert(err, check.Equals, "ObjectNotFound") c.Assert(err, check.Equals, "ObjectNotFound")
} }
c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
} }
func testDefaultContentType(c *check.C, create func() Filesystem) { func testDefaultContentType(c *check.C, create func() ObjectAPI) {
fs := create() fs := create()
err := fs.MakeBucket("bucket") err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
// Test empty // Test empty
_, err = fs.CreateObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil) _, err = fs.PutObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil)
metadata, err := fs.GetObjectInfo("bucket", "one") objInfo, err := fs.GetObjectInfo("bucket", "one")
c.Assert(err, check.IsNil) c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/octet-stream") c.Assert(objInfo.ContentType, check.Equals, "application/octet-stream")
} }

18
pkg/fs/fs_test.go → fs_test.go
View File

@ -14,29 +14,22 @@
* limitations under the License. * limitations under the License.
*/ */
package fs package main
import ( import (
"io/ioutil" "io/ioutil"
"os" "os"
"testing"
. "gopkg.in/check.v1" . "gopkg.in/check.v1"
) )
func Test(t *testing.T) { TestingT(t) } func (s *MyAPISuite) TestAPISuite(c *C) {
type MySuite struct{}
var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
var storageList []string var storageList []string
create := func() Filesystem { create := func() ObjectAPI {
path, e := ioutil.TempDir(os.TempDir(), "minio-") path, e := ioutil.TempDir(os.TempDir(), "minio-")
c.Check(e, IsNil) c.Check(e, IsNil)
storageList = append(storageList, path) storageList = append(storageList, path)
store, err := New(path) store, err := newFS(path)
c.Check(err, IsNil) c.Check(err, IsNil)
return store return store
} }
@ -46,7 +39,6 @@ func (s *MySuite) TestAPISuite(c *C) {
func removeRoots(c *C, roots []string) { func removeRoots(c *C, roots []string) {
for _, root := range roots { for _, root := range roots {
err := os.RemoveAll(root) os.RemoveAll(root)
c.Check(err, IsNil)
} }
} }

13
httprange.go
View File

@ -22,7 +22,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/probe"
) )
@ -60,7 +59,7 @@ func getRequestedRange(hrange string, size int64) (*httpRange, *probe.Error) {
func (r *httpRange) parse(ra string) *probe.Error { func (r *httpRange) parse(ra string) *probe.Error {
i := strings.Index(ra, "-") i := strings.Index(ra, "-")
if i < 0 { if i < 0 {
return probe.NewError(fs.InvalidRange{}) return probe.NewError(InvalidRange{})
} }
start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:]) start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:])
if start == "" { if start == "" {
@ -68,7 +67,7 @@ func (r *httpRange) parse(ra string) *probe.Error {
// range start relative to the end of the file. // range start relative to the end of the file.
i, err := strconv.ParseInt(end, 10, 64) i, err := strconv.ParseInt(end, 10, 64)
if err != nil { if err != nil {
return probe.NewError(fs.InvalidRange{}) return probe.NewError(InvalidRange{})
} }
if i > r.size { if i > r.size {
i = r.size i = r.size
@ -78,7 +77,7 @@ func (r *httpRange) parse(ra string) *probe.Error {
} else { } else {
i, err := strconv.ParseInt(start, 10, 64) i, err := strconv.ParseInt(start, 10, 64)
if err != nil || i > r.size || i < 0 { if err != nil || i > r.size || i < 0 {
return probe.NewError(fs.InvalidRange{}) return probe.NewError(InvalidRange{})
} }
r.start = i r.start = i
if end == "" { if end == "" {
@ -87,7 +86,7 @@ func (r *httpRange) parse(ra string) *probe.Error {
} else { } else {
i, err := strconv.ParseInt(end, 10, 64) i, err := strconv.ParseInt(end, 10, 64)
if err != nil || r.start > i { if err != nil || r.start > i {
return probe.NewError(fs.InvalidRange{}) return probe.NewError(InvalidRange{})
} }
if i >= r.size { if i >= r.size {
i = r.size - 1 i = r.size - 1
@ -104,7 +103,7 @@ func (r *httpRange) parseRange(s string) *probe.Error {
return probe.NewError(errors.New("header not present")) return probe.NewError(errors.New("header not present"))
} }
if !strings.HasPrefix(s, b) { if !strings.HasPrefix(s, b) {
return probe.NewError(fs.InvalidRange{}) return probe.NewError(InvalidRange{})
} }
ras := strings.Split(s[len(b):], ",") ras := strings.Split(s[len(b):], ",")
@ -118,7 +117,7 @@ func (r *httpRange) parseRange(s string) *probe.Error {
ra := strings.TrimSpace(ras[0]) ra := strings.TrimSpace(ras[0])
if ra == "" { if ra == "" {
return probe.NewError(fs.InvalidRange{}) return probe.NewError(InvalidRange{})
} }
return r.parse(ra) return r.parse(ra)
} }

88
minio-main.go
View File

@ -30,7 +30,6 @@ import (
"github.com/minio/cli" "github.com/minio/cli"
"github.com/minio/mc/pkg/console" "github.com/minio/mc/pkg/console"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/minhttp" "github.com/minio/minio/pkg/minhttp"
"github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/probe"
) )
@ -92,11 +91,11 @@ EXAMPLES:
} }
// configureServer configure a new server instance // configureServer configure a new server instance
func configureServer(filesystem fs.Filesystem) *http.Server { func configureServer(objectAPI ObjectAPI) *http.Server {
// Minio server config // Minio server config
apiServer := &http.Server{ apiServer := &http.Server{
Addr: serverConfig.GetAddr(), Addr: serverConfig.GetAddr(),
Handler: configureServerHandler(filesystem), Handler: configureServerHandler(objectAPI),
MaxHeaderBytes: 1 << 20, MaxHeaderBytes: 1 << 20,
} }
@ -306,49 +305,52 @@ func serverMain(c *cli.Context) {
cli.ShowCommandHelpAndExit(c, "server", 1) cli.ShowCommandHelpAndExit(c, "server", 1)
} }
var objectAPI ObjectAPI
var err *probe.Error
// get backend. // get backend.
backend := serverConfig.GetBackend() backend := serverConfig.GetBackend()
if backend.Type == "fs" { if backend.Type == "fs" {
// Initialize file system. // Initialize filesystem storage layer.
filesystem, err := fs.New(backend.Disk) objectAPI, err = newFS(backend.Disk)
fatalIf(err.Trace(backend.Type, backend.Disk), "Initializing filesystem failed.", nil) fatalIf(err.Trace(backend.Type, backend.Disk), "Initializing filesystem failed.", nil)
} else { // else if backend.Type == "xl" { here.
// Configure server. console.Fatalln("No known backends configured, please use ‘minio init --help’ to initialize a backend.")
apiServer := configureServer(filesystem)
// Credential.
cred := serverConfig.GetCredential()
// Region.
region := serverConfig.GetRegion()
// Print credentials and region.
console.Println("\n" + cred.String() + " " + colorMagenta("Region: ") + colorWhite(region))
console.Println("\nMinio Object Storage:")
// Print api listen ips.
printListenIPs(apiServer)
console.Println("\nMinio Browser:")
// Print browser listen ips.
printListenIPs(apiServer)
console.Println("\nTo configure Minio Client:")
// Download 'mc' links.
if runtime.GOOS == "windows" {
console.Println(" Download 'mc' from https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe")
console.Println(" $ mc.exe config host add myminio http://localhost:9000 " + cred.AccessKeyID + " " + cred.SecretAccessKey)
} else {
console.Println(" $ wget https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc")
console.Println(" $ chmod 755 mc")
console.Println(" $ ./mc config host add myminio http://localhost:9000 " + cred.AccessKeyID + " " + cred.SecretAccessKey)
}
// Start server.
err = minhttp.ListenAndServe(apiServer)
errorIf(err.Trace(), "Failed to start the minio server.", nil)
return
} }
console.Println(colorGreen("No known backends configured, please use ‘minio init --help’ to initialize a backend."))
// Configure server.
apiServer := configureServer(objectAPI)
// Credential.
cred := serverConfig.GetCredential()
// Region.
region := serverConfig.GetRegion()
// Print credentials and region.
console.Println("\n" + cred.String() + " " + colorMagenta("Region: ") + colorWhite(region))
console.Println("\nMinio Object Storage:")
// Print api listen ips.
printListenIPs(apiServer)
console.Println("\nMinio Browser:")
// Print browser listen ips.
printListenIPs(apiServer)
console.Println("\nTo configure Minio Client:")
// Download 'mc' links.
if runtime.GOOS == "windows" {
console.Println(" Download 'mc' from https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe")
console.Println(" $ mc.exe config host add myminio http://localhost:9000 " + cred.AccessKeyID + " " + cred.SecretAccessKey)
} else {
console.Println(" $ wget https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc")
console.Println(" $ chmod 755 mc")
console.Println(" $ ./mc config host add myminio http://localhost:9000 " + cred.AccessKeyID + " " + cred.SecretAccessKey)
}
// Start server.
err = minhttp.ListenAndServe(apiServer)
errorIf(err.Trace(), "Failed to start the minio server.", nil)
} }

33
object-api-interface.go Normal file
View File

@ -0,0 +1,33 @@
package main
import (
"io"
"github.com/minio/minio/pkg/probe"
)
// ObjectAPI interface.
type ObjectAPI interface {
// Bucket resource API.
DeleteBucket(bucket string) *probe.Error
ListBuckets() ([]BucketInfo, *probe.Error)
MakeBucket(bucket string) *probe.Error
GetBucketInfo(bucket string) (BucketInfo, *probe.Error)
// Bucket query API.
ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error)
ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error)
// Object resource API.
GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error)
GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error)
PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error)
DeleteObject(bucket, object string) *probe.Error
// Object query API.
NewMultipartUpload(bucket, object string) (string, *probe.Error)
PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error)
ListObjectParts(bucket, object string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error)
CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (ObjectInfo, *probe.Error)
AbortMultipartUpload(bucket, object, uploadID string) *probe.Error
}

274
object-handlers.go
View File

@ -9,7 +9,7 @@
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implieapi.Filesystem. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implieapi.ObjectAPI.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
@ -19,17 +19,18 @@ package main
import ( import (
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/xml"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
mux "github.com/gorilla/mux" mux "github.com/gorilla/mux"
"github.com/minio/minio/pkg/fs"
"github.com/minio/minio/pkg/probe" "github.com/minio/minio/pkg/probe"
) )
@ -59,7 +60,7 @@ func setGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
// ---------- // ----------
// This implementation of the GET operation retrieves object. To use GET, // This implementation of the GET operation retrieves object. To use GET,
// you must have READ access to the object. // you must have READ access to the object.
func (api storageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
var object, bucket string var object, bucket string
vars := mux.Vars(r) vars := mux.Vars(r)
bucket = vars["bucket"] bucket = vars["bucket"]
@ -83,17 +84,17 @@ func (api storageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
} }
} }
objectInfo, err := api.Filesystem.GetObjectInfo(bucket, object) objectInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
if err != nil { if err != nil {
errorIf(err.Trace(), "GetObject failed.", nil) errorIf(err.Trace(), "GetObject failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.ObjectNotFound: case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.ObjectNameInvalid: case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -125,10 +126,27 @@ func (api storageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
setGetRespHeaders(w, r.URL.Query()) setGetRespHeaders(w, r.URL.Query())
// Get the object. // Get the object.
if _, err = api.Filesystem.GetObject(w, bucket, object, hrange.start, hrange.length); err != nil { startOffset := hrange.start
readCloser, err := api.ObjectAPI.GetObject(bucket, object, startOffset)
if err != nil {
errorIf(err.Trace(), "GetObject failed.", nil) errorIf(err.Trace(), "GetObject failed.", nil)
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
return return
} }
defer readCloser.Close() // Close after this handler returns.
if hrange.length > 0 {
if _, e := io.CopyN(w, readCloser, hrange.length); e != nil {
errorIf(probe.NewError(e), "Writing to client failed", nil)
// Do not send error response here, since client could have died.
return
}
} else {
if _, e := io.Copy(w, readCloser); e != nil {
errorIf(probe.NewError(e), "Writing to client failed", nil)
// Do not send error response here, since client could have died.
return
}
}
} }
var unixEpochTime = time.Unix(0, 0) var unixEpochTime = time.Unix(0, 0)
@ -228,7 +246,7 @@ func checkETag(w http.ResponseWriter, r *http.Request) bool {
// HeadObjectHandler - HEAD Object // HeadObjectHandler - HEAD Object
// ----------- // -----------
// The HEAD operation retrieves metadata from an object without returning the object itself. // The HEAD operation retrieves metadata from an object without returning the object itself.
func (api storageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
var object, bucket string var object, bucket string
vars := mux.Vars(r) vars := mux.Vars(r)
bucket = vars["bucket"] bucket = vars["bucket"]
@ -246,17 +264,17 @@ func (api storageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request)
} }
} }
objectInfo, err := api.Filesystem.GetObjectInfo(bucket, object) objectInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
if err != nil { if err != nil {
errorIf(err.Trace(bucket, object), "GetObjectInfo failed.", nil) errorIf(err.Trace(bucket, object), "GetObjectInfo failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.ObjectNotFound: case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.ObjectNameInvalid: case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -286,7 +304,7 @@ func (api storageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request)
// ---------- // ----------
// This implementation of the PUT operation adds an object to a bucket // This implementation of the PUT operation adds an object to a bucket
// while reading the object from another source. // while reading the object from another source.
func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
object := vars["object"] object := vars["object"]
@ -339,17 +357,17 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request)
return return
} }
objectInfo, err := api.Filesystem.GetObjectInfo(sourceBucket, sourceObject) objectInfo, err := api.ObjectAPI.GetObjectInfo(sourceBucket, sourceObject)
if err != nil { if err != nil {
errorIf(err.Trace(), "GetObjectInfo failed.", nil) errorIf(err.Trace(), "GetObjectInfo failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, objectSource) writeErrorResponse(w, r, ErrInvalidBucketName, objectSource)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, objectSource) writeErrorResponse(w, r, ErrNoSuchBucket, objectSource)
case fs.ObjectNotFound: case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, objectSource) writeErrorResponse(w, r, ErrNoSuchKey, objectSource)
case fs.ObjectNameInvalid: case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, objectSource) writeErrorResponse(w, r, ErrNoSuchKey, objectSource)
default: default:
writeErrorResponse(w, r, ErrInternalError, objectSource) writeErrorResponse(w, r, ErrInternalError, objectSource)
@ -388,37 +406,45 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request)
} }
} }
// Initialize a pipe for data pipe line. startOffset := int64(0) // Read the whole file.
reader, writer := io.Pipe() // Get the object.
readCloser, getErr := api.ObjectAPI.GetObject(sourceBucket, sourceObject, startOffset)
// Start writing in a routine. if getErr != nil {
go func() { errorIf(getErr.Trace(sourceBucket, sourceObject), "Reading "+objectSource+" failed.", nil)
defer writer.Close() switch err.ToGoError().(type) {
if _, getErr := api.Filesystem.GetObject(writer, sourceBucket, sourceObject, 0, 0); getErr != nil { case BucketNotFound:
writer.CloseWithError(probe.WrapError(getErr)) writeErrorResponse(w, r, ErrNoSuchBucket, objectSource)
return case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, objectSource)
default:
writeErrorResponse(w, r, ErrInternalError, objectSource)
} }
}() return
}
// Size of object. // Size of object.
size := objectInfo.Size size := objectInfo.Size
// Save metadata.
metadata := make(map[string]string)
metadata["md5Sum"] = hex.EncodeToString(md5Bytes)
// Create the object. // Create the object.
objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, reader, md5Bytes) objectInfo, err = api.ObjectAPI.PutObject(bucket, object, size, readCloser, metadata)
if err != nil { if err != nil {
errorIf(err.Trace(), "CreateObject failed.", nil) errorIf(err.Trace(), "PutObject failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.RootPathFull: case RootPathFull:
writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BadDigest: case BadDigest:
writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
case fs.IncompleteBody: case IncompleteBody:
writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path)
case fs.ObjectExistsAsPrefix: case ObjectExistsAsPrefix:
writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path) writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -431,6 +457,8 @@ func (api storageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request)
setCommonHeaders(w) setCommonHeaders(w)
// write success response. // write success response.
writeSuccessResponse(w, encodedSuccessResponse) writeSuccessResponse(w, encodedSuccessResponse)
// Explicitly close the reader, to avoid fd leaks.
readCloser.Close()
} }
// checkCopySource implements x-amz-copy-source-if-modified-since and // checkCopySource implements x-amz-copy-source-if-modified-since and
@ -528,7 +556,7 @@ func checkCopySourceETag(w http.ResponseWriter, r *http.Request) bool {
// PutObjectHandler - PUT Object // PutObjectHandler - PUT Object
// ---------- // ----------
// This implementation of the PUT operation adds an object to a bucket. // This implementation of the PUT operation adds an object to a bucket.
func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
// If the matching failed, it means that the X-Amz-Copy-Source was // If the matching failed, it means that the X-Amz-Copy-Source was
// wrong, fail right here. // wrong, fail right here.
if _, ok := r.Header["X-Amz-Copy-Source"]; ok { if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
@ -558,7 +586,7 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
var objectInfo fs.ObjectInfo var objectInfo ObjectInfo
switch getRequestAuthType(r) { switch getRequestAuthType(r) {
default: default:
// For all unknown auth types return error. // For all unknown auth types return error.
@ -571,7 +599,7 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
// Create anonymous object. // Create anonymous object.
objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, r.Body, nil) objectInfo, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, nil)
case authTypePresigned: case authTypePresigned:
// For presigned requests verify them right here. // For presigned requests verify them right here.
if apiErr := doesPresignedSignatureMatch(r); apiErr != ErrNone { if apiErr := doesPresignedSignatureMatch(r); apiErr != ErrNone {
@ -579,7 +607,7 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
// Create presigned object. // Create presigned object.
objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, r.Body, nil) objectInfo, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, nil)
case authTypeSigned: case authTypeSigned:
// Initialize a pipe for data pipe line. // Initialize a pipe for data pipe line.
reader, writer := io.Pipe() reader, writer := io.Pipe()
@ -605,11 +633,15 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
writer.Close() writer.Close()
}() }()
// Save metadata.
metadata := make(map[string]string)
metadata["md5Sum"] = hex.EncodeToString(md5Bytes)
// Create object. // Create object.
objectInfo, err = api.Filesystem.CreateObject(bucket, object, size, reader, md5Bytes) objectInfo, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata)
} }
if err != nil { if err != nil {
errorIf(err.Trace(), "CreateObject failed.", nil) errorIf(err.Trace(), "PutObject failed.", nil)
e := err.ToGoError() e := err.ToGoError()
// Verify if the underlying error is signature mismatch. // Verify if the underlying error is signature mismatch.
if e == errSignatureMismatch { if e == errSignatureMismatch {
@ -617,17 +649,17 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
switch e.(type) { switch e.(type) {
case fs.RootPathFull: case RootPathFull:
writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BadDigest: case BadDigest:
writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
case fs.IncompleteBody: case IncompleteBody:
writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path)
case fs.ObjectExistsAsPrefix: case ObjectExistsAsPrefix:
writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path) writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -640,10 +672,10 @@ func (api storageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
writeSuccessResponse(w, nil) writeSuccessResponse(w, nil)
} }
/// Multipart storageAPI /// Multipart objectStorageAPI
// NewMultipartUploadHandler - New multipart upload // NewMultipartUploadHandler - New multipart upload
func (api storageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
var object, bucket string var object, bucket string
vars := mux.Vars(r) vars := mux.Vars(r)
bucket = vars["bucket"] bucket = vars["bucket"]
@ -667,19 +699,19 @@ func (api storageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.R
} }
} }
uploadID, err := api.Filesystem.NewMultipartUpload(bucket, object) uploadID, err := api.ObjectAPI.NewMultipartUpload(bucket, object)
if err != nil { if err != nil {
errorIf(err.Trace(), "NewMultipartUpload failed.", nil) errorIf(err.Trace(), "NewMultipartUpload failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.RootPathFull: case RootPathFull:
writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.ObjectNotFound: case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.ObjectNameInvalid: case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -696,7 +728,7 @@ func (api storageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.R
} }
// PutObjectPartHandler - Upload part // PutObjectPartHandler - Upload part
func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
object := vars["object"] object := vars["object"]
@ -745,7 +777,7 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques
} }
// No need to verify signature, anonymous request access is // No need to verify signature, anonymous request access is
// already allowed. // already allowed.
partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, partID, size, r.Body, nil) partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, hex.EncodeToString(md5Bytes))
case authTypePresigned: case authTypePresigned:
// For presigned requests verify right here. // For presigned requests verify right here.
apiErr := doesPresignedSignatureMatch(r) apiErr := doesPresignedSignatureMatch(r)
@ -753,7 +785,7 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques
writeErrorResponse(w, r, apiErr, r.URL.Path) writeErrorResponse(w, r, apiErr, r.URL.Path)
return return
} }
partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, partID, size, r.Body, nil) partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, hex.EncodeToString(md5Bytes))
case authTypeSigned: case authTypeSigned:
// Initialize a pipe for data pipe line. // Initialize a pipe for data pipe line.
reader, writer := io.Pipe() reader, writer := io.Pipe()
@ -778,10 +810,10 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques
} }
writer.Close() writer.Close()
}() }()
partMD5, err = api.Filesystem.CreateObjectPart(bucket, object, uploadID, partID, size, reader, md5Bytes) partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, hex.EncodeToString(md5Bytes))
} }
if err != nil { if err != nil {
errorIf(err.Trace(), "CreateObjectPart failed.", nil) errorIf(err.Trace(), "PutObjectPart failed.", nil)
e := err.ToGoError() e := err.ToGoError()
// Verify if the underlying error is signature mismatch. // Verify if the underlying error is signature mismatch.
if e == errSignatureMismatch { if e == errSignatureMismatch {
@ -789,13 +821,13 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques
return return
} }
switch e.(type) { switch e.(type) {
case fs.RootPathFull: case RootPathFull:
writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path) writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
case fs.InvalidUploadID: case InvalidUploadID:
writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path)
case fs.BadDigest: case BadDigest:
writeErrorResponse(w, r, ErrBadDigest, r.URL.Path) writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
case fs.IncompleteBody: case IncompleteBody:
writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -809,7 +841,7 @@ func (api storageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Reques
} }
// AbortMultipartUploadHandler - Abort multipart upload // AbortMultipartUploadHandler - Abort multipart upload
func (api storageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
object := vars["object"] object := vars["object"]
@ -832,20 +864,20 @@ func (api storageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http
} }
} }
objectResourcesMetadata := getObjectResources(r.URL.Query()) uploadID := getUploadID(r.URL.Query()) // Get upload id.
err := api.Filesystem.AbortMultipartUpload(bucket, object, objectResourcesMetadata.UploadID) err := api.ObjectAPI.AbortMultipartUpload(bucket, object, uploadID)
if err != nil { if err != nil {
errorIf(err.Trace(), "AbortMutlipartUpload failed.", nil) errorIf(err.Trace(), "AbortMutlipartUpload failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.ObjectNotFound: case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.ObjectNameInvalid: case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.InvalidUploadID: case InvalidUploadID:
writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -856,7 +888,7 @@ func (api storageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http
} }
// ListObjectPartsHandler - List object parts // ListObjectPartsHandler - List object parts
func (api storageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
object := vars["object"] object := vars["object"]
@ -892,19 +924,19 @@ func (api storageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Requ
objectResourcesMetadata.MaxParts = maxPartsList objectResourcesMetadata.MaxParts = maxPartsList
} }
objectResourcesMetadata, err := api.Filesystem.ListObjectParts(bucket, object, objectResourcesMetadata) objectResourcesMetadata, err := api.ObjectAPI.ListObjectParts(bucket, object, objectResourcesMetadata)
if err != nil { if err != nil {
errorIf(err.Trace(), "ListObjectParts failed.", nil) errorIf(err.Trace(), "ListObjectParts failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.ObjectNotFound: case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.ObjectNameInvalid: case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.InvalidUploadID: case InvalidUploadID:
writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -920,16 +952,14 @@ func (api storageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Requ
} }
// CompleteMultipartUploadHandler - Complete multipart upload // CompleteMultipartUploadHandler - Complete multipart upload
func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
object := vars["object"] object := vars["object"]
// Extract object resources. // Get upload id.
objectResourcesMetadata := getObjectResources(r.URL.Query()) uploadID := getUploadID(r.URL.Query()) // Get upload id.
var objectInfo fs.ObjectInfo
var err *probe.Error
switch getRequestAuthType(r) { switch getRequestAuthType(r) {
default: default:
// For all unknown auth types return error. // For all unknown auth types return error.
@ -941,48 +971,52 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h
writeErrorResponse(w, r, s3Error, r.URL.Path) writeErrorResponse(w, r, s3Error, r.URL.Path)
return return
} }
completePartBytes, e := ioutil.ReadAll(r.Body)
if e != nil {
errorIf(probe.NewError(e), "CompleteMultipartUpload failed.", nil)
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
return
}
// Complete multipart upload anonymous.
objectInfo, err = api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, completePartBytes)
case authTypePresigned, authTypeSigned: case authTypePresigned, authTypeSigned:
if s3Error := isReqAuthenticated(r); s3Error != ErrNone { if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
writeErrorResponse(w, r, s3Error, r.URL.Path) writeErrorResponse(w, r, s3Error, r.URL.Path)
return return
} }
completePartBytes, e := ioutil.ReadAll(r.Body)
if e != nil {
errorIf(probe.NewError(e), "CompleteMultipartUpload failed.", nil)
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
return
}
// Complete multipart upload presigned.
objectInfo, err = api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, completePartBytes)
} }
completeMultipartBytes, e := ioutil.ReadAll(r.Body)
if e != nil {
errorIf(probe.NewError(e), "CompleteMultipartUpload failed.", nil)
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
return
}
completeMultipartUpload := &CompleteMultipartUpload{}
if e = xml.Unmarshal(completeMultipartBytes, completeMultipartUpload); e != nil {
writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
return
}
if !sort.IsSorted(completedParts(completeMultipartUpload.Parts)) {
writeErrorResponse(w, r, ErrInvalidPartOrder, r.URL.Path)
return
}
// Complete parts.
completeParts := completeMultipartUpload.Parts
// Complete multipart upload.
objectInfo, err := api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts)
if err != nil { if err != nil {
errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil) errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.ObjectNotFound: case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.ObjectNameInvalid: case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.InvalidUploadID: case InvalidUploadID:
writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path)
case fs.InvalidPart: case InvalidPart:
writeErrorResponse(w, r, ErrInvalidPart, r.URL.Path) writeErrorResponse(w, r, ErrInvalidPart, r.URL.Path)
case fs.InvalidPartOrder: case InvalidPartOrder:
writeErrorResponse(w, r, ErrInvalidPartOrder, r.URL.Path) writeErrorResponse(w, r, ErrInvalidPartOrder, r.URL.Path)
case fs.IncompleteBody: case IncompleteBody:
writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path) writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path)
case fs.MalformedXML: case MalformedXML:
writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path) writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@ -1000,10 +1034,10 @@ func (api storageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *h
writeSuccessResponse(w, encodedSuccessResponse) writeSuccessResponse(w, encodedSuccessResponse)
} }
/// Delete storageAPI /// Delete objectStorageAPI
// DeleteObjectHandler - delete an object // DeleteObjectHandler - delete an object
func (api storageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { func (api objectStorageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
object := vars["object"] object := vars["object"]
@ -1025,17 +1059,17 @@ func (api storageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request
return return
} }
} }
err := api.Filesystem.DeleteObject(bucket, object) err := api.ObjectAPI.DeleteObject(bucket, object)
if err != nil { if err != nil {
errorIf(err.Trace(), "DeleteObject failed.", nil) errorIf(err.Trace(), "DeleteObject failed.", nil)
switch err.ToGoError().(type) { switch err.ToGoError().(type) {
case fs.BucketNameInvalid: case BucketNameInvalid:
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path) writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case fs.BucketNotFound: case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case fs.ObjectNotFound: case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case fs.ObjectNameInvalid: case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path) writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
default: default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path) writeErrorResponse(w, r, ErrInternalError, r.URL.Path)

1
object-interface.go Normal file
View File

@ -0,0 +1 @@
package main

10
routers.go
View File

@ -20,18 +20,18 @@ import (
"net/http" "net/http"
router "github.com/gorilla/mux" router "github.com/gorilla/mux"
"github.com/minio/minio/pkg/fs"
) )
// configureServer handler returns final handler for the http server. // configureServer handler returns final handler for the http server.
func configureServerHandler(filesystem fs.Filesystem) http.Handler { func configureServerHandler(objectAPI ObjectAPI) http.Handler {
// Initialize API. // Initialize API.
api := storageAPI{ api := objectStorageAPI{
Filesystem: filesystem, ObjectAPI: objectAPI,
} }
// Initialize Web. // Initialize Web.
web := &webAPI{ web := &webAPI{
Filesystem: filesystem, ObjectAPI: objectAPI,
} }
// Initialize router. // Initialize router.

87
server_fs_test.go
View File

@ -35,22 +35,23 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"github.com/minio/minio/pkg/fs"
. "gopkg.in/check.v1" . "gopkg.in/check.v1"
) )
// Concurreny level.
const ( const (
ConcurrencyLevel = 10 ConcurrencyLevel = 10
) )
type MyAPIFSCacheSuite struct { // API suite container.
type MyAPISuite struct {
root string root string
req *http.Request req *http.Request
body io.ReadSeeker body io.ReadSeeker
credential credential credential credential
} }
var _ = Suite(&MyAPIFSCacheSuite{}) var _ = Suite(&MyAPISuite{})
var testAPIFSCacheServer *httptest.Server var testAPIFSCacheServer *httptest.Server
@ -69,7 +70,7 @@ func getFreePort() int {
return l.Addr().(*net.TCPAddr).Port return l.Addr().(*net.TCPAddr).Port
} }
func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) { func (s *MyAPISuite) SetUpSuite(c *C) {
root, e := ioutil.TempDir(os.TempDir(), "api-") root, e := ioutil.TempDir(os.TempDir(), "api-")
c.Assert(e, IsNil) c.Assert(e, IsNil)
s.root = root s.root = root
@ -95,14 +96,14 @@ func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) {
// Save config. // Save config.
c.Assert(serverConfig.Save(), IsNil) c.Assert(serverConfig.Save(), IsNil)
fs, err := fs.New(fsroot) fs, err := newFS(fsroot)
c.Assert(err, IsNil) c.Assert(err, IsNil)
httpHandler := configureServerHandler(fs) httpHandler := configureServerHandler(fs)
testAPIFSCacheServer = httptest.NewServer(httpHandler) testAPIFSCacheServer = httptest.NewServer(httpHandler)
} }
func (s *MyAPIFSCacheSuite) TearDownSuite(c *C) { func (s *MyAPISuite) TearDownSuite(c *C) {
os.RemoveAll(s.root) os.RemoveAll(s.root)
testAPIFSCacheServer.Close() testAPIFSCacheServer.Close()
} }
@ -142,7 +143,7 @@ var ignoredHeaders = map[string]bool{
"User-Agent": true, "User-Agent": true,
} }
func (s *MyAPIFSCacheSuite) newRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { func (s *MyAPISuite) newRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) {
if method == "" { if method == "" {
method = "POST" method = "POST"
} }
@ -267,7 +268,7 @@ func (s *MyAPIFSCacheSuite) newRequest(method, urlStr string, contentLength int6
return req, nil return req, nil
} }
func (s *MyAPIFSCacheSuite) TestAuth(c *C) { func (s *MyAPISuite) TestAuth(c *C) {
secretID, err := genSecretAccessKey() secretID, err := genSecretAccessKey()
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -278,7 +279,7 @@ func (s *MyAPIFSCacheSuite) TestAuth(c *C) {
c.Assert(len(accessID), Equals, minioAccessID) c.Assert(len(accessID), Equals, minioAccessID)
} }
func (s *MyAPIFSCacheSuite) TestBucketPolicy(c *C) { func (s *MyAPISuite) TestBucketPolicy(c *C) {
// Sample bucket policy. // Sample bucket policy.
bucketPolicyBuf := `{ bucketPolicyBuf := `{
"Version": "2012-10-17", "Version": "2012-10-17",
@ -348,7 +349,7 @@ func (s *MyAPIFSCacheSuite) TestBucketPolicy(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusNoContent) c.Assert(response.StatusCode, Equals, http.StatusNoContent)
} }
func (s *MyAPIFSCacheSuite) TestDeleteBucket(c *C) { func (s *MyAPISuite) TestDeleteBucket(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/deletebucket", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/deletebucket", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -366,7 +367,7 @@ func (s *MyAPIFSCacheSuite) TestDeleteBucket(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusNoContent) c.Assert(response.StatusCode, Equals, http.StatusNoContent)
} }
func (s *MyAPIFSCacheSuite) TestDeleteObject(c *C) { func (s *MyAPISuite) TestDeleteObject(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/deletebucketobject", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/deletebucketobject", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -391,7 +392,7 @@ func (s *MyAPIFSCacheSuite) TestDeleteObject(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusNoContent) c.Assert(response.StatusCode, Equals, http.StatusNoContent)
} }
func (s *MyAPIFSCacheSuite) TestNonExistantBucket(c *C) { func (s *MyAPISuite) TestNonExistantBucket(c *C) {
request, err := s.newRequest("HEAD", testAPIFSCacheServer.URL+"/nonexistantbucket", 0, nil) request, err := s.newRequest("HEAD", testAPIFSCacheServer.URL+"/nonexistantbucket", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -401,7 +402,7 @@ func (s *MyAPIFSCacheSuite) TestNonExistantBucket(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusNotFound) c.Assert(response.StatusCode, Equals, http.StatusNotFound)
} }
func (s *MyAPIFSCacheSuite) TestEmptyObject(c *C) { func (s *MyAPISuite) TestEmptyObject(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/emptyobject", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/emptyobject", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -432,7 +433,7 @@ func (s *MyAPIFSCacheSuite) TestEmptyObject(c *C) {
c.Assert(true, Equals, bytes.Equal(responseBody, buffer.Bytes())) c.Assert(true, Equals, bytes.Equal(responseBody, buffer.Bytes()))
} }
func (s *MyAPIFSCacheSuite) TestBucket(c *C) { func (s *MyAPISuite) TestBucket(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/bucket", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/bucket", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -450,7 +451,7 @@ func (s *MyAPIFSCacheSuite) TestBucket(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(response.StatusCode, Equals, http.StatusOK)
} }
func (s *MyAPIFSCacheSuite) TestObject(c *C) { func (s *MyAPISuite) TestObject(c *C) {
buffer := bytes.NewReader([]byte("hello world")) buffer := bytes.NewReader([]byte("hello world"))
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/testobject", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/testobject", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -482,7 +483,7 @@ func (s *MyAPIFSCacheSuite) TestObject(c *C) {
} }
func (s *MyAPIFSCacheSuite) TestMultipleObjects(c *C) { func (s *MyAPISuite) TestMultipleObjects(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/multipleobjects", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/multipleobjects", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -569,7 +570,7 @@ func (s *MyAPIFSCacheSuite) TestMultipleObjects(c *C) {
c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello three"))) c.Assert(true, Equals, bytes.Equal(responseBody, []byte("hello three")))
} }
func (s *MyAPIFSCacheSuite) TestNotImplemented(c *C) { func (s *MyAPISuite) TestNotImplemented(c *C) {
request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/bucket/object?policy", 0, nil) request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/bucket/object?policy", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -579,7 +580,7 @@ func (s *MyAPIFSCacheSuite) TestNotImplemented(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusNotImplemented) c.Assert(response.StatusCode, Equals, http.StatusNotImplemented)
} }
func (s *MyAPIFSCacheSuite) TestHeader(c *C) { func (s *MyAPISuite) TestHeader(c *C) {
request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/bucket/object", 0, nil) request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/bucket/object", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -590,7 +591,7 @@ func (s *MyAPIFSCacheSuite) TestHeader(c *C) {
verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound)
} }
func (s *MyAPIFSCacheSuite) TestPutBucket(c *C) { func (s *MyAPISuite) TestPutBucket(c *C) {
// Block 1: Testing for racey access // Block 1: Testing for racey access
// The assertion is removed from this block since the purpose of this block is to find races // The assertion is removed from this block since the purpose of this block is to find races
// The purpose this block is not to check for correctness of functionality // The purpose this block is not to check for correctness of functionality
@ -602,7 +603,6 @@ func (s *MyAPIFSCacheSuite) TestPutBucket(c *C) {
defer wg.Done() defer wg.Done()
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-bucket", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-bucket", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
request.Header.Add("x-amz-acl", "private")
client := http.Client{} client := http.Client{}
response, err := client.Do(request) response, err := client.Do(request)
@ -614,7 +614,6 @@ func (s *MyAPIFSCacheSuite) TestPutBucket(c *C) {
//Block 2: testing for correctness of the functionality //Block 2: testing for correctness of the functionality
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-bucket-slash/", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-bucket-slash/", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
request.Header.Add("x-amz-acl", "private")
client := http.Client{} client := http.Client{}
response, err := client.Do(request) response, err := client.Do(request)
@ -624,10 +623,9 @@ func (s *MyAPIFSCacheSuite) TestPutBucket(c *C) {
} }
func (s *MyAPIFSCacheSuite) TestCopyObject(c *C) { func (s *MyAPISuite) TestCopyObject(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-object-copy", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-object-copy", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
request.Header.Add("x-amz-acl", "private")
client := http.Client{} client := http.Client{}
response, err := client.Do(request) response, err := client.Do(request)
@ -662,7 +660,7 @@ func (s *MyAPIFSCacheSuite) TestCopyObject(c *C) {
c.Assert(string(object), Equals, "hello world") c.Assert(string(object), Equals, "hello world")
} }
func (s *MyAPIFSCacheSuite) TestPutObject(c *C) { func (s *MyAPISuite) TestPutObject(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-object", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/put-object", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -680,7 +678,7 @@ func (s *MyAPIFSCacheSuite) TestPutObject(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(response.StatusCode, Equals, http.StatusOK)
} }
func (s *MyAPIFSCacheSuite) TestListBuckets(c *C) { func (s *MyAPISuite) TestListBuckets(c *C) {
request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/", 0, nil) request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -695,7 +693,7 @@ func (s *MyAPIFSCacheSuite) TestListBuckets(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
} }
func (s *MyAPIFSCacheSuite) TestNotBeAbleToCreateObjectInNonexistantBucket(c *C) { func (s *MyAPISuite) TestNotBeAbleToCreateObjectInNonexistantBucket(c *C) {
buffer1 := bytes.NewReader([]byte("hello world")) buffer1 := bytes.NewReader([]byte("hello world"))
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/innonexistantbucket/object", int64(buffer1.Len()), buffer1) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/innonexistantbucket/object", int64(buffer1.Len()), buffer1)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -706,7 +704,7 @@ func (s *MyAPIFSCacheSuite) TestNotBeAbleToCreateObjectInNonexistantBucket(c *C)
verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist.", http.StatusNotFound) verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist.", http.StatusNotFound)
} }
func (s *MyAPIFSCacheSuite) TestHeadOnObject(c *C) { func (s *MyAPISuite) TestHeadOnObject(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/headonobject", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/headonobject", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -749,7 +747,7 @@ func (s *MyAPIFSCacheSuite) TestHeadOnObject(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusPreconditionFailed) c.Assert(response.StatusCode, Equals, http.StatusPreconditionFailed)
} }
func (s *MyAPIFSCacheSuite) TestHeadOnBucket(c *C) { func (s *MyAPISuite) TestHeadOnBucket(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/headonbucket", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/headonbucket", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -766,7 +764,7 @@ func (s *MyAPIFSCacheSuite) TestHeadOnBucket(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(response.StatusCode, Equals, http.StatusOK)
} }
func (s *MyAPIFSCacheSuite) TestXMLNameNotInBucketListJson(c *C) { func (s *MyAPISuite) TestXMLNameNotInBucketListJson(c *C) {
request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/", 0, nil) request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
request.Header.Add("Accept", "application/json") request.Header.Add("Accept", "application/json")
@ -781,7 +779,7 @@ func (s *MyAPIFSCacheSuite) TestXMLNameNotInBucketListJson(c *C) {
c.Assert(strings.Contains(string(byteResults), "XML"), Equals, false) c.Assert(strings.Contains(string(byteResults), "XML"), Equals, false)
} }
func (s *MyAPIFSCacheSuite) TestXMLNameNotInObjectListJson(c *C) { func (s *MyAPISuite) TestXMLNameNotInObjectListJson(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/xmlnamenotinobjectlistjson", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/xmlnamenotinobjectlistjson", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
request.Header.Add("Accept", "application/json") request.Header.Add("Accept", "application/json")
@ -805,7 +803,7 @@ func (s *MyAPIFSCacheSuite) TestXMLNameNotInObjectListJson(c *C) {
c.Assert(strings.Contains(string(byteResults), "XML"), Equals, false) c.Assert(strings.Contains(string(byteResults), "XML"), Equals, false)
} }
func (s *MyAPIFSCacheSuite) TestContentTypePersists(c *C) { func (s *MyAPISuite) TestContentTypePersists(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/contenttype-persists", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/contenttype-persists", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -865,7 +863,7 @@ func (s *MyAPIFSCacheSuite) TestContentTypePersists(c *C) {
c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream") c.Assert(response.Header.Get("Content-Type"), Equals, "application/octet-stream")
} }
func (s *MyAPIFSCacheSuite) TestPartialContent(c *C) { func (s *MyAPISuite) TestPartialContent(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/partial-content", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/partial-content", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -898,7 +896,7 @@ func (s *MyAPIFSCacheSuite) TestPartialContent(c *C) {
c.Assert(string(partialObject), Equals, "Wo") c.Assert(string(partialObject), Equals, "Wo")
} }
func (s *MyAPIFSCacheSuite) TestListObjectsHandlerErrors(c *C) { func (s *MyAPISuite) TestListObjectsHandlerErrors(c *C) {
request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/objecthandlererrors-.", 0, nil) request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/objecthandlererrors-.", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -931,7 +929,7 @@ func (s *MyAPIFSCacheSuite) TestListObjectsHandlerErrors(c *C) {
verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647.", http.StatusBadRequest) verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647.", http.StatusBadRequest)
} }
func (s *MyAPIFSCacheSuite) TestPutBucketErrors(c *C) { func (s *MyAPISuite) TestPutBucketErrors(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/putbucket-.", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/putbucket-.", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -963,7 +961,7 @@ func (s *MyAPIFSCacheSuite) TestPutBucketErrors(c *C) {
verifyError(c, response, "NotImplemented", "A header you provided implies functionality that is not implemented.", http.StatusNotImplemented) verifyError(c, response, "NotImplemented", "A header you provided implies functionality that is not implemented.", http.StatusNotImplemented)
} }
func (s *MyAPIFSCacheSuite) TestGetObjectErrors(c *C) { func (s *MyAPISuite) TestGetObjectErrors(c *C) {
request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/getobjecterrors", 0, nil) request, err := s.newRequest("GET", testAPIFSCacheServer.URL+"/getobjecterrors", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -997,7 +995,7 @@ func (s *MyAPIFSCacheSuite) TestGetObjectErrors(c *C) {
} }
func (s *MyAPIFSCacheSuite) TestGetObjectRangeErrors(c *C) { func (s *MyAPISuite) TestGetObjectRangeErrors(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/getobjectrangeerrors", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/getobjectrangeerrors", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -1025,7 +1023,7 @@ func (s *MyAPIFSCacheSuite) TestGetObjectRangeErrors(c *C) {
verifyError(c, response, "InvalidRange", "The requested range cannot be satisfied.", http.StatusRequestedRangeNotSatisfiable) verifyError(c, response, "InvalidRange", "The requested range cannot be satisfied.", http.StatusRequestedRangeNotSatisfiable)
} }
func (s *MyAPIFSCacheSuite) TestObjectMultipartAbort(c *C) { func (s *MyAPISuite) TestObjectMultipartAbort(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartabort", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartabort", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -1072,7 +1070,8 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipartAbort(c *C) {
c.Assert(response3.StatusCode, Equals, http.StatusNoContent) c.Assert(response3.StatusCode, Equals, http.StatusNoContent)
} }
func (s *MyAPIFSCacheSuite) TestBucketMultipartList(c *C) { /*
func (s *MyAPISuite) TestBucketMultipartList(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/bucketmultipartlist", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/bucketmultipartlist", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -1085,6 +1084,7 @@ func (s *MyAPIFSCacheSuite) TestBucketMultipartList(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
response, err = client.Do(request) response, err = client.Do(request)
c.Assert(err, IsNil)
c.Assert(response.StatusCode, Equals, http.StatusOK) c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body) decoder := xml.NewDecoder(response.Body)
@ -1159,8 +1159,9 @@ func (s *MyAPIFSCacheSuite) TestBucketMultipartList(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(newResponse3.Bucket, Equals, "bucketmultipartlist") c.Assert(newResponse3.Bucket, Equals, "bucketmultipartlist")
} }
*/
func (s *MyAPIFSCacheSuite) TestValidateObjectMultipartUploadID(c *C) { func (s *MyAPISuite) TestValidateObjectMultipartUploadID(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartlist-uploadid", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartlist-uploadid", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -1183,7 +1184,7 @@ func (s *MyAPIFSCacheSuite) TestValidateObjectMultipartUploadID(c *C) {
c.Assert(len(newResponse.UploadID) > 0, Equals, true) c.Assert(len(newResponse.UploadID) > 0, Equals, true)
} }
func (s *MyAPIFSCacheSuite) TestObjectMultipartList(c *C) { func (s *MyAPISuite) TestObjectMultipartList(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartlist", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultipartlist", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -1237,7 +1238,7 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipartList(c *C) {
verifyError(c, response4, "InvalidArgument", "Argument maxParts must be an integer between 1 and 10000.", http.StatusBadRequest) verifyError(c, response4, "InvalidArgument", "Argument maxParts must be an integer between 1 and 10000.", http.StatusBadRequest)
} }
func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) { func (s *MyAPISuite) TestObjectMultipart(c *C) {
request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultiparts", 0, nil) request, err := s.newRequest("PUT", testAPIFSCacheServer.URL+"/objectmultiparts", 0, nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
@ -1287,8 +1288,8 @@ func (s *MyAPIFSCacheSuite) TestObjectMultipart(c *C) {
c.Assert(response2.StatusCode, Equals, http.StatusOK) c.Assert(response2.StatusCode, Equals, http.StatusOK)
// Complete multipart upload // Complete multipart upload
completeUploads := &fs.CompleteMultipartUpload{ completeUploads := &CompleteMultipartUpload{
Part: []fs.CompletePart{ Parts: []CompletePart{
{ {
PartNumber: 1, PartNumber: 1,
ETag: response1.Header.Get("ETag"), ETag: response1.Header.Get("ETag"),

60
web-handlers.go
View File

@ -18,6 +18,7 @@ package main
import ( import (
"fmt" "fmt"
"io"
"net/http" "net/http"
"os" "os"
"path" "path"
@ -31,7 +32,6 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/gorilla/rpc/v2/json2" "github.com/gorilla/rpc/v2/json2"
"github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/fs"
"github.com/minio/miniobrowser" "github.com/minio/miniobrowser"
) )
@ -110,7 +110,7 @@ func (web *webAPI) DiskInfo(r *http.Request, args *GenericArgs, reply *DiskInfoR
if !isJWTReqAuthenticated(r) { if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"} return &json2.Error{Message: "Unauthorized request"}
} }
info, e := disk.GetInfo(web.Filesystem.GetRootPath()) info, e := disk.GetInfo(web.ObjectAPI.(*Filesystem).GetRootPath())
if e != nil { if e != nil {
return &json2.Error{Message: e.Error()} return &json2.Error{Message: e.Error()}
} }
@ -130,7 +130,7 @@ func (web *webAPI) MakeBucket(r *http.Request, args *MakeBucketArgs, reply *Gene
return &json2.Error{Message: "Unauthorized request"} return &json2.Error{Message: "Unauthorized request"}
} }
reply.UIVersion = miniobrowser.UIVersion reply.UIVersion = miniobrowser.UIVersion
e := web.Filesystem.MakeBucket(args.BucketName) e := web.ObjectAPI.MakeBucket(args.BucketName)
if e != nil { if e != nil {
return &json2.Error{Message: e.Cause.Error()} return &json2.Error{Message: e.Cause.Error()}
} }
@ -139,12 +139,12 @@ func (web *webAPI) MakeBucket(r *http.Request, args *MakeBucketArgs, reply *Gene
// ListBucketsRep - list buckets response // ListBucketsRep - list buckets response
type ListBucketsRep struct { type ListBucketsRep struct {
Buckets []BucketInfo `json:"buckets"` Buckets []BketInfo `json:"buckets"`
UIVersion string `json:"uiVersion"` UIVersion string `json:"uiVersion"`
} }
// BucketInfo container for list buckets metadata. // BketInfo container for list buckets.
type BucketInfo struct { type BketInfo struct {
// The name of the bucket. // The name of the bucket.
Name string `json:"name"` Name string `json:"name"`
// Date the bucket was created. // Date the bucket was created.
@ -156,14 +156,14 @@ func (web *webAPI) ListBuckets(r *http.Request, args *GenericArgs, reply *ListBu
if !isJWTReqAuthenticated(r) { if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"} return &json2.Error{Message: "Unauthorized request"}
} }
buckets, e := web.Filesystem.ListBuckets() buckets, e := web.ObjectAPI.ListBuckets()
if e != nil { if e != nil {
return &json2.Error{Message: e.Cause.Error()} return &json2.Error{Message: e.Cause.Error()}
} }
for _, bucket := range buckets { for _, bucket := range buckets {
// List all buckets which are not private. // List all buckets which are not private.
if bucket.Name != path.Base(reservedBucket) { if bucket.Name != path.Base(reservedBucket) {
reply.Buckets = append(reply.Buckets, BucketInfo{ reply.Buckets = append(reply.Buckets, BketInfo{
Name: bucket.Name, Name: bucket.Name,
CreationDate: bucket.Created, CreationDate: bucket.Created,
}) })
@ -181,12 +181,12 @@ type ListObjectsArgs struct {
// ListObjectsRep - list objects response. // ListObjectsRep - list objects response.
type ListObjectsRep struct { type ListObjectsRep struct {
Objects []ObjectInfo `json:"objects"` Objects []ObjInfo `json:"objects"`
UIVersion string `json:"uiVersion"` UIVersion string `json:"uiVersion"`
} }
// ObjectInfo container for list objects metadata. // ObjInfo container for list objects.
type ObjectInfo struct { type ObjInfo struct {
// Name of the object // Name of the object
Key string `json:"name"` Key string `json:"name"`
// Date and time the object was last modified. // Date and time the object was last modified.
@ -204,20 +204,20 @@ func (web *webAPI) ListObjects(r *http.Request, args *ListObjectsArgs, reply *Li
return &json2.Error{Message: "Unauthorized request"} return &json2.Error{Message: "Unauthorized request"}
} }
for { for {
lo, err := web.Filesystem.ListObjects(args.BucketName, args.Prefix, marker, "/", 1000) lo, err := web.ObjectAPI.ListObjects(args.BucketName, args.Prefix, marker, "/", 1000)
if err != nil { if err != nil {
return &json2.Error{Message: err.Cause.Error()} return &json2.Error{Message: err.Cause.Error()}
} }
marker = lo.NextMarker marker = lo.NextMarker
for _, obj := range lo.Objects { for _, obj := range lo.Objects {
reply.Objects = append(reply.Objects, ObjectInfo{ reply.Objects = append(reply.Objects, ObjInfo{
Key: obj.Name, Key: obj.Name,
LastModified: obj.ModifiedTime, LastModified: obj.ModifiedTime,
Size: obj.Size, Size: obj.Size,
}) })
} }
for _, prefix := range lo.Prefixes { for _, prefix := range lo.Prefixes {
reply.Objects = append(reply.Objects, ObjectInfo{ reply.Objects = append(reply.Objects, ObjInfo{
Key: prefix, Key: prefix,
}) })
} }
@ -242,7 +242,7 @@ func (web *webAPI) RemoveObject(r *http.Request, args *RemoveObjectArgs, reply *
return &json2.Error{Message: "Unauthorized request"} return &json2.Error{Message: "Unauthorized request"}
} }
reply.UIVersion = miniobrowser.UIVersion reply.UIVersion = miniobrowser.UIVersion
e := web.Filesystem.DeleteObject(args.BucketName, args.ObjectName) e := web.ObjectAPI.DeleteObject(args.BucketName, args.ObjectName)
if e != nil { if e != nil {
return &json2.Error{Message: e.Cause.Error()} return &json2.Error{Message: e.Cause.Error()}
} }
@ -364,7 +364,7 @@ func (web *webAPI) Upload(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r) vars := mux.Vars(r)
bucket := vars["bucket"] bucket := vars["bucket"]
object := vars["object"] object := vars["object"]
if _, err := web.Filesystem.CreateObject(bucket, object, -1, r.Body, nil); err != nil { if _, err := web.ObjectAPI.PutObject(bucket, object, -1, r.Body, nil); err != nil {
writeWebErrorResponse(w, err.ToGoError()) writeWebErrorResponse(w, err.ToGoError())
} }
} }
@ -389,8 +389,14 @@ func (web *webAPI) Download(w http.ResponseWriter, r *http.Request) {
} }
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base(object))) w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base(object)))
if _, err := web.Filesystem.GetObject(w, bucket, object, 0, 0); err != nil { objReader, err := web.ObjectAPI.GetObject(bucket, object, 0)
if err != nil {
writeWebErrorResponse(w, err.ToGoError()) writeWebErrorResponse(w, err.ToGoError())
return
}
if _, e := io.Copy(w, objReader); e != nil {
/// No need to print error, response writer already written to.
return
} }
} }
@ -402,35 +408,35 @@ func writeWebErrorResponse(w http.ResponseWriter, err error) {
return return
} }
switch err.(type) { switch err.(type) {
case fs.RootPathFull: case RootPathFull:
apiErr := getAPIError(ErrRootPathFull) apiErr := getAPIError(ErrRootPathFull)
w.WriteHeader(apiErr.HTTPStatusCode) w.WriteHeader(apiErr.HTTPStatusCode)
w.Write([]byte(apiErr.Description)) w.Write([]byte(apiErr.Description))
case fs.BucketNotFound: case BucketNotFound:
apiErr := getAPIError(ErrNoSuchBucket) apiErr := getAPIError(ErrNoSuchBucket)
w.WriteHeader(apiErr.HTTPStatusCode) w.WriteHeader(apiErr.HTTPStatusCode)
w.Write([]byte(apiErr.Description)) w.Write([]byte(apiErr.Description))
case fs.BucketNameInvalid: case BucketNameInvalid:
apiErr := getAPIError(ErrInvalidBucketName) apiErr := getAPIError(ErrInvalidBucketName)
w.WriteHeader(apiErr.HTTPStatusCode) w.WriteHeader(apiErr.HTTPStatusCode)
w.Write([]byte(apiErr.Description)) w.Write([]byte(apiErr.Description))
case fs.BadDigest: case BadDigest:
apiErr := getAPIError(ErrBadDigest) apiErr := getAPIError(ErrBadDigest)
w.WriteHeader(apiErr.HTTPStatusCode) w.WriteHeader(apiErr.HTTPStatusCode)
w.Write([]byte(apiErr.Description)) w.Write([]byte(apiErr.Description))
case fs.IncompleteBody: case IncompleteBody:
apiErr := getAPIError(ErrIncompleteBody) apiErr := getAPIError(ErrIncompleteBody)
w.WriteHeader(apiErr.HTTPStatusCode) w.WriteHeader(apiErr.HTTPStatusCode)
w.Write([]byte(apiErr.Description)) w.Write([]byte(apiErr.Description))
case fs.ObjectExistsAsPrefix: case ObjectExistsAsPrefix:
apiErr := getAPIError(ErrObjectExistsAsPrefix) apiErr := getAPIError(ErrObjectExistsAsPrefix)
w.WriteHeader(apiErr.HTTPStatusCode) w.WriteHeader(apiErr.HTTPStatusCode)
w.Write([]byte(apiErr.Description)) w.Write([]byte(apiErr.Description))
case fs.ObjectNotFound: case ObjectNotFound:
apiErr := getAPIError(ErrNoSuchKey) apiErr := getAPIError(ErrNoSuchKey)
w.WriteHeader(apiErr.HTTPStatusCode) w.WriteHeader(apiErr.HTTPStatusCode)
w.Write([]byte(apiErr.Description)) w.Write([]byte(apiErr.Description))
case fs.ObjectNameInvalid: case ObjectNameInvalid:
apiErr := getAPIError(ErrNoSuchKey) apiErr := getAPIError(ErrNoSuchKey)
w.WriteHeader(apiErr.HTTPStatusCode) w.WriteHeader(apiErr.HTTPStatusCode)
w.Write([]byte(apiErr.Description)) w.Write([]byte(apiErr.Description))

3
web-router.go
View File

@ -25,13 +25,12 @@ import (
router "github.com/gorilla/mux" router "github.com/gorilla/mux"
jsonrpc "github.com/gorilla/rpc/v2" jsonrpc "github.com/gorilla/rpc/v2"
"github.com/gorilla/rpc/v2/json2" "github.com/gorilla/rpc/v2/json2"
"github.com/minio/minio/pkg/fs"
"github.com/minio/miniobrowser" "github.com/minio/miniobrowser"
) )
// webAPI container for Web API. // webAPI container for Web API.
type webAPI struct { type webAPI struct {
Filesystem fs.Filesystem ObjectAPI ObjectAPI
} }
// indexHandler - Handler to serve index.html // indexHandler - Handler to serve index.html