MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 13:39:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

Adding multipart support

Browse Source
This commit is contained in:
Frederick F. Kautz IV
2015-05-07 19:55:30 -07:00
parent a4b4e9c148
commit f050d5e974
9 changed files with 314 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
pkg/api/api_definitions.go
View File

@@ -95,6 +95,26 @@ type Owner struct {
DisplayName string
}
// InitiateMultipartUploadResult - Returns upload id to use
type InitiateMultipartUploadResult struct {
XMLName xml.Name `xml:"http://doc.s3.amazonaws.com/2006-03-01 InitiateMultipartUploadResult" json:"-"`
Bucket string
Key string
UploadID string `xml:"UploadId"`
}
// CompleteMultipartUpload - Construct object from uploaded parts
type CompleteMultipartUpload struct {
Part []Part
}
// Part - Description of a multipart part
type Part struct {
PartNumber int
ETag string
}
// List of not implemented bucket queries
var notimplementedBucketResourceNames = map[string]bool{
"policy": true,
@@ -113,7 +133,6 @@ var notimplementedBucketResourceNames = map[string]bool{
// List of not implemented object queries
var notimplementedObjectResourceNames = map[string]bool{
"uploadId": true,
"torrent": true,
"uploads": true,
"torrent": true,
"uploads": true,
}

158
pkg/api/api_object_handlers.go
View File

@@ -20,6 +20,7 @@ import (
"net/http"
"strconv"
"encoding/xml"
"github.com/gorilla/mux"
"github.com/minio-io/minio/pkg/iodine"
"github.com/minio-io/minio/pkg/storage/drivers"
@@ -173,8 +174,8 @@ func (server *minioAPI) putObjectHandler(w http.ResponseWriter, req *http.Reques
return
}
// ignoring error here, TODO find a way to reply back if we can
sizeInt, _ := strconv.ParseInt(size, 10, 64)
calculatedMD5, err := server.driver.CreateObject(bucket, object, "", md5, sizeInt, req.Body)
sizeInt64, _ := strconv.ParseInt(size, 10, 64)
calculatedMD5, err := server.driver.CreateObject(bucket, object, "", md5, sizeInt64, req.Body)
switch err := iodine.ToError(err).(type) {
case nil:
{
@@ -210,3 +211,156 @@ func (server *minioAPI) putObjectHandler(w http.ResponseWriter, req *http.Reques
}
}
}
func (server *minioAPI) newMultipartUploadHandler(w http.ResponseWriter, req *http.Request) {
acceptsContentType := getContentType(req)
if acceptsContentType == unknownContentType {
writeErrorResponse(w, req, NotAcceptable, acceptsContentType, req.URL.Path)
return
}
// handle ACL's here at bucket level
if !server.isValidOp(w, req, acceptsContentType) {
return
}
var object, bucket string
vars := mux.Vars(req)
bucket = vars["bucket"]
object = vars["object"]
var uploadID string
var err error
if uploadID, err = server.driver.NewMultipartUpload(bucket, object, ""); err != nil {
log.Println(iodine.New(err, nil))
writeErrorResponse(w, req, NotImplemented, acceptsContentType, req.URL.Path)
return
}
response := generateInitiateMultipartUploadResult(bucket, object, uploadID)
encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
// write headers
setCommonHeaders(w, getContentTypeString(acceptsContentType))
// set content-length to the size of the body
w.Header().Set("Content-Length", strconv.Itoa(len(encodedSuccessResponse)))
w.WriteHeader(http.StatusOK)
// write body
w.Write(encodedSuccessResponse)
}
func (server *minioAPI) putObjectPartHandler(w http.ResponseWriter, req *http.Request) {
acceptsContentType := getContentType(req)
if acceptsContentType == unknownContentType {
writeErrorResponse(w, req, NotAcceptable, acceptsContentType, req.URL.Path)
return
}
// handle ACL's here at bucket level
if !server.isValidOp(w, req, acceptsContentType) {
return
}
// get Content-MD5 sent by client and verify if valid
md5 := req.Header.Get("Content-MD5")
if !isValidMD5(md5) {
writeErrorResponse(w, req, InvalidDigest, acceptsContentType, req.URL.Path)
return
}
/// if Content-Length missing, throw away
size := req.Header.Get("Content-Length")
if size == "" {
writeErrorResponse(w, req, MissingContentLength, acceptsContentType, req.URL.Path)
return
}
/// maximum Upload size for objects in a single operation
if isMaxObjectSize(size) {
writeErrorResponse(w, req, EntityTooLarge, acceptsContentType, req.URL.Path)
return
}
/// minimum Upload size for objects in a single operation
if isMinObjectSize(size) {
writeErrorResponse(w, req, EntityTooSmall, acceptsContentType, req.URL.Path)
return
}
// ignoring error here, TODO find a way to reply back if we can
sizeInt64, _ := strconv.ParseInt(size, 10, 64)
var object, bucket string
vars := mux.Vars(req)
bucket = vars["bucket"]
object = vars["object"]
uploadID := vars["uploadID"]
partIDString := vars["partNumber"]
partID, err := strconv.Atoi(partIDString)
if err != nil {
// TODO find the write value for this error
writeErrorResponse(w, req, NotAcceptable, acceptsContentType, req.URL.Path)
}
calculatedMD5, err := server.driver.CreateObjectPart(bucket, object, uploadID, partID, "", md5, sizeInt64, req.Body)
switch err := iodine.ToError(err).(type) {
case nil:
{
w.Header().Set("ETag", calculatedMD5)
writeSuccessResponse(w, acceptsContentType)
}
case drivers.ObjectExists:
{
writeErrorResponse(w, req, MethodNotAllowed, acceptsContentType, req.URL.Path)
}
case drivers.BadDigest:
{
writeErrorResponse(w, req, BadDigest, acceptsContentType, req.URL.Path)
}
case drivers.EntityTooLarge:
{
writeErrorResponse(w, req, EntityTooLarge, acceptsContentType, req.URL.Path)
}
case drivers.InvalidDigest:
{
writeErrorResponse(w, req, InvalidDigest, acceptsContentType, req.URL.Path)
}
case drivers.ImplementationError:
{
log.Error.Println(err)
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
}
default:
{
log.Error.Println(err)
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
}
}
}
func (server *minioAPI) completeMultipartUploadHandler(w http.ResponseWriter, req *http.Request) {
acceptsContentType := getContentType(req)
if acceptsContentType == unknownContentType {
writeErrorResponse(w, req, NotAcceptable, acceptsContentType, req.URL.Path)
return
}
decoder := xml.NewDecoder(req.Body)
parts := &CompleteMultipartUpload{}
err := decoder.Decode(parts)
if err != nil {
log.Error.Println(err)
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
}
partMap := make(map[int]string)
vars := mux.Vars(req)
bucket := vars["bucket"]
object := vars["object"]
uploadID := vars["uploadID"]
for _, part := range parts.Part {
partMap[part.PartNumber] = part.ETag
}
err = server.driver.CompleteMultipartUpload(bucket, object, uploadID, partMap)
}
func (server *minioAPI) notImplementedHandler(w http.ResponseWriter, req *http.Request) {
acceptsContentType := getContentType(req)
writeErrorResponse(w, req, NotImplemented, acceptsContentType, req.URL.Path)
}

8
pkg/api/api_response.go
View File

@@ -112,6 +112,14 @@ func generateListObjectsResponse(bucket string, objects []drivers.ObjectMetadata
return data
}
func generateInitiateMultipartUploadResult(bucket, key, uploadID string) InitiateMultipartUploadResult {
return InitiateMultipartUploadResult{
Bucket: bucket,
Key: key,
UploadID: uploadID,
}
}
// writeSuccessResponse - write success headers
func writeSuccessResponse(w http.ResponseWriter, acceptsContentType contentType) {
setCommonHeaders(w, getContentTypeString(acceptsContentType))

7
pkg/api/api_router.go
View File

@@ -24,6 +24,7 @@ import (
"github.com/minio-io/minio/pkg/api/config"
"github.com/minio-io/minio/pkg/api/logging"
"github.com/minio-io/minio/pkg/api/quota"
"github.com/minio-io/minio/pkg/featureflags"
"github.com/minio-io/minio/pkg/iodine"
"github.com/minio-io/minio/pkg/storage/drivers"
)
@@ -46,6 +47,12 @@ func HTTPHandler(driver drivers.Driver) http.Handler {
mux.HandleFunc("/{bucket}", api.headBucketHandler).Methods("HEAD")
mux.HandleFunc("/{bucket}/{object:.*}", api.getObjectHandler).Methods("GET")
mux.HandleFunc("/{bucket}/{object:.*}", api.headObjectHandler).Methods("HEAD")
if featureflags.Get(featureflags.MultipartPutObject) {
log.Println("Enabling feature", featureflags.MultipartPutObject)
mux.HandleFunc("/{bucket}/{object:.*}?uploads", api.newMultipartUploadHandler).Methods("POST")
mux.HandleFunc("/{bucket}/{object:.*}", api.putObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}").Methods("PUT")
mux.HandleFunc("/{bucket}/{object:.*}", api.completeMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}").Methods("POST")
}
mux.HandleFunc("/{bucket}/{object:.*}", api.putObjectHandler).Methods("PUT")
var conf = config.Config{}