MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 22:25:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

CompleteMultipartUpload and CreateObjectPart now fully support signature v4

Browse Source
This commit is contained in:
Harshavardhana 2015-07-09 19:01:15 -07:00
parent 89c1215194
commit 00890c254e
11 changed files with 156 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pkg/donut/definitions.go
View File

@ -87,6 +87,24 @@ type PartMetadata struct {
Size int64
}
// CompletePart - completed part container
type CompletePart struct {
PartNumber int
ETag string
}
// completedParts is a sortable interface for Part slice
type completedParts []CompletePart
func (a completedParts) Len() int { return len(a) }
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
// CompleteMultipartUpload container for completing multipart upload
type CompleteMultipartUpload struct {
Part []CompletePart
}
// ObjectResourcesMetadata - various types of object resources
type ObjectResourcesMetadata struct {
Bucket string

5
pkg/donut/donut-v2.go
View File

@ -371,11 +371,6 @@ func (donut API) createObject(bucket, key, contentType, expectedMD5Sum string, s
var length int
byteBuffer := make([]byte, 1024*1024)
length, err = data.Read(byteBuffer)
// While hash.Write() wouldn't mind a Nil byteBuffer
// It is necessary for us to verify this and break
if length == 0 {
break
}
hash.Write(byteBuffer[0:length])
sha256hash.Write(byteBuffer[0:length])
ok := donut.objects.Append(objectKey, byteBuffer[0:length])

16
pkg/donut/errors.go
View File

@ -334,3 +334,19 @@ type MissingDateHeader struct{}
func (e MissingDateHeader) Error() string {
return "Missing date header"
}
// InvalidPartOrder parts are not ordered as Requested
type InvalidPartOrder struct {
UploadID string
}
func (e InvalidPartOrder) Error() string {
return "Invalid part order sent for " + e.UploadID
}
// MalformedXML invalid xml format
type MalformedXML struct{}
func (e MalformedXML) Error() string {
return "Malformed XML"
}

4
pkg/donut/interfaces.go
View File

@ -51,8 +51,8 @@ type ObjectStorage interface {
type Multipart interface {
NewMultipartUpload(bucket, key, contentType string) (string, error)
AbortMultipartUpload(bucket, key, uploadID string) error
CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error)
CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (ObjectMetadata, error)
CreateObjectPart(string, string, string, int, string, string, int64, io.Reader, *Signature) (string, error)
CompleteMultipartUpload(bucket, key, uploadID string, data io.Reader, signature *Signature) (ObjectMetadata, error)
ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, error)
ListObjectParts(bucket, key string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, error)
}

60
pkg/donut/multipart.go
View File

@ -22,8 +22,10 @@ import (
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"errors"
"io"
"io/ioutil"
"math/rand"
"runtime/debug"
"sort"
@ -31,6 +33,7 @@ import (
"strings"
"time"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/donut/cache/data"
"github.com/minio/minio/pkg/iodine"
)
@ -90,11 +93,11 @@ func (donut API) AbortMultipartUpload(bucket, key, uploadID string) error {
}
// CreateObjectPart - create a part in a multipart session
func (donut API) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
func (donut API) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (string, error) {
donut.lock.Lock()
defer donut.lock.Unlock()
etag, err := donut.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data)
etag, err := donut.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data, signature)
// possible free
debug.FreeOSMemory()
@ -102,7 +105,7 @@ func (donut API) CreateObjectPart(bucket, key, uploadID string, partID int, cont
}
// createObject - internal wrapper function called by CreateObjectPart
func (donut API) createObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) {
func (donut API) createObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (string, error) {
if !IsValidBucket(bucket) {
return "", iodine.New(BucketNameInvalid{Bucket: bucket}, nil)
}
@ -138,6 +141,7 @@ func (donut API) createObjectPart(bucket, key, uploadID string, partID int, cont
// calculate md5
hash := md5.New()
sha256hash := sha256.New()
var err error
var totalLength int64
@ -145,12 +149,8 @@ func (donut API) createObjectPart(bucket, key, uploadID string, partID int, cont
var length int
byteBuffer := make([]byte, 1024*1024)
length, err = data.Read(byteBuffer)
// While hash.Write() wouldn't mind a Nil byteBuffer
// It is necessary for us to verify this and break
if length == 0 {
break
}
hash.Write(byteBuffer[0:length])
sha256hash.Write(byteBuffer[0:length])
ok := donut.multiPartObjects[uploadID].Append(partID, byteBuffer[0:length])
if !ok {
return "", iodine.New(InternalError{}, nil)
@ -174,6 +174,17 @@ func (donut API) createObjectPart(bucket, key, uploadID string, partID int, cont
return "", iodine.New(BadDigest{}, nil)
}
}
if signature != nil {
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256hash.Sum(nil)))
if err != nil {
return "", iodine.New(err, nil)
}
if !ok {
return "", iodine.New(SignatureDoesNotMatch{}, nil)
}
}
newPart := PartMetadata{
PartNumber: partID,
LastModified: time.Now().UTC(),
@ -200,7 +211,7 @@ func (donut API) cleanupMultipartSession(bucket, key, uploadID string) {
}
// CompleteMultipartUpload - complete a multipart upload and persist the data
func (donut API) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (ObjectMetadata, error) {
func (donut API) CompleteMultipartUpload(bucket, key, uploadID string, data io.Reader, signature *Signature) (ObjectMetadata, error) {
donut.lock.Lock()
if !IsValidBucket(bucket) {
@ -221,11 +232,36 @@ func (donut API) CompleteMultipartUpload(bucket, key, uploadID string, parts map
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(InvalidUploadID{UploadID: uploadID}, nil)
}
partBytes, err := ioutil.ReadAll(data)
if err != nil {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(err, nil)
}
if signature != nil {
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256.Sum256(partBytes)[:]))
if err != nil {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(err, nil)
}
if !ok {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(SignatureDoesNotMatch{}, nil)
}
}
parts := &CompleteMultipartUpload{}
if err := xml.Unmarshal(partBytes, parts); err != nil {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(MalformedXML{}, nil)
}
if !sort.IsSorted(completedParts(parts.Part)) {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(InvalidPartOrder{}, nil)
}
var size int64
var fullObject bytes.Buffer
for i := 1; i <= len(parts); i++ {
recvMD5 := parts[i]
object, ok := donut.multiPartObjects[uploadID].Get(i)
for i := 1; i <= len(parts.Part); i++ {
recvMD5 := parts.Part[i-1].ETag
object, ok := donut.multiPartObjects[uploadID].Get(parts.Part[i-1].PartNumber)
if ok == false {
donut.lock.Unlock()
return ObjectMetadata{}, iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil)

2
pkg/server/api/bucket-handlers.go
View File

@ -100,7 +100,7 @@ func (api Minio) ListMultipartUploadsHandler(w http.ResponseWriter, req *http.Re
case nil: // success
{
// generate response
response := generateListMultipartUploadsResult(bucket, resources)
response := generateListMultipartUploadsResponse(bucket, resources)
encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
// write headers
setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse))

36
pkg/server/api/definitions.go
View File

@ -62,6 +62,14 @@ type ListObjectsResponse struct {
Prefix string
}
// Part container for part metadata
type Part struct {
PartNumber int
ETag string
LastModified string
Size int64
}
// ListPartsResponse - format for list parts response
type ListPartsResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
@ -134,14 +142,6 @@ type Bucket struct {
CreationDate string
}
// Part container for part metadata
type Part struct {
PartNumber int
ETag string
LastModified string
Size int64
}
// Object container for object metadata
type Object struct {
ETag string
@ -164,8 +164,8 @@ type Owner struct {
DisplayName string
}
// InitiateMultipartUploadResult container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
type InitiateMultipartUploadResult struct {
// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
type InitiateMultipartUploadResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
Bucket string
@ -173,20 +173,8 @@ type InitiateMultipartUploadResult struct {
UploadID string `xml:"UploadId"`
}
// completedParts is a sortable interface for Part slice
type completedParts []Part
func (a completedParts) Len() int { return len(a) }
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
// CompleteMultipartUpload container for completing multipart upload
type CompleteMultipartUpload struct {
Part []Part
}
// CompleteMultipartUploadResult container for completed multipart upload response
type CompleteMultipartUploadResult struct {
// CompleteMultipartUploadResponse container for completed multipart upload response
type CompleteMultipartUploadResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
Location string

62
pkg/server/api/object-handlers.go
View File

@ -18,11 +18,8 @@ package api
import (
"net/http"
"sort"
"strconv"
"encoding/xml"
"github.com/gorilla/mux"
"github.com/minio/minio/pkg/donut"
"github.com/minio/minio/pkg/iodine"
@ -277,7 +274,7 @@ func (api Minio) NewMultipartUploadHandler(w http.ResponseWriter, req *http.Requ
switch iodine.ToError(err).(type) {
case nil:
{
response := generateInitiateMultipartUploadResult(bucket, object, uploadID)
response := generateInitiateMultipartUploadResponse(bucket, object, uploadID)
encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
// write headers
setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse))
@ -346,7 +343,18 @@ func (api Minio) PutObjectPartHandler(w http.ResponseWriter, req *http.Request)
writeErrorResponse(w, req, InvalidPart, acceptsContentType, req.URL.Path)
}
calculatedMD5, err := api.Donut.CreateObjectPart(bucket, object, uploadID, partID, "", md5, sizeInt64, req.Body)
var signature *donut.Signature
if _, ok := req.Header["Authorization"]; ok {
// Init signature V4 verification
var err error
signature, err = InitSignatureV4(req)
if err != nil {
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
return
}
}
calculatedMD5, err := api.Donut.CreateObjectPart(bucket, object, uploadID, partID, "", md5, sizeInt64, req.Body, signature)
switch iodine.ToError(err).(type) {
case nil:
w.Header().Set("ETag", calculatedMD5)
@ -357,6 +365,8 @@ func (api Minio) PutObjectPartHandler(w http.ResponseWriter, req *http.Request)
writeErrorResponse(w, req, MethodNotAllowed, acceptsContentType, req.URL.Path)
case donut.BadDigest:
writeErrorResponse(w, req, BadDigest, acceptsContentType, req.URL.Path)
case donut.SignatureDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, acceptsContentType, req.URL.Path)
case donut.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, acceptsContentType, req.URL.Path)
case donut.EntityTooLarge:
@ -433,7 +443,7 @@ func (api Minio) ListObjectPartsHandler(w http.ResponseWriter, req *http.Request
switch iodine.ToError(err).(type) {
case nil:
{
response := generateListPartsResult(objectResourcesMetadata)
response := generateListPartsResponse(objectResourcesMetadata)
encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
// write headers
setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse))
@ -464,35 +474,27 @@ func (api Minio) CompleteMultipartUploadHandler(w http.ResponseWriter, req *http
return
}
decoder := xml.NewDecoder(req.Body)
parts := &CompleteMultipartUpload{}
err := decoder.Decode(parts)
if err != nil {
log.Error.Println(iodine.New(err, nil))
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
return
}
if !sort.IsSorted(completedParts(parts.Part)) {
writeErrorResponse(w, req, InvalidPartOrder, acceptsContentType, req.URL.Path)
return
}
vars := mux.Vars(req)
bucket := vars["bucket"]
object := vars["object"]
objectResourcesMetadata := getObjectResources(req.URL.Query())
partMap := make(map[int]string)
for _, part := range parts.Part {
partMap[part.PartNumber] = part.ETag
var signature *donut.Signature
if _, ok := req.Header["Authorization"]; ok {
// Init signature V4 verification
var err error
signature, err = InitSignatureV4(req)
if err != nil {
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)
return
}
}
metadata, err := api.Donut.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, partMap)
metadata, err := api.Donut.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, req.Body, signature)
switch iodine.ToError(err).(type) {
case nil:
{
response := generateCompleteMultpartUploadResult(bucket, object, "", metadata.MD5Sum)
response := generateCompleteMultpartUploadResponse(bucket, object, "", metadata.MD5Sum)
encodedSuccessResponse := encodeSuccessResponse(response, acceptsContentType)
// write headers
setCommonHeaders(w, getContentTypeString(acceptsContentType), len(encodedSuccessResponse))
@ -501,6 +503,16 @@ func (api Minio) CompleteMultipartUploadHandler(w http.ResponseWriter, req *http
}
case donut.InvalidUploadID:
writeErrorResponse(w, req, NoSuchUpload, acceptsContentType, req.URL.Path)
case donut.InvalidPartOrder:
writeErrorResponse(w, req, InvalidPartOrder, acceptsContentType, req.URL.Path)
case donut.MissingDateHeader:
writeErrorResponse(w, req, RequestTimeTooSkewed, acceptsContentType, req.URL.Path)
case donut.SignatureDoesNotMatch:
writeErrorResponse(w, req, SignatureDoesNotMatch, acceptsContentType, req.URL.Path)
case donut.IncompleteBody:
writeErrorResponse(w, req, IncompleteBody, acceptsContentType, req.URL.Path)
case donut.MalformedXML:
writeErrorResponse(w, req, MalformedXML, acceptsContentType, req.URL.Path)
default:
log.Error.Println(iodine.New(err, nil))
writeErrorResponse(w, req, InternalError, acceptsContentType, req.URL.Path)

18
pkg/server/api/response.go
View File

@ -111,18 +111,18 @@ func generateListObjectsResponse(bucket string, objects []donut.ObjectMetadata,
return data
}
// generateInitiateMultipartUploadResult
func generateInitiateMultipartUploadResult(bucket, key, uploadID string) InitiateMultipartUploadResult {
return InitiateMultipartUploadResult{
// generateInitiateMultipartUploadResponse
func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse {
return InitiateMultipartUploadResponse{
Bucket: bucket,
Key: key,
UploadID: uploadID,
}
}
// generateCompleteMultipartUploadResult
func generateCompleteMultpartUploadResult(bucket, key, location, etag string) CompleteMultipartUploadResult {
return CompleteMultipartUploadResult{
// generateCompleteMultipartUploadResponse
func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
return CompleteMultipartUploadResponse{
Location: location,
Bucket: bucket,
Key: key,
@ -131,7 +131,7 @@ func generateCompleteMultpartUploadResult(bucket, key, location, etag string) Co
}
// generateListPartsResult
func generateListPartsResult(objectMetadata donut.ObjectResourcesMetadata) ListPartsResponse {
func generateListPartsResponse(objectMetadata donut.ObjectResourcesMetadata) ListPartsResponse {
// TODO - support EncodingType in xml decoding
listPartsResponse := ListPartsResponse{}
listPartsResponse.Bucket = objectMetadata.Bucket
@ -160,8 +160,8 @@ func generateListPartsResult(objectMetadata donut.ObjectResourcesMetadata) ListP
return listPartsResponse
}
// generateListMultipartUploadsResult
func generateListMultipartUploadsResult(bucket string, metadata donut.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse {
// generateListMultipartUploadsResponse
func generateListMultipartUploadsResponse(bucket string, metadata donut.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse {
listMultipartUploadsResponse := ListMultipartUploadsResponse{}
listMultipartUploadsResponse.Bucket = bucket
listMultipartUploadsResponse.Delimiter = metadata.Delimiter

13
pkg/server/api_donut_cache_test.go
View File

@ -27,6 +27,7 @@ import (
"net/http/httptest"
. "github.com/minio/check"
"github.com/minio/minio/pkg/donut"
"github.com/minio/minio/pkg/server/api"
)
@ -627,7 +628,7 @@ func (s *MyAPIDonutCacheSuite) TestObjectMultipartAbort(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -672,7 +673,7 @@ func (s *MyAPIDonutCacheSuite) TestBucketMultipartList(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -723,7 +724,7 @@ func (s *MyAPIDonutCacheSuite) TestObjectMultipartList(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -771,7 +772,7 @@ func (s *MyAPIDonutCacheSuite) TestObjectMultipart(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -795,8 +796,8 @@ func (s *MyAPIDonutCacheSuite) TestObjectMultipart(c *C) {
c.Assert(response2.StatusCode, Equals, http.StatusOK)
// complete multipart upload
completeUploads := &api.CompleteMultipartUpload{
Part: []api.Part{
completeUploads := &donut.CompleteMultipartUpload{
Part: []donut.CompletePart{
{
PartNumber: 1,
ETag: response1.Header.Get("ETag"),

12
pkg/server/api_donut_test.go
View File

@ -664,7 +664,7 @@ func (s *MyAPIDonutSuite) TestObjectMultipartAbort(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -709,7 +709,7 @@ func (s *MyAPIDonutSuite) TestBucketMultipartList(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -760,7 +760,7 @@ func (s *MyAPIDonutSuite) TestObjectMultipartList(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -808,7 +808,7 @@ func (s *MyAPIDonutSuite) TestObjectMultipart(c *C) {
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
newResponse := &api.InitiateMultipartUploadResult{}
newResponse := &api.InitiateMultipartUploadResponse{}
err = decoder.Decode(newResponse)
c.Assert(err, IsNil)
@ -832,8 +832,8 @@ func (s *MyAPIDonutSuite) TestObjectMultipart(c *C) {
c.Assert(response2.StatusCode, Equals, http.StatusOK)
// complete multipart upload
completeUploads := &api.CompleteMultipartUpload{
Part: []api.Part{
completeUploads := &donut.CompleteMultipartUpload{
Part: []donut.CompletePart{
{
PartNumber: 1,
ETag: response1.Header.Get("ETag"),