Simplify signature handling

This change brings a new SignatureHandler where Presigned.
Requests without Payload are handled very early before even
going through the call.

This change simplifies Donut codebase to not have signature related
logic for all API's.

Simplification is still needed for Payload based signature eg. PUT/POST calls
, which are still part of the donut codebase, which will be done subsequently
after donut re-write.
This commit is contained in:
Harshavardhana
2015-10-04 12:06:14 -07:00
parent 3de10f9472
commit cfdb29cac0
13 changed files with 156 additions and 353 deletions

View File

@@ -31,6 +31,7 @@ var fsType2StringMap = map[string]string{
"6969": "NFS",
"ef51": "EXT2OLD",
"ef53": "EXT4",
"f15f": "ecryptfs",
}
// getFSType - get filesystem type

View File

@@ -619,10 +619,11 @@ func (donut API) makeDonutBucket(bucketName, acl string) *probe.Error {
}
nodeNumber = nodeNumber + 1
}
metadata, err := donut.getDonutBucketMetadata()
var metadata *AllBuckets
metadata, err = donut.getDonutBucketMetadata()
if err != nil {
if os.IsNotExist(err.ToGoError()) {
metadata := new(AllBuckets)
metadata = new(AllBuckets)
metadata.Buckets = make(map[string]BucketMetadata)
metadata.Buckets[bucketName] = bucketMetadata
err = donut.setDonutBucketMetadata(metadata)

View File

@@ -74,7 +74,7 @@ func (s *MyDonutSuite) SetUpSuite(c *C) {
c.Assert(perr, IsNil)
// testing empty donut
buckets, perr := dd.ListBuckets(nil)
buckets, perr := dd.ListBuckets()
c.Assert(perr, IsNil)
c.Assert(len(buckets), Equals, 0)
}
@@ -99,7 +99,7 @@ func (s *MyDonutSuite) TestEmptyBucket(c *C) {
// check if bucket is empty
var resources BucketResourcesMetadata
resources.Maxkeys = 1
objectsMetadata, resources, err := dd.ListObjects("foo1", resources, nil)
objectsMetadata, resources, err := dd.ListObjects("foo1", resources)
c.Assert(err, IsNil)
c.Assert(len(objectsMetadata), Equals, 0)
c.Assert(resources.CommonPrefixes, DeepEquals, []string{})
@@ -113,7 +113,7 @@ func (s *MyDonutSuite) TestMakeBucketAndList(c *C) {
c.Assert(err, IsNil)
// check bucket exists
buckets, err := dd.ListBuckets(nil)
buckets, err := dd.ListBuckets()
c.Assert(err, IsNil)
c.Assert(len(buckets), Equals, 5)
c.Assert(buckets[0].ACL, Equals, BucketACL("private"))
@@ -137,7 +137,7 @@ func (s *MyDonutSuite) TestCreateMultipleBucketsAndList(c *C) {
err = dd.MakeBucket("bar1", "private", nil, nil)
c.Assert(err, IsNil)
buckets, err := dd.ListBuckets(nil)
buckets, err := dd.ListBuckets()
c.Assert(err, IsNil)
c.Assert(len(buckets), Equals, 2)
@@ -147,7 +147,7 @@ func (s *MyDonutSuite) TestCreateMultipleBucketsAndList(c *C) {
err = dd.MakeBucket("foobar1", "private", nil, nil)
c.Assert(err, IsNil)
buckets, err = dd.ListBuckets(nil)
buckets, err = dd.ListBuckets()
c.Assert(err, IsNil)
c.Assert(len(buckets), Equals, 3)
@@ -205,7 +205,7 @@ func (s *MyDonutSuite) TestNewObjectCanBeWritten(c *C) {
c.Assert(size, Equals, int64(len(data)))
c.Assert(buffer.Bytes(), DeepEquals, []byte(data))
actualMetadata, err = dd.GetObjectMetadata("foo", "obj", nil)
actualMetadata, err = dd.GetObjectMetadata("foo", "obj")
c.Assert(err, IsNil)
c.Assert(hex.EncodeToString(hasher.Sum(nil)), Equals, actualMetadata.MD5Sum)
c.Assert(int64(len(data)), Equals, actualMetadata.Size)
@@ -244,7 +244,7 @@ func (s *MyDonutSuite) TestMultipleNewObjects(c *C) {
resources.Prefix = "o"
resources.Delimiter = "1"
resources.Maxkeys = 10
objectsMetadata, resources, err := dd.ListObjects("foo5", resources, nil)
objectsMetadata, resources, err := dd.ListObjects("foo5", resources)
c.Assert(err, IsNil)
c.Assert(resources.IsTruncated, Equals, false)
c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
@@ -253,7 +253,7 @@ func (s *MyDonutSuite) TestMultipleNewObjects(c *C) {
resources.Prefix = ""
resources.Delimiter = "1"
resources.Maxkeys = 10
objectsMetadata, resources, err = dd.ListObjects("foo5", resources, nil)
objectsMetadata, resources, err = dd.ListObjects("foo5", resources)
c.Assert(err, IsNil)
c.Assert(objectsMetadata[0].Object, Equals, "obj2")
c.Assert(resources.IsTruncated, Equals, false)
@@ -263,7 +263,7 @@ func (s *MyDonutSuite) TestMultipleNewObjects(c *C) {
resources.Prefix = "o"
resources.Delimiter = ""
resources.Maxkeys = 10
objectsMetadata, resources, err = dd.ListObjects("foo5", resources, nil)
objectsMetadata, resources, err = dd.ListObjects("foo5", resources)
c.Assert(err, IsNil)
c.Assert(resources.IsTruncated, Equals, false)
c.Assert(objectsMetadata[0].Object, Equals, "obj1")
@@ -283,7 +283,7 @@ func (s *MyDonutSuite) TestMultipleNewObjects(c *C) {
resources.Prefix = "o"
resources.Delimiter = ""
resources.Maxkeys = 2
objectsMetadata, resources, err = dd.ListObjects("foo5", resources, nil)
objectsMetadata, resources, err = dd.ListObjects("foo5", resources)
c.Assert(err, IsNil)
c.Assert(resources.IsTruncated, Equals, true)
c.Assert(len(objectsMetadata), Equals, 2)

View File

@@ -205,20 +205,10 @@ func (donut API) GetObject(w io.Writer, bucket string, object string, start, len
}
// GetBucketMetadata -
func (donut API) GetBucketMetadata(bucket string, signature *signv4.Signature) (BucketMetadata, *probe.Error) {
func (donut API) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error) {
donut.lock.Lock()
defer donut.lock.Unlock()
if signature != nil {
ok, err := signature.DoesSignatureMatch("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
return BucketMetadata{}, err.Trace()
}
if !ok {
return BucketMetadata{}, probe.NewError(signv4.DoesNotMatch{})
}
}
if !IsValidBucket(bucket) {
return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
@@ -238,20 +228,10 @@ func (donut API) GetBucketMetadata(bucket string, signature *signv4.Signature) (
}
// SetBucketMetadata -
func (donut API) SetBucketMetadata(bucket string, metadata map[string]string, signature *signv4.Signature) *probe.Error {
func (donut API) SetBucketMetadata(bucket string, metadata map[string]string) *probe.Error {
donut.lock.Lock()
defer donut.lock.Unlock()
if signature != nil {
ok, err := signature.DoesSignatureMatch("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
return err.Trace()
}
if !ok {
return probe.NewError(signv4.DoesNotMatch{})
}
}
if !IsValidBucket(bucket) {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
@@ -487,20 +467,10 @@ func (donut API) MakeBucket(bucketName, acl string, location io.Reader, signatur
}
// ListObjects - list objects from cache
func (donut API) ListObjects(bucket string, resources BucketResourcesMetadata, signature *signv4.Signature) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error) {
func (donut API) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error) {
donut.lock.Lock()
defer donut.lock.Unlock()
if signature != nil {
ok, err := signature.DoesSignatureMatch("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
return nil, BucketResourcesMetadata{}, err.Trace()
}
if !ok {
return nil, BucketResourcesMetadata{}, probe.NewError(signv4.DoesNotMatch{})
}
}
if !IsValidBucket(bucket) {
return nil, BucketResourcesMetadata{IsTruncated: false}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
@@ -590,20 +560,10 @@ func (b byBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
// ListBuckets - List buckets from cache
func (donut API) ListBuckets(signature *signv4.Signature) ([]BucketMetadata, *probe.Error) {
func (donut API) ListBuckets() ([]BucketMetadata, *probe.Error) {
donut.lock.Lock()
defer donut.lock.Unlock()
if signature != nil {
ok, err := signature.DoesSignatureMatch("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
return nil, err.Trace()
}
if !ok {
return nil, probe.NewError(signv4.DoesNotMatch{})
}
}
var results []BucketMetadata
if len(donut.config.NodeDiskMap) > 0 {
buckets, err := donut.listBuckets()
@@ -624,30 +584,10 @@ func (donut API) ListBuckets(signature *signv4.Signature) ([]BucketMetadata, *pr
}
// GetObjectMetadata - get object metadata from cache
func (donut API) GetObjectMetadata(bucket, key string, signature *signv4.Signature) (ObjectMetadata, *probe.Error) {
func (donut API) GetObjectMetadata(bucket, key string) (ObjectMetadata, *probe.Error) {
donut.lock.Lock()
defer donut.lock.Unlock()
if signature != nil {
if signature.Presigned {
ok, err := signature.DoesPresignedSignatureMatch()
if err != nil {
return ObjectMetadata{}, err.Trace()
}
if !ok {
return ObjectMetadata{}, probe.NewError(signv4.DoesNotMatch{})
}
} else {
ok, err := signature.DoesSignatureMatch("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
return ObjectMetadata{}, err.Trace()
}
if !ok {
return ObjectMetadata{}, probe.NewError(signv4.DoesNotMatch{})
}
}
}
// check if bucket exists
if !IsValidBucket(bucket) {
return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})

View File

@@ -49,7 +49,7 @@ func (s *MyCacheSuite) SetUpSuite(c *C) {
// testing empty cache
var buckets []BucketMetadata
buckets, perr := dc.ListBuckets(nil)
buckets, perr := dc.ListBuckets()
c.Assert(perr, IsNil)
c.Assert(len(buckets), Equals, 0)
}
@@ -74,7 +74,7 @@ func (s *MyCacheSuite) TestEmptyBucket(c *C) {
// check if bucket is empty
var resources BucketResourcesMetadata
resources.Maxkeys = 1
objectsMetadata, resources, err := dc.ListObjects("foo1", resources, nil)
objectsMetadata, resources, err := dc.ListObjects("foo1", resources)
c.Assert(err, IsNil)
c.Assert(len(objectsMetadata), Equals, 0)
c.Assert(resources.CommonPrefixes, DeepEquals, []string{})
@@ -88,7 +88,7 @@ func (s *MyCacheSuite) TestMakeBucketAndList(c *C) {
c.Assert(err, IsNil)
// check bucket exists
buckets, err := dc.ListBuckets(nil)
buckets, err := dc.ListBuckets()
c.Assert(err, IsNil)
c.Assert(len(buckets), Equals, 5)
c.Assert(buckets[0].ACL, Equals, BucketACL("private"))
@@ -112,7 +112,7 @@ func (s *MyCacheSuite) TestCreateMultipleBucketsAndList(c *C) {
err = dc.MakeBucket("bar1", "private", nil, nil)
c.Assert(err, IsNil)
buckets, err := dc.ListBuckets(nil)
buckets, err := dc.ListBuckets()
c.Assert(err, IsNil)
c.Assert(len(buckets), Equals, 2)
@@ -122,7 +122,7 @@ func (s *MyCacheSuite) TestCreateMultipleBucketsAndList(c *C) {
err = dc.MakeBucket("foobar1", "private", nil, nil)
c.Assert(err, IsNil)
buckets, err = dc.ListBuckets(nil)
buckets, err = dc.ListBuckets()
c.Assert(err, IsNil)
c.Assert(len(buckets), Equals, 3)
@@ -180,7 +180,7 @@ func (s *MyCacheSuite) TestNewObjectCanBeWritten(c *C) {
c.Assert(size, Equals, int64(len(data)))
c.Assert(buffer.Bytes(), DeepEquals, []byte(data))
actualMetadata, err = dc.GetObjectMetadata("foo", "obj", nil)
actualMetadata, err = dc.GetObjectMetadata("foo", "obj")
c.Assert(err, IsNil)
c.Assert(hex.EncodeToString(hasher.Sum(nil)), Equals, actualMetadata.MD5Sum)
c.Assert(int64(len(data)), Equals, actualMetadata.Size)
@@ -219,7 +219,7 @@ func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {
resources.Prefix = "o"
resources.Delimiter = "1"
resources.Maxkeys = 10
objectsMetadata, resources, err := dc.ListObjects("foo5", resources, nil)
objectsMetadata, resources, err := dc.ListObjects("foo5", resources)
c.Assert(err, IsNil)
c.Assert(resources.IsTruncated, Equals, false)
c.Assert(resources.CommonPrefixes[0], Equals, "obj1")
@@ -228,7 +228,7 @@ func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {
resources.Prefix = ""
resources.Delimiter = "1"
resources.Maxkeys = 10
objectsMetadata, resources, err = dc.ListObjects("foo5", resources, nil)
objectsMetadata, resources, err = dc.ListObjects("foo5", resources)
c.Assert(err, IsNil)
c.Assert(objectsMetadata[0].Object, Equals, "obj2")
c.Assert(resources.IsTruncated, Equals, false)
@@ -238,7 +238,7 @@ func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {
resources.Prefix = "o"
resources.Delimiter = ""
resources.Maxkeys = 10
objectsMetadata, resources, err = dc.ListObjects("foo5", resources, nil)
objectsMetadata, resources, err = dc.ListObjects("foo5", resources)
c.Assert(err, IsNil)
c.Assert(resources.IsTruncated, Equals, false)
c.Assert(objectsMetadata[0].Object, Equals, "obj1")
@@ -258,7 +258,7 @@ func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {
resources.Prefix = "o"
resources.Delimiter = ""
resources.Maxkeys = 2
objectsMetadata, resources, err = dc.ListObjects("foo5", resources, nil)
objectsMetadata, resources, err = dc.ListObjects("foo5", resources)
c.Assert(err, IsNil)
c.Assert(resources.IsTruncated, Equals, true)
c.Assert(len(objectsMetadata), Equals, 2)

View File

@@ -34,17 +34,17 @@ type Interface interface {
// CloudStorage is a donut cloud storage interface
type CloudStorage interface {
// Storage service operations
GetBucketMetadata(bucket string, signature *signv4.Signature) (BucketMetadata, *probe.Error)
SetBucketMetadata(bucket string, metadata map[string]string, signature *signv4.Signature) *probe.Error
ListBuckets(signature *signv4.Signature) ([]BucketMetadata, *probe.Error)
GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error)
SetBucketMetadata(bucket string, metadata map[string]string) *probe.Error
ListBuckets() ([]BucketMetadata, *probe.Error)
MakeBucket(bucket string, ACL string, location io.Reader, signature *signv4.Signature) *probe.Error
// Bucket operations
ListObjects(string, BucketResourcesMetadata, *signv4.Signature) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error)
ListObjects(string, BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error)
// Object operations
GetObject(w io.Writer, bucket, object string, start, length int64) (int64, *probe.Error)
GetObjectMetadata(bucket, object string, signature *signv4.Signature) (ObjectMetadata, *probe.Error)
GetObjectMetadata(bucket, object string) (ObjectMetadata, *probe.Error)
// bucket, object, expectedMD5Sum, size, reader, metadata, signature
CreateObject(string, string, string, int64, io.Reader, map[string]string, *signv4.Signature) (ObjectMetadata, *probe.Error)
@@ -53,12 +53,12 @@ type CloudStorage interface {
// Multipart API
type Multipart interface {
NewMultipartUpload(bucket, key, contentType string, signature *signv4.Signature) (string, *probe.Error)
AbortMultipartUpload(bucket, key, uploadID string, signature *signv4.Signature) *probe.Error
NewMultipartUpload(bucket, key, contentType string) (string, *probe.Error)
AbortMultipartUpload(bucket, key, uploadID string) *probe.Error
CreateObjectPart(string, string, string, int, string, string, int64, io.Reader, *signv4.Signature) (string, *probe.Error)
CompleteMultipartUpload(bucket, key, uploadID string, data io.Reader, signature *signv4.Signature) (ObjectMetadata, *probe.Error)
ListMultipartUploads(string, BucketMultipartResourcesMetadata, *signv4.Signature) (BucketMultipartResourcesMetadata, *probe.Error)
ListObjectParts(string, string, ObjectResourcesMetadata, *signv4.Signature) (ObjectResourcesMetadata, *probe.Error)
ListMultipartUploads(string, BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error)
ListObjectParts(string, string, ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error)
}
// Management is a donut management system interface

View File

@@ -41,7 +41,7 @@ import (
/// V2 API functions
// NewMultipartUpload - initiate a new multipart session
func (donut API) NewMultipartUpload(bucket, key, contentType string, signature *signv4.Signature) (string, *probe.Error) {
func (donut API) NewMultipartUpload(bucket, key, contentType string) (string, *probe.Error) {
donut.lock.Lock()
defer donut.lock.Unlock()
@@ -51,15 +51,6 @@ func (donut API) NewMultipartUpload(bucket, key, contentType string, signature *
if !IsValidObjectName(key) {
return "", probe.NewError(ObjectNameInvalid{Object: key})
}
if signature != nil {
ok, err := signature.DoesSignatureMatch("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
return "", err.Trace()
}
if !ok {
return "", probe.NewError(signv4.DoesNotMatch{})
}
}
// if len(donut.config.NodeDiskMap) > 0 {
// return donut.newMultipartUpload(bucket, key, contentType)
// }
@@ -89,7 +80,7 @@ func (donut API) NewMultipartUpload(bucket, key, contentType string, signature *
}
// AbortMultipartUpload - abort an incomplete multipart session
func (donut API) AbortMultipartUpload(bucket, key, uploadID string, signature *signv4.Signature) *probe.Error {
func (donut API) AbortMultipartUpload(bucket, key, uploadID string) *probe.Error {
donut.lock.Lock()
defer donut.lock.Unlock()
@@ -99,15 +90,6 @@ func (donut API) AbortMultipartUpload(bucket, key, uploadID string, signature *s
if !IsValidObjectName(key) {
return probe.NewError(ObjectNameInvalid{Object: key})
}
if signature != nil {
ok, err := signature.DoesSignatureMatch("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
return err.Trace()
}
if !ok {
return probe.NewError(signv4.DoesNotMatch{})
}
}
// TODO: multipart support for donut is broken, since we haven't finalized the format in which
// it can be stored, disabling this for now until we get the underlying layout stable.
//
@@ -381,21 +363,11 @@ func (a byKey) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byKey) Less(i, j int) bool { return a[i].Key < a[j].Key }
// ListMultipartUploads - list incomplete multipart sessions for a given bucket
func (donut API) ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata, signature *signv4.Signature) (BucketMultipartResourcesMetadata, *probe.Error) {
func (donut API) ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error) {
// TODO handle delimiter, low priority
donut.lock.Lock()
defer donut.lock.Unlock()
if signature != nil {
ok, err := signature.DoesSignatureMatch("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
return BucketMultipartResourcesMetadata{}, err.Trace()
}
if !ok {
return BucketMultipartResourcesMetadata{}, probe.NewError(signv4.DoesNotMatch{})
}
}
if !IsValidBucket(bucket) {
return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
@@ -466,21 +438,11 @@ func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
// ListObjectParts - list parts from incomplete multipart session for a given object
func (donut API) ListObjectParts(bucket, key string, resources ObjectResourcesMetadata, signature *signv4.Signature) (ObjectResourcesMetadata, *probe.Error) {
func (donut API) ListObjectParts(bucket, key string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error) {
// Verify upload id
donut.lock.Lock()
defer donut.lock.Unlock()
if signature != nil {
ok, err := signature.DoesSignatureMatch("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
if err != nil {
return ObjectResourcesMetadata{}, err.Trace()
}
if !ok {
return ObjectResourcesMetadata{}, probe.NewError(signv4.DoesNotMatch{})
}
}
if !IsValidBucket(bucket) {
return ObjectResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}