PutObject handler gets initial support for signature v4, working

2025-11-07 12:52:58 -05:00 · 2015-07-09 14:42:04 -07:00
parent 4f29dc9134
commit 89c1215194
12 changed files with 196 additions and 324 deletions
--- a/pkg/donut/donut-v1_test.go
+++ b/pkg/donut/donut-v1_test.go
@@ -156,7 +156,7 @@ func (s *MyDonutSuite) TestCreateMultipleBucketsAndList(c *C) {

 // test object create without bucket
 func (s *MyDonutSuite) TestNewObjectFailsWithoutBucket(c *C) {
-	_, err := dd.CreateObject("unknown", "obj", "", 0, nil, nil)
+	_, err := dd.CreateObject("unknown", "obj", "", 0, nil, nil, nil)
 	c.Assert(err, Not(IsNil))
 }

@@ -171,7 +171,7 @@ func (s *MyDonutSuite) TestNewObjectMetadata(c *C) {
 	err := dd.MakeBucket("foo6", "private")
 	c.Assert(err, IsNil)

-	objectMetadata, err := dd.CreateObject("foo6", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/json"})
+	objectMetadata, err := dd.CreateObject("foo6", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/json"}, nil)
 	c.Assert(err, IsNil)
 	c.Assert(objectMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
 	c.Assert(objectMetadata.Metadata["contentType"], Equals, "application/json")
@@ -179,7 +179,7 @@ func (s *MyDonutSuite) TestNewObjectMetadata(c *C) {

 // test create object fails without name
 func (s *MyDonutSuite) TestNewObjectFailsWithEmptyName(c *C) {
-	_, err := dd.CreateObject("foo", "", "", 0, nil, nil)
+	_, err := dd.CreateObject("foo", "", "", 0, nil, nil, nil)
 	c.Assert(err, Not(IsNil))
 }

@@ -195,7 +195,7 @@ func (s *MyDonutSuite) TestNewObjectCanBeWritten(c *C) {
 	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
 	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))

-	actualMetadata, err := dd.CreateObject("foo", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/octet-stream"})
+	actualMetadata, err := dd.CreateObject("foo", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/octet-stream"}, nil)
 	c.Assert(err, IsNil)
 	c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))

@@ -217,11 +217,11 @@ func (s *MyDonutSuite) TestMultipleNewObjects(c *C) {

 	one := ioutil.NopCloser(bytes.NewReader([]byte("one")))

-	_, err := dd.CreateObject("foo5", "obj1", "", int64(len("one")), one, nil)
+	_, err := dd.CreateObject("foo5", "obj1", "", int64(len("one")), one, nil, nil)
 	c.Assert(err, IsNil)

 	two := ioutil.NopCloser(bytes.NewReader([]byte("two")))
-	_, err = dd.CreateObject("foo5", "obj2", "", int64(len("two")), two, nil)
+	_, err = dd.CreateObject("foo5", "obj2", "", int64(len("two")), two, nil, nil)
 	c.Assert(err, IsNil)

 	var buffer1 bytes.Buffer
@@ -270,7 +270,7 @@ func (s *MyDonutSuite) TestMultipleNewObjects(c *C) {
 	c.Assert(objectsMetadata[1].Object, Equals, "obj2")

 	three := ioutil.NopCloser(bytes.NewReader([]byte("three")))
-	_, err = dd.CreateObject("foo5", "obj3", "", int64(len("three")), three, nil)
+	_, err = dd.CreateObject("foo5", "obj3", "", int64(len("three")), three, nil, nil)
 	c.Assert(err, IsNil)

 	var buffer bytes.Buffer
--- a/pkg/donut/donut-v2.go
+++ b/pkg/donut/donut-v2.go
@@ -25,7 +25,6 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
-	"net/http"
 	"runtime/debug"
 	"sort"
 	"strconv"
@@ -33,6 +32,7 @@ import (
 	"sync"
 	"time"

+	"github.com/minio/minio/pkg/crypto/sha256"
 	"github.com/minio/minio/pkg/donut/cache/data"
 	"github.com/minio/minio/pkg/donut/cache/metadata"
 	"github.com/minio/minio/pkg/iodine"
@@ -55,7 +55,6 @@ type Config struct {
 // API - local variables
 type API struct {
 	config           *Config
-	req              *http.Request
 	lock             *sync.Mutex
 	objects          *data.Cache
 	multiPartObjects map[string]*data.Cache
@@ -124,11 +123,6 @@ func New() (Interface, error) {
 	return a, nil
 }

-// SetRequest API for setting request header
-func (donut API) SetRequest(req *http.Request) {
-	donut.req = req
-}
-
 // GetObject - GET object from cache buffer
 func (donut API) GetObject(w io.Writer, bucket string, object string) (int64, error) {
 	donut.lock.Lock()
@@ -296,12 +290,12 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error {
 }

 // CreateObject - create an object
-func (donut API) CreateObject(bucket, key, expectedMD5Sum string, size int64, data io.Reader, metadata map[string]string) (ObjectMetadata, error) {
+func (donut API) CreateObject(bucket, key, expectedMD5Sum string, size int64, data io.Reader, metadata map[string]string, signature *Signature) (ObjectMetadata, error) {
 	donut.lock.Lock()
 	defer donut.lock.Unlock()

 	contentType := metadata["contentType"]
-	objectMetadata, err := donut.createObject(bucket, key, contentType, expectedMD5Sum, size, data)
+	objectMetadata, err := donut.createObject(bucket, key, contentType, expectedMD5Sum, size, data, signature)
 	// free
 	debug.FreeOSMemory()

@@ -309,7 +303,7 @@ func (donut API) CreateObject(bucket, key, expectedMD5Sum string, size int64, da
 }

 // createObject - PUT object to cache buffer
-func (donut API) createObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (ObjectMetadata, error) {
+func (donut API) createObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (ObjectMetadata, error) {
 	if len(donut.config.NodeDiskMap) == 0 {
 		if size > int64(donut.config.MaxSize) {
 			generic := GenericObjectError{Bucket: bucket, Object: key}
@@ -369,6 +363,7 @@ func (donut API) createObject(bucket, key, contentType, expectedMD5Sum string, s
 	}
 	// calculate md5
 	hash := md5.New()
+	sha256hash := sha256.New()

 	var err error
 	var totalLength int64
@@ -382,6 +377,7 @@ func (donut API) createObject(bucket, key, contentType, expectedMD5Sum string, s
 			break
 		}
 		hash.Write(byteBuffer[0:length])
+		sha256hash.Write(byteBuffer[0:length])
 		ok := donut.objects.Append(objectKey, byteBuffer[0:length])
 		if !ok {
 			return ObjectMetadata{}, iodine.New(InternalError{}, nil)
@@ -405,6 +401,15 @@ func (donut API) createObject(bucket, key, contentType, expectedMD5Sum string, s
 			return ObjectMetadata{}, iodine.New(BadDigest{}, nil)
 		}
 	}
+	if signature != nil {
+		ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256hash.Sum(nil)))
+		if err != nil {
+			return ObjectMetadata{}, iodine.New(err, nil)
+		}
+		if !ok {
+			return ObjectMetadata{}, iodine.New(SignatureDoesNotMatch{}, nil)
+		}
+	}

 	m := make(map[string]string)
 	m["contentType"] = contentType
--- a/pkg/donut/donut-v2_test.go
+++ b/pkg/donut/donut-v2_test.go
@@ -131,7 +131,7 @@ func (s *MyCacheSuite) TestCreateMultipleBucketsAndList(c *C) {

 // test object create without bucket
 func (s *MyCacheSuite) TestNewObjectFailsWithoutBucket(c *C) {
-	_, err := dc.CreateObject("unknown", "obj", "", 0, nil, nil)
+	_, err := dc.CreateObject("unknown", "obj", "", 0, nil, nil, nil)
 	c.Assert(err, Not(IsNil))
 }

@@ -146,7 +146,7 @@ func (s *MyCacheSuite) TestNewObjectMetadata(c *C) {
 	err := dc.MakeBucket("foo6", "private")
 	c.Assert(err, IsNil)

-	objectMetadata, err := dc.CreateObject("foo6", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/json"})
+	objectMetadata, err := dc.CreateObject("foo6", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/json"}, nil)
 	c.Assert(err, IsNil)
 	c.Assert(objectMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))
 	c.Assert(objectMetadata.Metadata["contentType"], Equals, "application/json")
@@ -154,7 +154,7 @@ func (s *MyCacheSuite) TestNewObjectMetadata(c *C) {

 // test create object fails without name
 func (s *MyCacheSuite) TestNewObjectFailsWithEmptyName(c *C) {
-	_, err := dc.CreateObject("foo", "", "", 0, nil, nil)
+	_, err := dc.CreateObject("foo", "", "", 0, nil, nil, nil)
 	c.Assert(err, Not(IsNil))
 }

@@ -170,7 +170,7 @@ func (s *MyCacheSuite) TestNewObjectCanBeWritten(c *C) {
 	expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
 	reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))

-	actualMetadata, err := dc.CreateObject("foo", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/octet-stream"})
+	actualMetadata, err := dc.CreateObject("foo", "obj", expectedMd5Sum, int64(len(data)), reader, map[string]string{"contentType": "application/octet-stream"}, nil)
 	c.Assert(err, IsNil)
 	c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil)))

@@ -192,11 +192,11 @@ func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {

 	one := ioutil.NopCloser(bytes.NewReader([]byte("one")))

-	_, err := dc.CreateObject("foo5", "obj1", "", int64(len("one")), one, nil)
+	_, err := dc.CreateObject("foo5", "obj1", "", int64(len("one")), one, nil, nil)
 	c.Assert(err, IsNil)

 	two := ioutil.NopCloser(bytes.NewReader([]byte("two")))
-	_, err = dc.CreateObject("foo5", "obj2", "", int64(len("two")), two, nil)
+	_, err = dc.CreateObject("foo5", "obj2", "", int64(len("two")), two, nil, nil)
 	c.Assert(err, IsNil)

 	var buffer1 bytes.Buffer
@@ -245,7 +245,7 @@ func (s *MyCacheSuite) TestMultipleNewObjects(c *C) {
 	c.Assert(objectsMetadata[1].Object, Equals, "obj2")

 	three := ioutil.NopCloser(bytes.NewReader([]byte("three")))
-	_, err = dc.CreateObject("foo5", "obj3", "", int64(len("three")), three, nil)
+	_, err = dc.CreateObject("foo5", "obj3", "", int64(len("three")), three, nil, nil)
 	c.Assert(err, IsNil)

 	var buffer bytes.Buffer
--- a/pkg/donut/interfaces.go
+++ b/pkg/donut/interfaces.go
@@ -41,7 +41,8 @@ type ObjectStorage interface {
 	GetObject(w io.Writer, bucket, object string) (int64, error)
 	GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error)
 	GetObjectMetadata(bucket, object string) (ObjectMetadata, error)
-	CreateObject(bucket, object, expectedMD5Sum string, size int64, reader io.Reader, metadata map[string]string) (ObjectMetadata, error)
+	// bucket, object, expectedMD5Sum, size, reader, metadata, signature
+	CreateObject(string, string, string, int64, io.Reader, map[string]string, *Signature) (ObjectMetadata, error)

 	Multipart
 }
--- a/pkg/donut/multipart.go
+++ b/pkg/donut/multipart.go
@@ -255,7 +255,7 @@ func (donut API) CompleteMultipartUpload(bucket, key, uploadID string, parts map
 	// this is needed for final verification inside CreateObject, do not convert this to hex
 	md5sum := base64.StdEncoding.EncodeToString(md5sumSlice[:])
 	donut.lock.Unlock()
-	objectMetadata, err := donut.CreateObject(bucket, key, md5sum, size, &fullObject, nil)
+	objectMetadata, err := donut.CreateObject(bucket, key, md5sum, size, &fullObject, nil, nil)
 	if err != nil {
 		// No need to call internal cleanup functions here, caller will call AbortMultipartUpload()
 		// which would in-turn cleanup properly in accordance with S3 Spec
--- a/pkg/donut/signature-v4.go
+++ b/pkg/donut/signature-v4.go
@@ -21,7 +21,6 @@ import (
 	"crypto/hmac"
 	"encoding/hex"
 	"errors"
-	"io"
 	"net/http"
 	"regexp"
 	"sort"
@@ -29,30 +28,30 @@ import (
 	"time"
 	"unicode/utf8"

-	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/crypto/sha256"
 	"github.com/minio/minio/pkg/iodine"
 )

-// request - a http request
-type request struct {
-	receivedReq   *http.Request
-	calculatedReq *http.Request
-	user          *auth.User
-	body          io.Reader
+// Signature - local variables
+type Signature struct {
+	AccessKeyID     string
+	SecretAccessKey string
+	AuthHeader      string
+	Request         *http.Request
 }

 const (
-	authHeader    = "AWS4-HMAC-SHA256"
-	iso8601Format = "20060102T150405Z"
-	yyyymmdd      = "20060102"
+	authHeaderPrefix = "AWS4-HMAC-SHA256"
+	iso8601Format    = "20060102T150405Z"
+	yyyymmdd         = "20060102"
 )

 var ignoredHeaders = map[string]bool{
-	"Authorization":  true,
-	"Content-Type":   true,
-	"Content-Length": true,
-	"User-Agent":     true,
+	"Authorization":   true,
+	"Content-Type":    true,
+	"Accept-Encoding": true,
+	"Content-Length":  true,
+	"User-Agent":      true,
 }

 // sumHMAC calculate hmac between two input byte array
@@ -101,38 +100,11 @@ func urlEncodeName(name string) (string, error) {
 	return encodedName, nil
 }

-// newSignV4Request - populate a new signature v4 request
-func newSignV4Request(user *auth.User, req *http.Request, body io.Reader) (*request, error) {
-	// save for subsequent use
-	r := new(request)
-	r.user = user
-	r.body = body
-	r.receivedReq = req
-	r.calculatedReq = req
-	return r, nil
-}
-
-// getHashedPayload get the hexadecimal value of the SHA256 hash of the request payload
-func (r *request) getHashedPayload() string {
-	hash := func() string {
-		switch {
-		case r.body == nil:
-			return hex.EncodeToString(sha256.Sum256([]byte{}))
-		default:
-			sum256Bytes, _ := sha256.Sum(r.body)
-			return hex.EncodeToString(sum256Bytes)
-		}
-	}
-	hashedPayload := hash()
-	r.calculatedReq.Header.Set("x-amz-content-sha256", hashedPayload)
-	return hashedPayload
-}
-
 // getCanonicalHeaders generate a list of request headers with their values
-func (r *request) getCanonicalHeaders() string {
+func (r *Signature) getCanonicalHeaders() string {
 	var headers []string
 	vals := make(map[string][]string)
-	for k, vv := range r.calculatedReq.Header {
+	for k, vv := range r.Request.Header {
 		if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
 			continue // ignored header
 		}
@@ -148,7 +120,7 @@ func (r *request) getCanonicalHeaders() string {
 		buf.WriteByte(':')
 		switch {
 		case k == "host":
-			buf.WriteString(r.calculatedReq.URL.Host)
+			buf.WriteString(r.Request.Host)
 			fallthrough
 		default:
 			for idx, v := range vals[k] {
@@ -164,9 +136,9 @@ func (r *request) getCanonicalHeaders() string {
 }

 // getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names
-func (r *request) getSignedHeaders() string {
+func (r *Signature) getSignedHeaders() string {
 	var headers []string
-	for k := range r.calculatedReq.Header {
+	for k := range r.Request.Header {
 		if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
 			continue // ignored header
 		}
@@ -187,24 +159,24 @@ func (r *request) getSignedHeaders() string {
 //  <SignedHeaders>\n
 //  <HashedPayload>
 //
-func (r *request) getCanonicalRequest(hashedPayload string) string {
-	r.calculatedReq.URL.RawQuery = strings.Replace(r.calculatedReq.URL.Query().Encode(), "+", "%20", -1)
-	encodedPath, _ := urlEncodeName(r.calculatedReq.URL.Path)
+func (r *Signature) getCanonicalRequest() string {
+	r.Request.URL.RawQuery = strings.Replace(r.Request.URL.Query().Encode(), "+", "%20", -1)
+	encodedPath, _ := urlEncodeName(r.Request.URL.Path)
 	// convert any space strings back to "+"
 	encodedPath = strings.Replace(encodedPath, "+", "%20", -1)
 	canonicalRequest := strings.Join([]string{
-		r.calculatedReq.Method,
+		r.Request.Method,
 		encodedPath,
-		r.calculatedReq.URL.RawQuery,
+		r.Request.URL.RawQuery,
 		r.getCanonicalHeaders(),
 		r.getSignedHeaders(),
-		hashedPayload,
+		r.Request.Header.Get("x-amz-content-sha256"),
 	}, "\n")
 	return canonicalRequest
 }

 // getScope generate a string of a specific date, an AWS region, and a service
-func (r *request) getScope(t time.Time) string {
+func (r *Signature) getScope(t time.Time) string {
 	scope := strings.Join([]string{
 		t.Format(yyyymmdd),
 		"milkyway",
@@ -215,16 +187,16 @@ func (r *request) getScope(t time.Time) string {
 }

 // getStringToSign a string based on selected query values
-func (r *request) getStringToSign(canonicalRequest string, t time.Time) string {
-	stringToSign := authHeader + "\n" + t.Format(iso8601Format) + "\n"
+func (r *Signature) getStringToSign(canonicalRequest string, t time.Time) string {
+	stringToSign := authHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n"
 	stringToSign = stringToSign + r.getScope(t) + "\n"
 	stringToSign = stringToSign + hex.EncodeToString(sha256.Sum256([]byte(canonicalRequest)))
 	return stringToSign
 }

 // getSigningKey hmac seed to calculate final signature
-func (r *request) getSigningKey(t time.Time) []byte {
-	secret := r.user.SecretAccessKey
+func (r *Signature) getSigningKey(t time.Time) []byte {
+	secret := r.SecretAccessKey
 	date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
 	region := sumHMAC(date, []byte("milkyway"))
 	service := sumHMAC(region, []byte("s3"))
@@ -233,26 +205,29 @@ func (r *request) getSigningKey(t time.Time) []byte {
 }

 // getSignature final signature in hexadecimal form
-func (r *request) getSignature(signingKey []byte, stringToSign string) string {
+func (r *Signature) getSignature(signingKey []byte, stringToSign string) string {
 	return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
 }

-// SignV4 the request before Do(), in accordance with - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
-func (r *request) SignV4() (string, error) {
+// DoesSignatureMatch - Verify authorization header with calculated header in accordance with - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
+// returns true if matches, false other wise if error is not nil then it is always false
+func (r *Signature) DoesSignatureMatch(hashedPayload string) (bool, error) {
+	// set new calulated payload
+	r.Request.Header.Set("x-amz-content-sha256", hashedPayload)
+
 	// Add date if not present
 	var date string
-	if date = r.calculatedReq.Header.Get("x-amz-date"); date == "" {
-		if date = r.calculatedReq.Header.Get("Date"); date == "" {
-			return "", iodine.New(MissingDateHeader{}, nil)
+	if date = r.Request.Header.Get("x-amz-date"); date == "" {
+		if date = r.Request.Header.Get("Date"); date == "" {
+			return false, iodine.New(MissingDateHeader{}, nil)
 		}
 	}
 	t, err := time.Parse(iso8601Format, date)
 	if err != nil {
-		return "", iodine.New(err, nil)
+		return false, iodine.New(err, nil)
 	}
-	hashedPayload := r.getHashedPayload()
 	signedHeaders := r.getSignedHeaders()
-	canonicalRequest := r.getCanonicalRequest(hashedPayload)
+	canonicalRequest := r.getCanonicalRequest()
 	scope := r.getScope(t)
 	stringToSign := r.getStringToSign(canonicalRequest, t)
 	signingKey := r.getSigningKey(t)
@@ -260,10 +235,13 @@ func (r *request) SignV4() (string, error) {

 	// final Authorization header
 	parts := []string{
-		authHeader + " Credential=" + r.user.AccessKeyID + "/" + scope,
+		authHeaderPrefix + " Credential=" + r.AccessKeyID + "/" + scope,
 		"SignedHeaders=" + signedHeaders,
 		"Signature=" + signature,
 	}
-	auth := strings.Join(parts, ", ")
-	return auth, nil
+	newAuthHeader := strings.Join(parts, ", ")
+	if newAuthHeader != r.AuthHeader {
+		return false, nil
+	}
+	return true, nil
 }