signature: Use a layered approach for signature verification.

Signature calculation has now moved out from being a package to
top-level as a layered mechanism.

In case of payload calculation with body, go-routines are initiated
to simultaneously write and calculate shasum. Errors are sent
over the writer so that the lower layer removes the temporary files
properly.
This commit is contained in:
Harshavardhana
2016-03-12 16:08:15 -08:00
parent 1b0bc814c4
commit 9dca46e156
39 changed files with 572 additions and 739 deletions

View File

@@ -19,7 +19,6 @@ package fs
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"math/rand"
@@ -61,7 +60,7 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
c.Assert(err, check.IsNil)
completedParts := CompleteMultipartUpload{}
completedParts.Part = make([]CompletePart, 0)
//completedParts.Part = make([]CompletePart, 10)
for i := 1; i <= 10; i++ {
randomPerm := rand.Perm(10)
randomString := ""
@@ -71,19 +70,17 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
hasher := md5.New()
hasher.Write([]byte(randomString))
expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
var calculatedmd5sum string
calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
bytes.NewBufferString(randomString), nil)
var calculatedMD5sum string
calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil))
c.Assert(err, check.IsNil)
c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedmd5sum})
c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedMD5sum})
}
completedPartsBytes, e := xml.Marshal(completedParts)
c.Assert(e, check.IsNil)
objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil)
objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedPartsBytes)
c.Assert(err, check.IsNil)
c.Assert(objectInfo.MD5Sum, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10")
}
@@ -105,15 +102,13 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
hasher := md5.New()
hasher.Write([]byte(randomString))
expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
var calculatedmd5sum string
calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
bytes.NewBufferString(randomString), nil)
var calculatedMD5sum string
calculatedMD5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil))
c.Assert(err, check.IsNil)
c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
parts[i] = calculatedmd5sum
c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
parts[i] = expectedMD5Sumhex
}
err = fs.AbortMultipartUpload("bucket", "key", uploadID)
c.Assert(err, check.IsNil)
@@ -133,14 +128,14 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
hasher := md5.New()
hasher.Write([]byte(randomString))
expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
key := "obj" + strconv.Itoa(i)
objects[key] = []byte(randomString)
objectInfo, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil)
var objectInfo ObjectInfo
objectInfo, err = fs.CreateObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), hasher.Sum(nil))
c.Assert(err, check.IsNil)
c.Assert(objectInfo.MD5Sum, check.Equals, expectedmd5Sumhex)
c.Assert(objectInfo.MD5Sum, check.Equals, expectedMD5Sumhex)
}
for key, value := range objects {
@@ -165,7 +160,7 @@ func testPaging(c *check.C, create func() Filesystem) {
// check before paging occurs
for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i)
_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
_, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "", "", "", 5)
c.Assert(err, check.IsNil)
@@ -175,7 +170,7 @@ func testPaging(c *check.C, create func() Filesystem) {
// check after paging occurs pages work
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
_, err = fs.CreateObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "obj", "", "", 5)
c.Assert(err, check.IsNil)
@@ -184,9 +179,9 @@ func testPaging(c *check.C, create func() Filesystem) {
}
// check paging with prefix at end returns less objects
{
_, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
_, err = fs.CreateObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
_, err = fs.CreateObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "new", "", "", 5)
c.Assert(err, check.IsNil)
@@ -206,9 +201,9 @@ func testPaging(c *check.C, create func() Filesystem) {
// check delimited results with delimiter and prefix
{
_, err = fs.CreateObject("bucket", "this/is/delimited", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
_, err = fs.CreateObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
_, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
c.Assert(err, check.IsNil)
@@ -260,18 +255,11 @@ func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
hasher1 := md5.New()
hasher1.Write([]byte("one"))
md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
objectInfo, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil)
_, err = fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
c.Assert(err, check.IsNil)
c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum)
// c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum)
hasher2 := md5.New()
hasher2.Write([]byte("three"))
md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil))
_, err = fs.CreateObject("bucket", "object", md5Sum2, int64(len("three")), bytes.NewBufferString("three"), nil)
_, err = fs.CreateObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil)
c.Assert(err, check.IsNil)
var bytesBuffer bytes.Buffer
@@ -283,7 +271,7 @@ func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
fs := create()
_, err := fs.CreateObject("bucket", "object", "", int64(len("one")), bytes.NewBufferString("one"), nil)
_, err := fs.CreateObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
c.Assert(err, check.Not(check.IsNil))
}
@@ -300,13 +288,8 @@ func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
hasher := md5.New()
hasher.Write([]byte("hello world"))
md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
objectInfo, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
_, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
c.Assert(err, check.IsNil)
c.Assert(objectInfo.MD5Sum, check.Equals, md5Sum1hex)
var bytesBuffer bytes.Buffer
length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0)
@@ -396,7 +379,7 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem)
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
_, err = fs.CreateObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
c.Assert(err, check.IsNil)
var byteBuffer bytes.Buffer
@@ -431,26 +414,9 @@ func testDefaultContentType(c *check.C, create func() Filesystem) {
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
// test empty
_, err = fs.CreateObject("bucket", "one", "", int64(len("one")), bytes.NewBufferString("one"), nil)
// Test empty
_, err = fs.CreateObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil)
metadata, err := fs.GetObjectInfo("bucket", "one")
c.Assert(err, check.IsNil)
c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
}
func testContentMD5Set(c *check.C, create func() Filesystem) {
fs := create()
err := fs.MakeBucket("bucket")
c.Assert(err, check.IsNil)
// test md5 invalid
badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA"
calculatedmd5sum, err := fs.CreateObject("bucket", "one", badmd5Sum, int64(len("one")), bytes.NewBufferString("one"), nil)
c.Assert(err, check.Not(check.IsNil))
c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum)
goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA="
calculatedmd5sum, err = fs.CreateObject("bucket", "two", goodmd5sum, int64(len("one")), bytes.NewBufferString("one"), nil)
c.Assert(err, check.IsNil)
c.Assert(calculatedmd5sum, check.Equals, goodmd5sum)
}

View File

@@ -27,7 +27,6 @@ import (
)
func TestListObjects(t *testing.T) {
// Make a temporary directory to use as the filesystem.
directory, e := ioutil.TempDir("", "minio-list-object-test")
if e != nil {
@@ -58,36 +57,36 @@ func TestListObjects(t *testing.T) {
}
defer os.Remove(tmpfile.Name()) // clean up
_, err = fs.CreateObject("test-bucket-list-object", "Asia-maps", "", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil)
_, err = fs.CreateObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil)
if err != nil {
t.Fatal(e)
}
_, err = fs.CreateObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", "", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
_, err = fs.CreateObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
if err != nil {
t.Fatal(e)
}
_, err = fs.CreateObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
_, err = fs.CreateObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
if err != nil {
t.Fatal(e)
}
for i := 0; i < 2; i++ {
key := "newPrefix" + strconv.Itoa(i)
_, err = fs.CreateObject("test-bucket-list-object", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
_, err = fs.CreateObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil {
t.Fatal(err)
}
}
_, err = fs.CreateObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", "", int64(len("recurse")), bytes.NewBufferString("recurse"), nil)
_, err = fs.CreateObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil)
if err != nil {
t.Fatal(e)
}
for i := 0; i < 3; i++ {
key := "obj" + strconv.Itoa(i)
_, err = fs.CreateObject("test-bucket-list-object", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
_, err = fs.CreateObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil {
t.Fatal(err)
}
@@ -592,7 +591,7 @@ func BenchmarkListObjects(b *testing.B) {
for i := 0; i < 20000; i++ {
key := "obj" + strconv.Itoa(i)
_, err = filesystem.CreateObject("ls-benchmark-bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
_, err = filesystem.CreateObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil {
b.Fatal(err)
}

View File

@@ -18,13 +18,6 @@ package fs
import "fmt"
// SignDoesNotMatch - signature does not match.
type SignDoesNotMatch struct{}
func (e SignDoesNotMatch) Error() string {
return "Signature does not match."
}
// InvalidArgument invalid argument
type InvalidArgument struct{}
@@ -131,15 +124,14 @@ func (e InvalidDisksArgument) Error() string {
return "Invalid number of disks per node"
}
// BadDigest bad md5sum
// BadDigest - Content-MD5 you specified did not match what we received.
type BadDigest struct {
MD5 string
Bucket string
Object string
ExpectedMD5 string
CalculatedMD5 string
}
func (e BadDigest) Error() string {
return "Bad digest"
return "Bad digest expected " + e.ExpectedMD5 + " is not valid with what we calculated " + e.CalculatedMD5
}
// InternalError - generic internal error
@@ -183,13 +175,6 @@ type ImplementationError struct {
Err error
}
// DigestError - Generic MD5 error
type DigestError struct {
Bucket string
Key string
MD5 string
}
/// Bucket related errors
// BucketNameInvalid - bucketname provided is invalid
@@ -200,9 +185,6 @@ type BucketNameInvalid GenericBucketError
// ObjectNameInvalid - object name provided is invalid
type ObjectNameInvalid GenericObjectError
// InvalidDigest - md5 in request header invalid
type InvalidDigest DigestError
// Return string an error formatted as the given text
func (e ImplementationError) Error() string {
error := ""
@@ -258,11 +240,6 @@ func (e BackendCorrupted) Error() string {
return "Backend corrupted: " + e.Path
}
// Return string an error formatted as the given text
func (e InvalidDigest) Error() string {
return "MD5 provided " + e.MD5 + " is invalid"
}
// OperationNotPermitted - operation not permitted
type OperationNotPermitted struct {
Op string

View File

@@ -24,7 +24,6 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
@@ -34,12 +33,10 @@ import (
"time"
"github.com/minio/minio/pkg/atomic"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/crypto/sha512"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/mimedb"
"github.com/minio/minio/pkg/probe"
"github.com/minio/minio/pkg/s3/signature4"
)
// isValidUploadID - is upload id.
@@ -322,7 +319,7 @@ func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
// CreateObjectPart - create a part in a multipart session
func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum string, partID int, size int64, data io.Reader, signature *signature4.Sign) (string, *probe.Error) {
func (fs Filesystem) CreateObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Bytes []byte) (string, *probe.Error) {
di, err := disk.GetInfo(fs.path)
if err != nil {
return "", probe.NewError(err)
@@ -355,71 +352,58 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s
return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
}
if expectedMD5Sum != "" {
var expectedMD5SumBytes []byte
expectedMD5SumBytes, err = base64.StdEncoding.DecodeString(expectedMD5Sum)
if err != nil {
// Pro-actively close the connection
return "", probe.NewError(InvalidDigest{MD5: expectedMD5Sum})
}
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
}
bucket = fs.denormalizeBucket(bucket)
bucketPath := filepath.Join(fs.path, bucket)
if _, err = os.Stat(bucketPath); err != nil {
if _, e := os.Stat(bucketPath); e != nil {
// Check bucket exists.
if os.IsNotExist(err) {
if os.IsNotExist(e) {
return "", probe.NewError(BucketNotFound{Bucket: bucket})
}
return "", probe.NewError(err)
return "", probe.NewError(e)
}
// md5Hex representation.
var md5Hex string
if len(md5Bytes) != 0 {
md5Hex = hex.EncodeToString(md5Bytes)
}
objectPath := filepath.Join(bucketPath, object)
partPathPrefix := objectPath + uploadID
partPath := partPathPrefix + expectedMD5Sum + fmt.Sprintf("$%d-$multiparts", partID)
partPath := partPathPrefix + md5Hex + fmt.Sprintf("$%d-$multiparts", partID)
partFile, e := atomic.FileCreateWithPrefix(partPath, "$multiparts")
if e != nil {
return "", probe.NewError(e)
}
defer partFile.Close()
md5Hasher := md5.New()
sha256Hasher := sha256.New()
partWriter := io.MultiWriter(partFile, md5Hasher, sha256Hasher)
if _, e = io.CopyN(partWriter, data, size); e != nil {
// Initialize md5 writer.
md5Writer := md5.New()
// Create a multiwriter.
multiWriter := io.MultiWriter(md5Writer, partFile)
if _, e = io.CopyN(multiWriter, data, size); e != nil {
partFile.CloseAndPurge()
return "", probe.NewError(e)
}
md5sum := hex.EncodeToString(md5Hasher.Sum(nil))
// Verify if the written object is equal to what is expected, only
// if it is requested as such.
if expectedMD5Sum != "" {
if !isMD5SumEqual(expectedMD5Sum, md5sum) {
partFile.CloseAndPurge()
return "", probe.NewError(BadDigest{MD5: expectedMD5Sum, Bucket: bucket, Object: object})
// Finalize new md5.
newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
if len(md5Bytes) != 0 {
if newMD5Hex != md5Hex {
return "", probe.NewError(BadDigest{md5Hex, newMD5Hex})
}
}
if signature != nil {
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256Hasher.Sum(nil)))
if err != nil {
partFile.CloseAndPurge()
return "", err.Trace()
}
if !ok {
partFile.CloseAndPurge()
return "", probe.NewError(SignDoesNotMatch{})
}
}
partFile.Close()
fi, e := os.Stat(partPath)
// Stat the file to get the latest information.
fi, e := os.Stat(partFile.Name())
if e != nil {
return "", probe.NewError(e)
}
partMetadata := PartMetadata{}
partMetadata.ETag = md5sum
partMetadata.PartNumber = partID
partMetadata.ETag = newMD5Hex
partMetadata.Size = fi.Size()
partMetadata.LastModified = fi.ModTime()
@@ -450,13 +434,11 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s
if err := saveMultipartsSession(*fs.multiparts); err != nil {
return "", err.Trace(partPathPrefix)
}
// Return etag.
return partMetadata.ETag, nil
return newMD5Hex, nil
}
// CompleteMultipartUpload - complete a multipart upload and persist the data
func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, data io.Reader, signature *signature4.Sign) (ObjectInfo, *probe.Error) {
func (fs Filesystem) CompleteMultipartUpload(bucket string, object string, uploadID string, completeMultipartBytes []byte) (ObjectInfo, *probe.Error) {
// Check bucket name is valid.
if !IsValidBucketName(bucket) {
return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
@@ -488,26 +470,8 @@ func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, da
return ObjectInfo{}, probe.NewError(e)
}
partBytes, e := ioutil.ReadAll(data)
if e != nil {
objectWriter.CloseAndPurge()
return ObjectInfo{}, probe.NewError(e)
}
if signature != nil {
sh := sha256.New()
sh.Write(partBytes)
ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
if err != nil {
objectWriter.CloseAndPurge()
return ObjectInfo{}, err.Trace()
}
if !ok {
objectWriter.CloseAndPurge()
return ObjectInfo{}, probe.NewError(SignDoesNotMatch{})
}
}
completeMultipartUpload := &CompleteMultipartUpload{}
if e = xml.Unmarshal(partBytes, completeMultipartUpload); e != nil {
if e = xml.Unmarshal(completeMultipartBytes, completeMultipartUpload); e != nil {
objectWriter.CloseAndPurge()
return ObjectInfo{}, probe.NewError(MalformedXML{})
}

View File

@@ -18,22 +18,19 @@ package fs
import (
"bytes"
"crypto/md5"
"io"
"os"
"path/filepath"
"strings"
"crypto/md5"
"encoding/base64"
"encoding/hex"
"runtime"
"github.com/minio/minio/pkg/atomic"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/mimedb"
"github.com/minio/minio/pkg/probe"
"github.com/minio/minio/pkg/s3/signature4"
)
/// Object Operations
@@ -200,7 +197,7 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
}
// CreateObject - create an object.
func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size int64, data io.Reader, sig *signature4.Sign) (ObjectInfo, *probe.Error) {
func (fs Filesystem) CreateObject(bucket string, object string, size int64, data io.Reader, md5Bytes []byte) (ObjectInfo, *probe.Error) {
di, e := disk.GetInfo(fs.path)
if e != nil {
return ObjectInfo{}, probe.NewError(e)
@@ -234,18 +231,15 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in
// Get object path.
objectPath := filepath.Join(bucketPath, object)
if expectedMD5Sum != "" {
var expectedMD5SumBytes []byte
expectedMD5SumBytes, e = base64.StdEncoding.DecodeString(expectedMD5Sum)
if e != nil {
// Pro-actively close the connection.
return ObjectInfo{}, probe.NewError(InvalidDigest{MD5: expectedMD5Sum})
}
expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
// md5Hex representation.
var md5Hex string
if len(md5Bytes) != 0 {
md5Hex = hex.EncodeToString(md5Bytes)
}
// Write object.
file, e := atomic.FileCreateWithPrefix(objectPath, expectedMD5Sum+"$tmpobject")
file, e := atomic.FileCreateWithPrefix(objectPath, md5Hex+"$tmpobject")
if e != nil {
switch e := e.(type) {
case *os.PathError:
@@ -259,52 +253,40 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in
return ObjectInfo{}, probe.NewError(e)
}
}
defer file.Close()
// Initialize md5 writer.
md5Writer := md5.New()
// Instantiate a new multi writer.
multiWriter := io.MultiWriter(md5Writer, file)
// Instantiate checksum hashers and create a multiwriter.
md5Hasher := md5.New()
sha256Hasher := sha256.New()
objectWriter := io.MultiWriter(file, md5Hasher, sha256Hasher)
if size > 0 {
if _, e = io.CopyN(objectWriter, data, size); e != nil {
if _, e = io.CopyN(multiWriter, data, size); e != nil {
file.CloseAndPurge()
return ObjectInfo{}, probe.NewError(e)
}
} else {
if _, e = io.Copy(objectWriter, data); e != nil {
if _, e = io.Copy(multiWriter, data); e != nil {
file.CloseAndPurge()
return ObjectInfo{}, probe.NewError(e)
}
}
md5Sum := hex.EncodeToString(md5Hasher.Sum(nil))
// Verify if the written object is equal to what is expected, only
// if it is requested as such.
if expectedMD5Sum != "" {
if !isMD5SumEqual(expectedMD5Sum, md5Sum) {
file.CloseAndPurge()
return ObjectInfo{}, probe.NewError(BadDigest{MD5: expectedMD5Sum, Bucket: bucket, Object: object})
newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
if len(md5Bytes) != 0 {
if newMD5Hex != md5Hex {
return ObjectInfo{}, probe.NewError(BadDigest{md5Hex, newMD5Hex})
}
}
sha256Sum := hex.EncodeToString(sha256Hasher.Sum(nil))
if sig != nil {
ok, err := sig.DoesSignatureMatch(sha256Sum)
if err != nil {
file.CloseAndPurge()
return ObjectInfo{}, err.Trace()
}
if !ok {
file.CloseAndPurge()
return ObjectInfo{}, probe.NewError(SignDoesNotMatch{})
}
}
file.Close()
// Set stat again to get the latest metadata.
st, e := os.Stat(objectPath)
st, e := os.Stat(file.Name())
if e != nil {
return ObjectInfo{}, probe.NewError(e)
}
contentType := "application/octet-stream"
if objectExt := filepath.Ext(objectPath); objectExt != "" {
content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
@@ -317,8 +299,8 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in
Name: object,
ModifiedTime: st.ModTime(),
Size: st.Size(),
MD5Sum: newMD5Hex,
ContentType: contentType,
MD5Sum: md5Sum,
}
return newObject, nil
}

View File

@@ -19,7 +19,6 @@ package fs
import (
"bytes"
"crypto/md5"
"encoding/base64"
"fmt"
"io/ioutil"
"os"
@@ -47,7 +46,7 @@ func TestGetObjectInfo(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = fs.CreateObject("test-getobjectinfo", "Asia/asiapics.jpg", "", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
_, err = fs.CreateObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
if err != nil {
t.Fatal(err)
}
@@ -138,7 +137,7 @@ func TestGetObjectInfoCore(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = fs.CreateObject("test-getobjinfo", "Asia/asiapics.jpg", "", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
_, err = fs.CreateObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
if err != nil {
t.Fatal(err)
}
@@ -228,16 +227,14 @@ func BenchmarkGetObject(b *testing.B) {
text := "Jack and Jill went up the hill / To fetch a pail of water."
hasher := md5.New()
hasher.Write([]byte(text))
sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
for i := 0; i < 10; i++ {
_, err = filesystem.CreateObject("bucket", "object"+strconv.Itoa(i), sum, int64(len(text)), bytes.NewBufferString(text), nil)
_, err = filesystem.CreateObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), hasher.Sum(nil))
if err != nil {
b.Fatal(err)
}
}
var w bytes.Buffer
b.ResetTimer()
for i := 0; i < b.N; i++ {

View File

@@ -1,57 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package signature4
import (
"fmt"
"github.com/minio/minio/pkg/probe"
)
type errFunc func(msg string, a ...string) *probe.Error
// generic error factory which wraps around probe.NewError()
func errFactory() errFunc {
return func(msg string, a ...string) *probe.Error {
return probe.NewError(fmt.Errorf("%s, Args: %s", msg, a)).Untrace()
}
}
// Various signature v4 errors.
var (
ErrPolicyAlreadyExpired = errFactory()
ErrInvalidRegion = errFactory()
ErrInvalidDateFormat = errFactory()
ErrInvalidService = errFactory()
ErrInvalidRequestVersion = errFactory()
ErrMissingFields = errFactory()
ErrMissingCredTag = errFactory()
ErrCredMalformed = errFactory()
ErrMissingSignTag = errFactory()
ErrMissingSignHeadersTag = errFactory()
ErrMissingDateHeader = errFactory()
ErrMalformedDate = errFactory()
ErrMalformedExpires = errFactory()
ErrAuthHeaderEmpty = errFactory()
ErrUnsuppSignAlgo = errFactory()
ErrMissingExpiresQuery = errFactory()
ErrExpiredPresignRequest = errFactory()
ErrSignDoesNotMath = errFactory()
ErrInvalidAccessKeyID = errFactory()
ErrInvalidSecretKey = errFactory()
ErrRegionISEmpty = errFactory()
)

View File

@@ -1,222 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package signature4
import (
"net/url"
"strings"
"time"
"github.com/minio/minio/pkg/probe"
)
// credential data type represents structured form of Credential
// string from authorization header.
type credential struct {
accessKeyID string
scope struct {
date time.Time
region string
service string
request string
}
}
// parse credential string into its structured form.
func parseCredential(credElement string) (credential, *probe.Error) {
creds := strings.Split(strings.TrimSpace(credElement), "=")
if len(creds) != 2 {
return credential{}, ErrMissingFields("Credential tag has missing fields.", credElement).Trace(credElement)
}
if creds[0] != "Credential" {
return credential{}, ErrMissingCredTag("Missing credentials tag.", credElement).Trace(credElement)
}
credElements := strings.Split(strings.TrimSpace(creds[1]), "/")
if len(credElements) != 5 {
return credential{}, ErrCredMalformed("Credential values malformed.", credElement).Trace(credElement)
}
if !isValidAccessKey.MatchString(credElements[0]) {
return credential{}, ErrInvalidAccessKeyID("Invalid access key id.", credElement).Trace(credElement)
}
cred := credential{
accessKeyID: credElements[0],
}
var e error
cred.scope.date, e = time.Parse(yyyymmdd, credElements[1])
if e != nil {
return credential{}, ErrInvalidDateFormat("Invalid date format.", credElement).Trace(credElement)
}
if credElements[2] == "" {
return credential{}, ErrRegionISEmpty("Region is empty.", credElement).Trace(credElement)
}
cred.scope.region = credElements[2]
if credElements[3] != "s3" {
return credential{}, ErrInvalidService("Invalid service detected.", credElement).Trace(credElement)
}
cred.scope.service = credElements[3]
if credElements[4] != "aws4_request" {
return credential{}, ErrInvalidRequestVersion("Invalid request version detected.", credElement).Trace(credElement)
}
cred.scope.request = credElements[4]
return cred, nil
}
// Parse signature string.
func parseSignature(signElement string) (string, *probe.Error) {
signFields := strings.Split(strings.TrimSpace(signElement), "=")
if len(signFields) != 2 {
return "", ErrMissingFields("Signature tag has missing fields.", signElement).Trace(signElement)
}
if signFields[0] != "Signature" {
return "", ErrMissingSignTag("Signature tag is missing", signElement).Trace(signElement)
}
signature := signFields[1]
return signature, nil
}
// Parse signed headers string.
func parseSignedHeaders(signedHdrElement string) ([]string, *probe.Error) {
signedHdrFields := strings.Split(strings.TrimSpace(signedHdrElement), "=")
if len(signedHdrFields) != 2 {
return nil, ErrMissingFields("Signed headers tag has missing fields.", signedHdrElement).Trace(signedHdrElement)
}
if signedHdrFields[0] != "SignedHeaders" {
return nil, ErrMissingSignHeadersTag("Signed headers tag is missing.", signedHdrElement).Trace(signedHdrElement)
}
signedHeaders := strings.Split(signedHdrFields[1], ";")
return signedHeaders, nil
}
// signValues data type represents structured form of AWS Signature V4 header.
type signValues struct {
Credential credential
SignedHeaders []string
Signature string
}
// preSignValues data type represents structued form of AWS Signature V4 query string.
type preSignValues struct {
signValues
Date time.Time
Expires time.Duration
}
// Parses signature version '4' query string of the following form.
//
// querystring = X-Amz-Algorithm=algorithm
// querystring += &X-Amz-Credential= urlencode(access_key_ID + '/' + credential_scope)
// querystring += &X-Amz-Date=date
// querystring += &X-Amz-Expires=timeout interval
// querystring += &X-Amz-SignedHeaders=signed_headers
// querystring += &X-Amz-Signature=signature
//
func parsePreSignV4(query url.Values) (preSignValues, *probe.Error) {
// Verify if the query algorithm is supported or not.
if query.Get("X-Amz-Algorithm") != signV4Algorithm {
return preSignValues{}, ErrUnsuppSignAlgo("Unsupported algorithm in query string.", query.Get("X-Amz-Algorithm"))
}
// Initialize signature version '4' structured header.
preSignV4Values := preSignValues{}
var err *probe.Error
// Save credential.
preSignV4Values.Credential, err = parseCredential("Credential=" + query.Get("X-Amz-Credential"))
if err != nil {
return preSignValues{}, err.Trace(query.Get("X-Amz-Credential"))
}
var e error
// Save date in native time.Time.
preSignV4Values.Date, e = time.Parse(iso8601Format, query.Get("X-Amz-Date"))
if e != nil {
return preSignValues{}, ErrMalformedDate("Malformed date string.", query.Get("X-Amz-Date")).Trace(query.Get("X-Amz-Date"))
}
// Save expires in native time.Duration.
preSignV4Values.Expires, e = time.ParseDuration(query.Get("X-Amz-Expires") + "s")
if e != nil {
return preSignValues{}, ErrMalformedExpires("Malformed expires string.", query.Get("X-Amz-Expires")).Trace(query.Get("X-Amz-Expires"))
}
// Save signed headers.
preSignV4Values.SignedHeaders, err = parseSignedHeaders("SignedHeaders=" + query.Get("X-Amz-SignedHeaders"))
if err != nil {
return preSignValues{}, err.Trace(query.Get("X-Amz-SignedHeaders"))
}
// Save signature.
preSignV4Values.Signature, err = parseSignature("Signature=" + query.Get("X-Amz-Signature"))
if err != nil {
return preSignValues{}, err.Trace(query.Get("X-Amz-Signature"))
}
// Return structed form of signature query string.
return preSignV4Values, nil
}
// Parses signature version '4' header of the following form.
//
// Authorization: algorithm Credential=access key ID/credential scope, \
// SignedHeaders=SignedHeaders, Signature=signature
//
func parseSignV4(v4Auth string) (signValues, *probe.Error) {
// Replace all spaced strings, some clients can send spaced
// parameters and some won't. So we pro-actively remove any spaces
// to make parsing easier.
v4Auth = strings.Replace(v4Auth, " ", "", -1)
if v4Auth == "" {
return signValues{}, ErrAuthHeaderEmpty("Auth header empty.").Trace(v4Auth)
}
// Verify if the header algorithm is supported or not.
if !strings.HasPrefix(v4Auth, signV4Algorithm) {
return signValues{}, ErrUnsuppSignAlgo("Unsupported algorithm in authorization header.", v4Auth).Trace(v4Auth)
}
// Strip off the Algorithm prefix.
v4Auth = strings.TrimPrefix(v4Auth, signV4Algorithm)
authFields := strings.Split(strings.TrimSpace(v4Auth), ",")
if len(authFields) != 3 {
return signValues{}, ErrMissingFields("Missing fields in authorization header.", v4Auth).Trace(v4Auth)
}
// Initialize signature version '4' structured header.
signV4Values := signValues{}
var err *probe.Error
// Save credentail values.
signV4Values.Credential, err = parseCredential(authFields[0])
if err != nil {
return signValues{}, err.Trace(v4Auth)
}
// Save signed headers.
signV4Values.SignedHeaders, err = parseSignedHeaders(authFields[1])
if err != nil {
return signValues{}, err.Trace(v4Auth)
}
// Save signature.
signV4Values.Signature, err = parseSignature(authFields[2])
if err != nil {
return signValues{}, err.Trace(v4Auth)
}
// Return the structure here.
return signV4Values, nil
}

View File

@@ -1,209 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package signature4
import (
"encoding/base64"
"encoding/json"
"fmt"
"reflect"
"strings"
"time"
"github.com/minio/minio/pkg/probe"
)
// toString - Safely convert interface to string without causing panic.
func toString(val interface{}) string {
switch v := val.(type) {
case string:
return v
}
return ""
}
// toInteger _ Safely convert interface to integer without causing panic.
func toInteger(val interface{}) int {
switch v := val.(type) {
case int:
return v
}
return 0
}
// isString - Safely check if val is of type string without causing panic.
func isString(val interface{}) bool {
switch val.(type) {
case string:
return true
}
return false
}
// PostPolicyForm provides strict static type conversion and validation for Amazon S3's POST policy JSON string.
type PostPolicyForm struct {
Expiration time.Time // Expiration date and time of the POST policy.
Conditions struct { // Conditional policy structure.
Policies map[string]struct {
Operator string
Value string
}
ContentLengthRange struct {
Min int
Max int
}
}
}
// parsePostPolicyFormV4 - Parse JSON policy string into typed POostPolicyForm structure.
func parsePostPolicyFormV4(policy string) (PostPolicyForm, *probe.Error) {
// Convert po into interfaces and
// perform strict type conversion using reflection.
var rawPolicy struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
}
e := json.Unmarshal([]byte(policy), &rawPolicy)
if e != nil {
return PostPolicyForm{}, probe.NewError(e)
}
parsedPolicy := PostPolicyForm{}
// Parse expiry time.
parsedPolicy.Expiration, e = time.Parse(time.RFC3339Nano, rawPolicy.Expiration)
if e != nil {
return PostPolicyForm{}, probe.NewError(e)
}
parsedPolicy.Conditions.Policies = make(map[string]struct {
Operator string
Value string
})
// Parse conditions.
for _, val := range rawPolicy.Conditions {
switch condt := val.(type) {
case map[string]interface{}: // Handle key:value map types.
for k, v := range condt {
if !isString(v) { // Pre-check value type.
// All values must be of type string.
return parsedPolicy, probe.NewError(fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.",
reflect.TypeOf(condt).String(), condt))
}
// {"acl": "public-read" } is an alternate way to indicate - [ "eq", "$acl", "public-read" ]
// In this case we will just collapse this into "eq" for all use cases.
parsedPolicy.Conditions.Policies["$"+k] = struct {
Operator string
Value string
}{
Operator: "eq",
Value: toString(v),
}
}
case []interface{}: // Handle array types.
if len(condt) != 3 { // Return error if we have insufficient elements.
return parsedPolicy, probe.NewError(fmt.Errorf("Malformed conditional fields %s of type %s found in POST policy form.",
condt, reflect.TypeOf(condt).String()))
}
switch toString(condt[0]) {
case "eq", "starts-with":
for _, v := range condt { // Pre-check all values for type.
if !isString(v) {
// All values must be of type string.
return parsedPolicy, probe.NewError(fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.",
reflect.TypeOf(condt).String(), condt))
}
}
operator, matchType, value := toString(condt[0]), toString(condt[1]), toString(condt[2])
parsedPolicy.Conditions.Policies[matchType] = struct {
Operator string
Value string
}{
Operator: operator,
Value: value,
}
case "content-length-range":
parsedPolicy.Conditions.ContentLengthRange = struct {
Min int
Max int
}{
Min: toInteger(condt[1]),
Max: toInteger(condt[2]),
}
default:
// Condition should be valid.
return parsedPolicy, probe.NewError(fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.",
reflect.TypeOf(condt).String(), condt))
}
default:
return parsedPolicy, probe.NewError(fmt.Errorf("Unknown field %s of type %s found in POST policy form.",
condt, reflect.TypeOf(condt).String()))
}
}
return parsedPolicy, nil
}
// ApplyPolicyCond - apply policy conditions and validate input values.
func ApplyPolicyCond(formValues map[string]string) *probe.Error {
if formValues["X-Amz-Algorithm"] != signV4Algorithm {
return ErrUnsuppSignAlgo("Unsupported signature algorithm in policy form data.", formValues["X-Amz-Algorithm"]).Trace(formValues["X-Amz-Algorithm"])
}
/// Decoding policy
policyBytes, e := base64.StdEncoding.DecodeString(formValues["Policy"])
if e != nil {
return probe.NewError(e)
}
postPolicyForm, err := parsePostPolicyFormV4(string(policyBytes))
if err != nil {
return err.Trace()
}
if !postPolicyForm.Expiration.After(time.Now().UTC()) {
return ErrPolicyAlreadyExpired("Policy has already expired, please generate a new one.")
}
if postPolicyForm.Conditions.Policies["$bucket"].Operator == "eq" {
if formValues["Bucket"] != postPolicyForm.Conditions.Policies["$bucket"].Value {
return ErrMissingFields("Policy bucket is missing.", formValues["Bucket"])
}
}
if postPolicyForm.Conditions.Policies["$x-amz-date"].Operator == "eq" {
if formValues["X-Amz-Date"] != postPolicyForm.Conditions.Policies["$x-amz-date"].Value {
return ErrMissingFields("Policy date is missing.", formValues["X-Amz-Date"])
}
}
if postPolicyForm.Conditions.Policies["$Content-Type"].Operator == "starts-with" {
if !strings.HasPrefix(formValues["Content-Type"], postPolicyForm.Conditions.Policies["$Content-Type"].Value) {
return ErrMissingFields("Policy content-type is missing or invalid.", formValues["Content-Type"])
}
}
if postPolicyForm.Conditions.Policies["$Content-Type"].Operator == "eq" {
if formValues["Content-Type"] != postPolicyForm.Conditions.Policies["$Content-Type"].Value {
return ErrMissingFields("Policy content-Type is missing or invalid.", formValues["Content-Type"])
}
}
if postPolicyForm.Conditions.Policies["$key"].Operator == "starts-with" {
if !strings.HasPrefix(formValues["Key"], postPolicyForm.Conditions.Policies["$key"].Value) {
return ErrMissingFields("Policy key is missing.", formValues["Key"])
}
}
if postPolicyForm.Conditions.Policies["$key"].Operator == "eq" {
if formValues["Key"] != postPolicyForm.Conditions.Policies["$key"].Value {
return ErrMissingFields("Policy key is missing.", formValues["Key"])
}
}
return nil
}

View File

@@ -1,391 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Package signature4 implements helper functions to validate AWS
// Signature Version '4' authorization header.
//
// This package provides comprehensive helpers for following signature
// types.
// - Based on Authorization header.
// - Based on Query parameters.
// - Based on Form POST policy.
package signature4
import (
"bytes"
"encoding/hex"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"time"
"github.com/minio/minio/pkg/crypto/sha256"
"github.com/minio/minio/pkg/probe"
)
// Sign - local variables
type Sign struct {
accessKeyID string
secretAccessKey string
region string
httpRequest *http.Request
extractedSignedHeaders http.Header
}
// AWS Signature Version '4' constants.
const (
signV4Algorithm = "AWS4-HMAC-SHA256"
iso8601Format = "20060102T150405Z"
yyyymmdd = "20060102"
)
// New - initialize a new authorization checkes.
func New(accessKeyID, secretAccessKey, region string) (*Sign, *probe.Error) {
if !isValidAccessKey.MatchString(accessKeyID) {
return nil, ErrInvalidAccessKeyID("Invalid access key id.", accessKeyID).Trace(accessKeyID)
}
if !isValidSecretKey.MatchString(secretAccessKey) {
return nil, ErrInvalidAccessKeyID("Invalid secret key.", secretAccessKey).Trace(secretAccessKey)
}
if region == "" {
return nil, ErrRegionISEmpty("Region is empty.").Trace()
}
signature := &Sign{
accessKeyID: accessKeyID,
secretAccessKey: secretAccessKey,
region: region,
}
return signature, nil
}
// SetHTTPRequestToVerify - sets the http request which needs to be verified.
func (s *Sign) SetHTTPRequestToVerify(r *http.Request) Sign {
// Do not set http request if its 'nil'.
if r == nil {
return *s
}
s.httpRequest = r
return *s
}
// getCanonicalHeaders generate a list of request headers with their values
func (s Sign) getCanonicalHeaders(signedHeaders http.Header) string {
var headers []string
vals := make(http.Header)
for k, vv := range signedHeaders {
headers = append(headers, strings.ToLower(k))
vals[strings.ToLower(k)] = vv
}
headers = append(headers, "host")
sort.Strings(headers)
var buf bytes.Buffer
for _, k := range headers {
buf.WriteString(k)
buf.WriteByte(':')
switch {
case k == "host":
buf.WriteString(s.httpRequest.Host)
fallthrough
default:
for idx, v := range vals[k] {
if idx > 0 {
buf.WriteByte(',')
}
buf.WriteString(v)
}
buf.WriteByte('\n')
}
}
return buf.String()
}
// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names
func (s Sign) getSignedHeaders(signedHeaders http.Header) string {
var headers []string
for k := range signedHeaders {
headers = append(headers, strings.ToLower(k))
}
headers = append(headers, "host")
sort.Strings(headers)
return strings.Join(headers, ";")
}
// getCanonicalRequest generate a canonical request of style
//
// canonicalRequest =
// <HTTPMethod>\n
// <CanonicalURI>\n
// <CanonicalQueryString>\n
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
//
func (s *Sign) getCanonicalRequest() string {
payload := s.httpRequest.Header.Get(http.CanonicalHeaderKey("x-amz-content-sha256"))
s.httpRequest.URL.RawQuery = strings.Replace(s.httpRequest.URL.Query().Encode(), "+", "%20", -1)
encodedPath := getURLEncodedName(s.httpRequest.URL.Path)
// Convert any space strings back to "+".
encodedPath = strings.Replace(encodedPath, "+", "%20", -1)
canonicalRequest := strings.Join([]string{
s.httpRequest.Method,
encodedPath,
s.httpRequest.URL.RawQuery,
s.getCanonicalHeaders(s.extractedSignedHeaders),
s.getSignedHeaders(s.extractedSignedHeaders),
payload,
}, "\n")
return canonicalRequest
}
// getCanonicalRequest generate a canonical request of style
//
// canonicalRequest =
// <HTTPMethod>\n
// <CanonicalURI>\n
// <CanonicalQueryString>\n
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
//
func (s Sign) getPresignedCanonicalRequest(presignedQuery string) string {
rawQuery := strings.Replace(presignedQuery, "+", "%20", -1)
encodedPath := getURLEncodedName(s.httpRequest.URL.Path)
// Convert any space strings back to "+".
encodedPath = strings.Replace(encodedPath, "+", "%20", -1)
canonicalRequest := strings.Join([]string{
s.httpRequest.Method,
encodedPath,
rawQuery,
s.getCanonicalHeaders(s.extractedSignedHeaders),
s.getSignedHeaders(s.extractedSignedHeaders),
"UNSIGNED-PAYLOAD",
}, "\n")
return canonicalRequest
}
// getScope generate a string of a specific date, an AWS region, and a service.
func (s Sign) getScope(t time.Time) string {
scope := strings.Join([]string{
t.Format(yyyymmdd),
s.region,
"s3",
"aws4_request",
}, "/")
return scope
}
// getStringToSign a string based on selected query values.
func (s Sign) getStringToSign(canonicalRequest string, t time.Time) string {
stringToSign := signV4Algorithm + "\n" + t.Format(iso8601Format) + "\n"
stringToSign = stringToSign + s.getScope(t) + "\n"
canonicalRequestBytes := sha256.Sum256([]byte(canonicalRequest))
stringToSign = stringToSign + hex.EncodeToString(canonicalRequestBytes[:])
return stringToSign
}
// getSigningKey hmac seed to calculate final signature.
func (s Sign) getSigningKey(t time.Time) []byte {
secret := s.secretAccessKey
date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
region := sumHMAC(date, []byte(s.region))
service := sumHMAC(region, []byte("s3"))
signingKey := sumHMAC(service, []byte("aws4_request"))
return signingKey
}
// getSignature final signature in hexadecimal form.
func (s Sign) getSignature(signingKey []byte, stringToSign string) string {
return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
}
// DoesPolicySignatureMatch - Verify query headers with post policy
// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
// returns true if matches, false otherwise. if error is not nil then it is always false
func (s *Sign) DoesPolicySignatureMatch(formValues map[string]string) (bool, *probe.Error) {
// Parse credential tag.
credential, err := parseCredential("Credential=" + formValues["X-Amz-Credential"])
if err != nil {
return false, err.Trace(formValues["X-Amz-Credential"])
}
// Verify if the access key id matches.
if credential.accessKeyID != s.accessKeyID {
return false, ErrInvalidAccessKeyID("Access key id does not match with our records.", credential.accessKeyID).Trace(credential.accessKeyID)
}
// Verify if the region is valid.
reqRegion := credential.scope.region
if !isValidRegion(reqRegion, s.region) {
return false, ErrInvalidRegion("Requested region is not recognized.", reqRegion).Trace(reqRegion)
}
// Save region.
s.region = reqRegion
// Parse date string.
t, e := time.Parse(iso8601Format, formValues["X-Amz-Date"])
if e != nil {
return false, probe.NewError(e)
}
signingKey := s.getSigningKey(t)
newSignature := s.getSignature(signingKey, formValues["Policy"])
if newSignature != formValues["X-Amz-Signature"] {
return false, nil
}
return true, nil
}
// DoesPresignedSignatureMatch - Verify query headers with presigned signature
// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
// returns true if matches, false otherwise. if error is not nil then it is always false
func (s *Sign) DoesPresignedSignatureMatch() (bool, *probe.Error) {
// Parse request query string.
preSignValues, err := parsePreSignV4(s.httpRequest.URL.Query())
if err != nil {
return false, err.Trace(s.httpRequest.URL.String())
}
// Verify if the access key id matches.
if preSignValues.Credential.accessKeyID != s.accessKeyID {
return false, ErrInvalidAccessKeyID("Access key id does not match with our records.", preSignValues.Credential.accessKeyID).Trace(preSignValues.Credential.accessKeyID)
}
// Verify if region is valid.
reqRegion := preSignValues.Credential.scope.region
if !isValidRegion(reqRegion, s.region) {
return false, ErrInvalidRegion("Requested region is not recognized.", reqRegion).Trace(reqRegion)
}
// Save region.
s.region = reqRegion
// Extract all the signed headers along with its values.
s.extractedSignedHeaders = extractSignedHeaders(preSignValues.SignedHeaders, s.httpRequest.Header)
// Construct new query.
query := make(url.Values)
query.Set("X-Amz-Algorithm", signV4Algorithm)
if time.Now().UTC().Sub(preSignValues.Date) > time.Duration(preSignValues.Expires) {
return false, ErrExpiredPresignRequest("Presigned request already expired, please initiate a new request.")
}
// Save the date and expires.
t := preSignValues.Date
expireSeconds := int(time.Duration(preSignValues.Expires) / time.Second)
// Construct the query.
query.Set("X-Amz-Date", t.Format(iso8601Format))
query.Set("X-Amz-Expires", strconv.Itoa(expireSeconds))
query.Set("X-Amz-SignedHeaders", s.getSignedHeaders(s.extractedSignedHeaders))
query.Set("X-Amz-Credential", s.accessKeyID+"/"+s.getScope(t))
// Save other headers available in the request parameters.
for k, v := range s.httpRequest.URL.Query() {
if strings.HasPrefix(strings.ToLower(k), "x-amz") {
continue
}
query[k] = v
}
// Get the encoded query.
encodedQuery := query.Encode()
// Verify if date query is same.
if s.httpRequest.URL.Query().Get("X-Amz-Date") != query.Get("X-Amz-Date") {
return false, nil
}
// Verify if expires query is same.
if s.httpRequest.URL.Query().Get("X-Amz-Expires") != query.Get("X-Amz-Expires") {
return false, nil
}
// Verify if signed headers query is same.
if s.httpRequest.URL.Query().Get("X-Amz-SignedHeaders") != query.Get("X-Amz-SignedHeaders") {
return false, nil
}
// Verify if credential query is same.
if s.httpRequest.URL.Query().Get("X-Amz-Credential") != query.Get("X-Amz-Credential") {
return false, nil
}
// Verify finally if signature is same.
newSignature := s.getSignature(s.getSigningKey(t), s.getStringToSign(s.getPresignedCanonicalRequest(encodedQuery), t))
if s.httpRequest.URL.Query().Get("X-Amz-Signature") != newSignature {
return false, nil
}
return true, nil
}
// DoesSignatureMatch - Verify authorization header with calculated header in accordance with
// - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
// returns true if matches, false otherwise. if error is not nil then it is always false
func (s *Sign) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) {
// Save authorization header.
v4Auth := s.httpRequest.Header.Get("Authorization")
// Parse signature version '4' header.
signV4Values, err := parseSignV4(v4Auth)
if err != nil {
return false, err.Trace(v4Auth)
}
// Extract all the signed headers along with its values.
s.extractedSignedHeaders = extractSignedHeaders(signV4Values.SignedHeaders, s.httpRequest.Header)
// Verify if the access key id matches.
if signV4Values.Credential.accessKeyID != s.accessKeyID {
return false, ErrInvalidAccessKeyID("Access key id does not match with our records.", signV4Values.Credential.accessKeyID).Trace(signV4Values.Credential.accessKeyID)
}
// Verify if region is valid.
reqRegion := signV4Values.Credential.scope.region
if !isValidRegion(reqRegion, s.region) {
return false, ErrInvalidRegion("Requested region is not recognized.", reqRegion).Trace(reqRegion)
}
// Save region.
s.region = reqRegion
// Set input payload.
s.httpRequest.Header.Set("X-Amz-Content-Sha256", hashedPayload)
// Extract date, if not present throw error.
var date string
if date = s.httpRequest.Header.Get(http.CanonicalHeaderKey("x-amz-date")); date == "" {
if date = s.httpRequest.Header.Get("Date"); date == "" {
return false, ErrMissingDateHeader("Date header is missing from the request.").Trace()
}
}
// Parse date header.
t, e := time.Parse(iso8601Format, date)
if e != nil {
return false, probe.NewError(e)
}
// Signature version '4'.
canonicalRequest := s.getCanonicalRequest()
stringToSign := s.getStringToSign(canonicalRequest, t)
signingKey := s.getSigningKey(t)
newSignature := s.getSignature(signingKey, stringToSign)
// Verify if signature match.
if newSignature != signV4Values.Signature {
return false, nil
}
return true, nil
}

View File

@@ -1,128 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package signature4
import (
"crypto/hmac"
"encoding/hex"
"net/http"
"regexp"
"strings"
"unicode/utf8"
"github.com/minio/minio/pkg/crypto/sha256"
)
/// helpers
// isValidSecretKey - validate secret key.
var isValidSecretKey = regexp.MustCompile("^.{40}$")
// isValidAccessKey - validate access key.
var isValidAccessKey = regexp.MustCompile("^[A-Z0-9\\-\\.\\_\\~]{20}$")
// isValidRegion - verify if incoming region value is valid with configured Region.
func isValidRegion(reqRegion string, confRegion string) bool {
if confRegion == "" || confRegion == "US" {
confRegion = "us-east-1"
}
// Some older s3 clients set region as "US" instead of
// "us-east-1", handle it.
if reqRegion == "US" {
reqRegion = "us-east-1"
}
return reqRegion == confRegion
}
// sumHMAC calculate hmac between two input byte array.
func sumHMAC(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}
// getURLEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
// non english characters cannot be parsed due to the nature in which url.Encode() is written
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
func getURLEncodedName(name string) string {
// if object matches reserved string, no need to encode them
reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
if reservedNames.MatchString(name) {
return name
}
var encodedName string
for _, s := range name {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedName = encodedName + string(s)
continue
}
switch s {
case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
encodedName = encodedName + string(s)
continue
default:
len := utf8.RuneLen(s)
if len < 0 {
return name
}
u := make([]byte, len)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
encodedName = encodedName + "%" + strings.ToUpper(hex)
}
}
}
return encodedName
}
// extractSignedHeaders extract signed headers from Authorization header
func extractSignedHeaders(signedHeaders []string, reqHeaders http.Header) http.Header {
extractedSignedHeaders := make(http.Header)
for _, header := range signedHeaders {
val, ok := reqHeaders[http.CanonicalHeaderKey(header)]
if !ok {
// Golang http server strips off 'Expect' header, if the
// client sent this as part of signed headers we need to
// handle otherwise we would see a signature mismatch.
// `aws-cli` sets this as part of signed headers.
//
// According to
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.20
// Expect header is always of form:
//
// Expect = "Expect" ":" 1#expectation
// expectation = "100-continue" | expectation-extension
//
// So it safe to assume that '100-continue' is what would
// be sent, for the time being keep this work around.
// Adding a *TODO* to remove this later when Golang server
// doesn't filter out the 'Expect' header.
if header == "expect" {
extractedSignedHeaders[header] = []string{"100-continue"}
}
// If not found continue, we will fail later.
continue
}
extractedSignedHeaders[header] = val
}
return extractedSignedHeaders
}

View File

@@ -1,3 +1,5 @@
// +build ignore
package xl
// BucketACL - bucket level access control

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

3
pkg/xl/doc.go Normal file
View File

@@ -0,0 +1,3 @@
// +build ignore
package xl

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*

View File

@@ -1,3 +1,5 @@
// +build ignore
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*