MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-25 22:55:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

new package: cmd/crypto (#6062)

Browse Source 
This commit introduces a new crypto package providing
AWS S3 related cryptographic building blocks to implement
SSE-S3 (master key or KMS) and SSE-C.

This change only adds some basic functionallity esp.
related to SSE-S3 and documents the general approach
for SSE-S3 and SSE-C.
This commit is contained in:
Andreas Auernhammer 2018-06-28 21:47:42 +02:00 committed by kannappanr
parent ea76e72054
commit 805186ab97
7 changed files with 524 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
cmd/crypto/doc.go Normal file
View File

@ -0,0 +1,116 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package crypto implements AWS S3 related cryptographic building blocks
// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption
// with customer provided keys (SSE-C).
//
// All objects are encrypted with an unique and randomly generated 'ObjectKey'.
// The ObjectKey itself is never stored in plaintext. Instead it is only stored
// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey'
// with an unique key-encryption-key. Given the correct key-encryption-key the
// sealed 'ObjectKey' can be unsealed and the object can be decrypted.
//
//
// ## SSE-C
//
// SSE-C computes the key-encryption-key from the client-provided key, an
// initialization vector (IV) and the bucket/object path.
//
// 1. Encrypt:
// Input: ClientKey, bucket, object, metadata, object_data
// - IV := Random({0,1}²⁵⁶)
// - ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶))
// - KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
// - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
// - metadata <- IV
// - metadata <- SealedKey
// Output: enc_object_data, metadata
//
// 2. Decrypt:
// Input: ClientKey, bucket, object, metadata, enc_object_data
// - IV <- metadata
// - SealedKey <- metadata
// - KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
// - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
// Output: object_data
//
//
// ## SSE-S3
//
// SSE-S3 can use either a master key or a KMS as root-of-trust.
// The en/decryption slightly depens upon which root-of-trust is used.
//
// ### SSE-S3 and single master key
//
// The master key is used to derive unique object- and key-encryption-keys.
// SSE-S3 with a single master key works as SSE-C where the master key is
// used as the client-provided key.
//
// 1. Encrypt:
// Input: MasterKey, bucket, object, metadata, object_data
// - IV := Random({0,1}²⁵⁶)
// - ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶))
// - KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
// - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
// - metadata <- IV
// - metadata <- SealedKey
// Output: enc_object_data, metadata
//
// 2. Decrypt:
// Input: MasterKey, bucket, object, metadata, enc_object_data
// - IV <- metadata
// - SealedKey <- metadata
// - KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
// - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
// Output: object_data
//
//
// ### SSE-S3 and KMS
//
// SSE-S3 requires that the KMS provides two functions:
// 1. Generate(KeyID) -> (Key, EncKey)
// 2. Unseal(KeyID, EncKey) -> Key
//
// 1. Encrypt:
// Input: KeyID, bucket, object, metadata, object_data
// - Key, EncKey := Generate(KeyID)
// - IV := Random({0,1}²⁵⁶)
// - ObjectKey := SHA256(Key, Random({0,1}²⁵⁶))
// - KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
// - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
// - metadata <- IV
// - metadata <- KeyID
// - metadata <- EncKey
// - metadata <- SealedKey
// Output: enc_object_data, metadata
//
// 2. Decrypt:
// Input: bucket, object, metadata, enc_object_data
// - KeyID <- metadata
// - EncKey <- metadata
// - IV <- metadata
// - SealedKey <- metadata
// - Key := Unseal(KeyID, EncKey)
// - KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
// - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
// Output: object_data
//
package crypto

23
cmd/crypto/error.go Normal file
View File

@ -0,0 +1,23 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import "errors"
var (
// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
// is not supported.
ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
)

49
cmd/crypto/header.go Normal file
View File

@ -0,0 +1,49 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"net/http"
)
// SSEHeader is the general AWS SSE HTTP header key.
const SSEHeader = "X-Amz-Server-Side-Encryption"
// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
// For SSE-C see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
const SSEAlgorithmAES256 = "AES256"
// S3 represents AWS SSE-S3. It provides functionality to handle
// SSE-S3 requests.
var S3 = s3{}
type s3 struct{}
// IsRequested returns true if the HTTP headers indicates that
// the S3 client requests SSE-S3.
func (s3) IsRequested(h http.Header) bool {
_, ok := h[SSEHeader]
return ok
}
// Parse parses the SSE-S3 related HTTP headers and checks
// whether they contain valid values.
func (s3) Parse(h http.Header) (err error) {
if h.Get(SSEHeader) != SSEAlgorithmAES256 {
err = ErrInvalidEncryptionMethod
}
return
}

56
cmd/crypto/header_test.go Normal file
View File

@ -0,0 +1,56 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"net/http"
"testing"
)
var isRequestedTests = []struct {
Header http.Header
Expected bool
}{
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true}, // 0
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true}, // 2
{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false}, // 3
}
func TestS3IsRequested(t *testing.T) {
for i, test := range isRequestedTests {
if got := S3.IsRequested(test.Header); got != test.Expected {
t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
}
}
}
var parseTests = []struct {
Header http.Header
ExpectedErr error
}{
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil}, // 0
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 2
{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 3
}
func TestS3Parse(t *testing.T) {
for i, test := range parseTests {
if err := S3.Parse(test.Header); err != test.ExpectedErr {
t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
}
}
}

93
cmd/crypto/key.go Normal file
View File

@ -0,0 +1,93 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"bytes"
"context"
"crypto/hmac"
"crypto/rand"
"encoding/binary"
"errors"
"io"
"path/filepath"
"github.com/minio/minio/cmd/logger"
sha256 "github.com/minio/sha256-simd"
"github.com/minio/sio"
)
// ObjectKey is a 256 bit secret key used to encrypt the object.
// It must never be stored in plaintext.
type ObjectKey [32]byte
// GenerateKey generates a unique ObjectKey from a 256 bit external key
// and a source of randomness. If random is nil the default PRNG of system
// (crypto/rand) is used.
func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
if random == nil {
random = rand.Reader
}
var nonce [32]byte
if _, err := io.ReadFull(random, nonce[:]); err != nil {
logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
}
sha := sha256.New()
sha.Write(extKey[:])
sha.Write(nonce[:])
sha.Sum(key[:0])
return
}
// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
// key is also cryptographically bound to the object's path (bucket/object).
func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
var sealedKey bytes.Buffer
mac := hmac.New(sha256.New, extKey[:])
mac.Write(iv[:])
mac.Write([]byte(filepath.Join(bucket, object)))
if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
}
return sealedKey.Bytes()
}
// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
// is cryptographically bound to the object's path the same bucket/object as during sealing
// must be provided. On success the ObjectKey contains the decrypted sealed key.
func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
var unsealedKey bytes.Buffer
mac := hmac.New(sha256.New, extKey[:])
mac.Write(iv[:])
mac.Write([]byte(filepath.Join(bucket, object)))
if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
return err // TODO(aead): upgrade sio to use sio.Error
}
copy(key[:], unsealedKey.Bytes())
return nil
}
// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index.
func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
var bin [4]byte
binary.LittleEndian.PutUint32(bin[:], id)
mac := hmac.New(sha256.New, key[:])
mac.Write(bin[:])
mac.Sum(partKey[:0])
return
}

127
cmd/crypto/key_test.go Normal file
View File

@ -0,0 +1,127 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"bytes"
"crypto/rand"
"encoding/hex"
"io"
"testing"
)
var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
func recoverTest(i int, shouldPass bool, t *testing.T) {
if err := recover(); err == nil && !shouldPass {
t.Errorf("Test %d should fail but passed successfully", i)
} else if err != nil && shouldPass {
t.Errorf("Test %d should pass but failed: %v", i, err)
}
}
var generateKeyTests = []struct {
ExtKey [32]byte
Random io.Reader
ShouldPass bool
}{
{ExtKey: [32]byte{}, Random: nil, ShouldPass: true}, // 0
{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true}, // 1
{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
}
func TestGenerateKey(t *testing.T) {
for i, test := range generateKeyTests {
func() {
defer recoverTest(i, test.ShouldPass, t)
key := GenerateKey(test.ExtKey, test.Random)
if [32]byte(key) == [32]byte{} {
t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key
}
}()
}
}
var sealUnsealKeyTests = []struct {
SealExtKey, SealIV [32]byte
SealBucket, SealObject string
UnsealExtKey, UnsealIV [32]byte
UnsealBucket, UnsealObject string
ShouldPass bool
}{
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
ShouldPass: true,
}, // 0
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
ShouldPass: false,
}, // 1
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
ShouldPass: false,
}, // 2
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
ShouldPass: false,
}, // 3
{
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
ShouldPass: false,
}, // 4
}
func TestSealUnsealKey(t *testing.T) {
for i, test := range sealUnsealKeyTests {
key := GenerateKey(test.SealExtKey, rand.Reader)
sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
t.Errorf("Test %d should fail but passed successfully", i)
} else if err != nil && test.ShouldPass {
t.Errorf("Test %d should pass put failed: %v", i, err)
}
}
}
var derivePartKeyTest = []struct {
PartID uint32
PartKey string
}{
{PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"},
{PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"},
{PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"},
}
func TestDerivePartKey(t *testing.T) {
var key ObjectKey
for i, test := range derivePartKeyTest {
expectedPartKey, err := hex.DecodeString(test.PartKey)
if err != nil {
t.Fatalf("Test %d failed to decode expected part-key: %v", i, err)
}
partKey := key.DerivePartKey(test.PartID)
if !bytes.Equal(partKey[:], expectedPartKey[:]) {
t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey)
}
}
}

60
cmd/crypto/sse.go Normal file
View File

@ -0,0 +1,60 @@
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"context"
"errors"
"io"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/ioutil"
"github.com/minio/sio"
)
const (
// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
// S3KMSKeyID is the metadata key referencing the KMS key-id used to
// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
// S3KMSSealedKey is the metadata key referencing the encrypted key generated
// by KMS. It is only used for SSE-S3 + KMS.
S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
)
// EncryptSinglePart encrypts an io.Reader which must be the
// the body of a single-part PUT request.
func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]})
if err != nil {
logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key"))
}
return r
}
// DecryptSinglePart decrypts an io.Writer which must an object
// uploaded with the single-part PUT API. The offset and length
// specify the requested range.
func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser {
const PayloadSize = 1 << 16 // DARE 2.0
w = ioutil.LimitedWriter(w, offset%PayloadSize, length)
decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]})
if err != nil {
logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key"))
}
return decWriter
}