MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-21 02:09:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add double encryption at S3 gateway. (#6423)

Browse Source 
This PR adds pass-through, single encryption at gateway and double
encryption support (gateway encryption with pass through of SSE
headers to backend).

If KMS is set up (either with Vault as KMS or using
MINIO_SSE_MASTER_KEY),gateway will automatically perform
single encryption. If MINIO_GATEWAY_SSE is set up in addition to
Vault KMS, double encryption is performed.When neither KMS nor
MINIO_GATEWAY_SSE is set, do a pass through to backend.

When double encryption is specified, MINIO_GATEWAY_SSE can be set to
"C" for SSE-C encryption at gateway and backend, "S3" for SSE-S3
encryption at gateway/backend or both to support more than one option.

Fixes #6323, #6696
This commit is contained in:
poornas
2019-01-05 14:16:43 -08:00
committed by kannappanr
parent 2d19011a1d
commit 5a80cbec2a
59 changed files with 1902 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
cmd/encryption-v1.go
View File

@@ -29,6 +29,7 @@ import (
"path"
"strconv"
"github.com/minio/minio-go/pkg/encrypt"
"github.com/minio/minio/cmd/crypto"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/ioutil"
@@ -55,11 +56,11 @@ const (
// SSEIVSize is the size of the IV data
SSEIVSize = 32 // 32 bytes
// SSE dare package block size.
sseDAREPackageBlockSize = 64 * 1024 // 64KiB bytes
// SSEDAREPackageBlockSize - SSE dare package block size.
SSEDAREPackageBlockSize = 64 * 1024 // 64KiB bytes
// SSE dare package meta padding bytes.
sseDAREPackageMetaSize = 32 // 32 bytes
// SSEDAREPackageMetaSize - SSE dare package meta padding bytes.
SSEDAREPackageMetaSize = 32 // 32 bytes
)
@@ -158,14 +159,14 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
crypto.SSEC.CreateMetadata(metadata, sealedKey)
return nil
case crypto.S3.IsEncrypted(metadata):
if globalKMS == nil {
if GlobalKMS == nil {
return errKMSNotConfigured
}
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
if err != nil {
return err
}
oldKey, err := globalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
oldKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return err
}
@@ -174,7 +175,7 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
return err
}
newKey, encKey, err := globalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
newKey, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return err
}
@@ -187,10 +188,10 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string, sseS3 bool) ([]byte, error) {
var sealedKey crypto.SealedKey
if sseS3 {
if globalKMS == nil {
if GlobalKMS == nil {
return nil, errKMSNotConfigured
}
key, encKey, err := globalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
key, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return nil, err
}
@@ -279,7 +280,7 @@ func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]st
default:
return nil, errObjectTampered
case crypto.S3.IsEncrypted(metadata):
if globalKMS == nil {
if GlobalKMS == nil {
return nil, errKMSNotConfigured
}
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
@@ -287,7 +288,7 @@ func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]st
if err != nil {
return nil, err
}
extKey, err := globalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
extKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return nil, err
}
@@ -395,7 +396,6 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
io.Reader, error) {
bucket, object := oi.Bucket, oi.Name
// Single part case
if !isEncryptedMultipart(oi) {
var reader io.Reader
@@ -411,8 +411,8 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
return reader, nil
}
partDecRelOffset := int64(seqNumber) * sseDAREPackageBlockSize
partEncRelOffset := int64(seqNumber) * (sseDAREPackageBlockSize + sseDAREPackageMetaSize)
partDecRelOffset := int64(seqNumber) * SSEDAREPackageBlockSize
partEncRelOffset := int64(seqNumber) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
w := &DecryptBlocksReader{
reader: inputReader,
@@ -477,7 +477,7 @@ type DecryptBlocksReader struct {
// Current part index
partIndex int
// Parts information
parts []objectPartInfo
parts []ObjectPartInfo
header http.Header
bucket, object string
metadata map[string]string
@@ -592,7 +592,7 @@ type DecryptBlocksWriter struct {
// Current part index
partIndex int
// Parts information
parts []objectPartInfo
parts []ObjectPartInfo
req *http.Request
bucket, object string
metadata map[string]string
@@ -743,6 +743,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
}
seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
var partStartIndex int
var partStartOffset = startOffset
// Skip parts until final offset maps to a particular part offset.
@@ -765,8 +766,8 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
partStartOffset -= int64(decryptedSize)
}
startSeqNum := partStartOffset / sseDAREPackageBlockSize
partEncRelOffset := int64(startSeqNum) * (sseDAREPackageBlockSize + sseDAREPackageMetaSize)
startSeqNum := partStartOffset / SSEDAREPackageBlockSize
partEncRelOffset := int64(startSeqNum) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
w := &DecryptBlocksWriter{
writer: client,
@@ -857,9 +858,9 @@ func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (u
// getEncryptedSinglePartOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
func getEncryptedSinglePartOffsetLength(offset, length int64, objInfo ObjectInfo) (seqNumber uint32, encOffset int64, encLength int64) {
onePkgSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
onePkgSize := int64(SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
seqNumber = uint32(offset / sseDAREPackageBlockSize)
seqNumber = uint32(offset / SSEDAREPackageBlockSize)
encOffset = int64(seqNumber) * onePkgSize
// The math to compute the encrypted length is always
// originalLength i.e (offset+length-1) to be divided under
@@ -867,10 +868,10 @@ func getEncryptedSinglePartOffsetLength(offset, length int64, objInfo ObjectInfo
// block. This is then multiplied by final package size which
// is basically 64KiB + 32. Finally negate the encrypted offset
// to get the final encrypted length on disk.
encLength = ((offset+length)/sseDAREPackageBlockSize)*onePkgSize - encOffset
encLength = ((offset+length)/SSEDAREPackageBlockSize)*onePkgSize - encOffset
// Check for the remainder, to figure if we need one extract package to read from.
if (offset+length)%sseDAREPackageBlockSize > 0 {
if (offset+length)%SSEDAREPackageBlockSize > 0 {
encLength += onePkgSize
}
@@ -1041,11 +1042,11 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
// partStart is always found in the loop above,
// because off is validated.
sseDAREEncPackageBlockSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
startPkgNum := (off - cumulativeSum) / sseDAREPackageBlockSize
sseDAREEncPackageBlockSize := int64(SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
startPkgNum := (off - cumulativeSum) / SSEDAREPackageBlockSize
// Now we can calculate the number of bytes to skip
skipLen = (off - cumulativeSum) % sseDAREPackageBlockSize
skipLen = (off - cumulativeSum) % SSEDAREPackageBlockSize
encOff = encCumulativeSum + startPkgNum*sseDAREEncPackageBlockSize
// Locate the part containing the end of the required range
@@ -1062,7 +1063,7 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
}
// partEnd is always found in the loop above, because off and
// length are validated.
endPkgNum := (endOffset - cumulativeSum) / sseDAREPackageBlockSize
endPkgNum := (endOffset - cumulativeSum) / SSEDAREPackageBlockSize
// Compute endEncOffset with one additional DARE package (so
// we read the package containing the last desired byte).
endEncOffset := encCumulativeSum + (endPkgNum+1)*sseDAREEncPackageBlockSize
@@ -1157,3 +1158,113 @@ func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, e
}
return
}
// The customer key in the header is used by the gateway for encryption in the case of
// s3 gateway double encryption. A new client key is derived from the customer provided
// key to be sent to the s3 backend for encryption at the backend.
func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
var key [32]byte
mac := hmac.New(sha256.New, clientKey[:])
mac.Write([]byte(crypto.SSEC.String()))
mac.Write([]byte(path.Join(bucket, object)))
mac.Sum(key[:0])
return key
}
// extract encryption options for pass through to backend in the case of gateway
func extractEncryptionOption(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
var clientKey [32]byte
var sse encrypt.ServerSide
if copySource {
if crypto.SSECopy.IsRequested(header) {
clientKey, err = crypto.SSECopy.ParseHTTP(header)
if err != nil {
return
}
if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
return
}
return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(sse)}, nil
}
return
}
if crypto.SSEC.IsRequested(header) {
clientKey, err = crypto.SSEC.ParseHTTP(header)
if err != nil {
return
}
if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
return
}
return ObjectOptions{ServerSideEncryption: sse}, nil
}
if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
}
return opts, nil
}
// get ObjectOptions for GET calls from encryption headers
func getEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
var (
encryption encrypt.ServerSide
opts ObjectOptions
)
if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
key, err := crypto.SSEC.ParseHTTP(r.Header)
if err != nil {
return opts, err
}
derivedKey := deriveClientKey(key, bucket, object)
encryption, err = encrypt.NewSSEC(derivedKey[:])
logger.CriticalIf(ctx, err)
return ObjectOptions{ServerSideEncryption: encryption}, nil
}
// default case of passing encryption headers to backend
return extractEncryptionOption(r.Header, false, nil)
}
// get ObjectOptions for PUT calls from encryption headers
func putEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {
return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
}
if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
return getEncryptionOpts(ctx, r, bucket, object)
}
// default case of passing encryption headers to backend
return extractEncryptionOption(r.Header, false, metadata)
}
// get ObjectOptions for Copy calls for encryption headers provided on the target side
func copyDstEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
return putEncryptionOpts(ctx, r, bucket, object, metadata)
}
// get ObjectOptions for Copy calls for encryption headers provided on the source side
func copySrcEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
var (
ssec encrypt.ServerSide
opts ObjectOptions
)
if GlobalGatewaySSE.SSEC() && crypto.SSECopy.IsRequested(r.Header) {
key, err := crypto.SSECopy.ParseHTTP(r.Header)
if err != nil {
return opts, err
}
derivedKey := deriveClientKey(key, bucket, object)
ssec, err = encrypt.NewSSEC(derivedKey[:])
if err != nil {
return opts, err
}
return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(ssec)}, nil
}
// default case of passing encryption headers to backend
return extractEncryptionOption(r.Header, true, nil)
}