mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
etag: add Format
and Decrypt
functions (#14659)
This commit adds two new functions to the internal `etag` package: - `ETag.Format` - `Decrypt` The `Decrypt` function decrypts an encrypted ETag using a decryption key. It returns not encrypted / multipart ETags unmodified. The `Decrypt` function is mainly used when handling SSE-S3 encrypted single-part objects. In particular, the ETag of an SSE-S3 encrypted single-part object needs to be decrypted since S3 clients expect that this ETag is equal to the content MD5. The `ETag.Format` method also covers SSE ETag handling. MinIO encrypts all ETags of SSE single part objects. However, only the ETag of SSE-S3 encrypted single part objects needs to be decrypted. The ETag of an SSE-C or SSE-KMS single part object does not correspond to its content MD5 and can be a random value. The `ETag.Format` function formats an ETag such that it is an AWS S3 compliant ETag. In particular, it returns non-encrypted ETags (single / multipart) unmodified. However, for encrypted ETags it returns the trailing 16 bytes as ETag. For encrypted ETags the last 16 bytes will be a random value. The main purpose of `Format` is to format ETags such that clients accept them as well-formed AWS S3 ETags. It differs from the `String` method since `String` will return string representations for encrypted ETags that are not AWS S3 compliant. Signed-off-by: Andreas Auernhammer <hi@aead.dev>
This commit is contained in:
parent
7c696e1cb6
commit
b9d1698d74
@ -19,19 +19,17 @@ package cmd
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/etag"
|
||||
)
|
||||
|
||||
// CacheStatusType - whether the request was served from cache.
|
||||
@ -234,77 +232,85 @@ func isCacheEncrypted(meta map[string]string) bool {
|
||||
|
||||
// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
|
||||
func decryptCacheObjectETag(info *ObjectInfo) error {
|
||||
// Directories are never encrypted.
|
||||
if info.IsDir {
|
||||
return nil
|
||||
return nil // Directories are never encrypted.
|
||||
}
|
||||
encrypted := crypto.S3.IsEncrypted(info.UserDefined) && isCacheEncrypted(info.UserDefined)
|
||||
|
||||
switch {
|
||||
case encrypted:
|
||||
if globalCacheKMS == nil {
|
||||
return errKMSNotConfigured
|
||||
// Depending on the SSE type we handle ETags slightly
|
||||
// differently. ETags encrypted with SSE-S3 must be
|
||||
// decrypted first, since the client expects that
|
||||
// a single-part SSE-S3 ETag is equal to the content MD5.
|
||||
//
|
||||
// For all other SSE types, the ETag is not the content MD5.
|
||||
// Therefore, we don't decrypt but only format it.
|
||||
switch kind, ok := crypto.IsEncrypted(info.UserDefined); {
|
||||
case ok && kind == crypto.S3 && isCacheEncrypted(info.UserDefined):
|
||||
ETag, err := etag.Parse(info.ETag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(info.Parts) > 0 { // multipart ETag is not encrypted since it is not md5sum
|
||||
if !ETag.IsEncrypted() {
|
||||
info.ETag = ETag.Format().String()
|
||||
return nil
|
||||
}
|
||||
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(info.UserDefined)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
extKey, err := globalCacheKMS.DecryptKey(keyID, kmsKey, kms.Context{info.Bucket: path.Join(info.Bucket, info.Name)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var objectKey crypto.ObjectKey
|
||||
if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), info.Bucket, info.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
etagStr := tryDecryptETag(objectKey[:], info.ETag, false)
|
||||
// backend ETag was hex encoded before encrypting, so hex decode to get actual ETag
|
||||
etag, err := hex.DecodeString(etagStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info.ETag = string(etag)
|
||||
return nil
|
||||
}
|
||||
|
||||
key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, info.UserDefined, info.Bucket, info.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ETag, err = etag.Decrypt(key[:], ETag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info.ETag = ETag.Format().String()
|
||||
case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(info.UserDefined):
|
||||
ETag, err := etag.Parse(info.ETag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info.ETag = ETag.Format().String()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
|
||||
func decryptCachePartETags(c *cacheMeta) ([]string, error) {
|
||||
var partETags []string
|
||||
encrypted := crypto.S3.IsEncrypted(c.Meta) && isCacheEncrypted(c.Meta)
|
||||
|
||||
switch {
|
||||
case encrypted:
|
||||
if globalCacheKMS == nil {
|
||||
return partETags, errKMSNotConfigured
|
||||
}
|
||||
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(c.Meta)
|
||||
// Depending on the SSE type we handle ETags slightly
|
||||
// differently. ETags encrypted with SSE-S3 must be
|
||||
// decrypted first, since the client expects that
|
||||
// a single-part SSE-S3 ETag is equal to the content MD5.
|
||||
//
|
||||
// For all other SSE types, the ETag is not the content MD5.
|
||||
// Therefore, we don't decrypt but only format it.
|
||||
switch kind, ok := crypto.IsEncrypted(c.Meta); {
|
||||
case ok && kind == crypto.S3 && isCacheEncrypted(c.Meta):
|
||||
key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, c.Meta, c.Bucket, c.Object)
|
||||
if err != nil {
|
||||
return partETags, err
|
||||
}
|
||||
extKey, err := globalCacheKMS.DecryptKey(keyID, kmsKey, kms.Context{c.Bucket: path.Join(c.Bucket, c.Object)})
|
||||
if err != nil {
|
||||
return partETags, err
|
||||
}
|
||||
var objectKey crypto.ObjectKey
|
||||
if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), c.Bucket, c.Object); err != nil {
|
||||
return partETags, err
|
||||
return nil, err
|
||||
}
|
||||
etags := make([]string, 0, len(c.PartETags))
|
||||
for i := range c.PartETags {
|
||||
etagStr := tryDecryptETag(objectKey[:], c.PartETags[i], false)
|
||||
// backend ETag was hex encoded before encrypting, so hex decode to get actual ETag
|
||||
etag, err := hex.DecodeString(etagStr)
|
||||
ETag, err := etag.Parse(c.PartETags[i])
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
return nil, err
|
||||
}
|
||||
partETags = append(partETags, string(etag))
|
||||
ETag, err = etag.Decrypt(key[:], ETag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
etags = append(etags, ETag.Format().String())
|
||||
}
|
||||
return partETags, nil
|
||||
return etags, nil
|
||||
case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(c.Meta):
|
||||
etags := make([]string, 0, len(c.PartETags))
|
||||
for i := range c.PartETags {
|
||||
ETag, err := etag.Parse(c.PartETags[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
etags = append(etags, ETag.Format().String())
|
||||
}
|
||||
return etags, nil
|
||||
default:
|
||||
return c.PartETags, nil
|
||||
}
|
||||
|
@ -108,7 +108,9 @@ package etag
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
@ -116,6 +118,9 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/internal/fips"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
// ETag is a single S3 ETag.
|
||||
@ -188,6 +193,47 @@ func (e ETag) Parts() int {
|
||||
return parts
|
||||
}
|
||||
|
||||
// Format returns an ETag that is formatted as specified
|
||||
// by AWS S3.
|
||||
//
|
||||
// An AWS S3 ETag is 16 bytes long and, in case of a multipart
|
||||
// upload, has a `-N` suffix encoding the number of object parts.
|
||||
// An ETag is not AWS S3 compatible when encrypted. When sending
|
||||
// an ETag back to an S3 client it has to be formatted to be
|
||||
// AWS S3 compatible.
|
||||
//
|
||||
// Therefore, Format returns the last 16 bytes of an encrypted
|
||||
// ETag.
|
||||
//
|
||||
// In general, a caller has to distinguish the following cases:
|
||||
// - The object is a multipart object. In this case,
|
||||
// Format returns the ETag unmodified.
|
||||
// - The object is a SSE-KMS or SSE-C encrypted single-
|
||||
// part object. In this case, Format returns the last
|
||||
// 16 bytes of the encrypted ETag which will be a random
|
||||
// value.
|
||||
// - The object is a SSE-S3 encrypted single-part object.
|
||||
// In this case, the caller has to decrypt the ETag first
|
||||
// before calling Format.
|
||||
// S3 clients expect that the ETag of an SSE-S3 encrypted
|
||||
// single-part object is equal to the object's content MD5.
|
||||
// Formatting the SSE-S3 ETag before decryption will result
|
||||
// in a random-looking ETag which an S3 client will not accept.
|
||||
//
|
||||
// Hence, a caller has to check:
|
||||
// if method == SSE-S3 {
|
||||
// ETag, err := Decrypt(key, ETag)
|
||||
// if err != nil {
|
||||
// }
|
||||
// }
|
||||
// ETag = ETag.Format()
|
||||
func (e ETag) Format() ETag {
|
||||
if !e.IsEncrypted() {
|
||||
return e
|
||||
}
|
||||
return e[len(e)-16:]
|
||||
}
|
||||
|
||||
var _ Tagger = ETag{} // compiler check
|
||||
|
||||
// ETag returns the ETag itself.
|
||||
@ -277,6 +323,31 @@ func Get(h http.Header) (ETag, error) {
|
||||
// identical.
|
||||
func Equal(a, b ETag) bool { return bytes.Equal(a, b) }
|
||||
|
||||
// Decrypt decrypts the ETag with the given key.
|
||||
//
|
||||
// If the ETag is not encrypted, Decrypt returns
|
||||
// the ETag unmodified.
|
||||
func Decrypt(key []byte, etag ETag) (ETag, error) {
|
||||
const HMACContext = "SSE-etag"
|
||||
|
||||
if !etag.IsEncrypted() {
|
||||
return etag, nil
|
||||
}
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte(HMACContext))
|
||||
decryptionKey := mac.Sum(nil)
|
||||
|
||||
plaintext := make([]byte, 0, 16)
|
||||
etag, err := sio.DecryptBuffer(plaintext, etag, sio.Config{
|
||||
Key: decryptionKey,
|
||||
CipherSuites: fips.CipherSuitesDARE(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return etag, nil
|
||||
}
|
||||
|
||||
// Parse parses s as an S3 ETag, returning the result.
|
||||
// The string can be an encrypted, singlepart
|
||||
// or multipart S3 ETag. It returns an error if s is
|
||||
|
@ -213,6 +213,29 @@ func TestIsEncrypted(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
var formatTests = []struct {
|
||||
ETag string
|
||||
AWSETag string
|
||||
}{
|
||||
{ETag: "3b83ef96387f14655fc854ddc3c6bd57", AWSETag: "3b83ef96387f14655fc854ddc3c6bd57"}, // 0
|
||||
{ETag: "7b976cc68452e003eec7cb0eb631a19a-1", AWSETag: "7b976cc68452e003eec7cb0eb631a19a-1"}, // 1
|
||||
{ETag: "a7d414b9133d6483d9a1c4e04e856e3b-2", AWSETag: "a7d414b9133d6483d9a1c4e04e856e3b-2"}, // 2
|
||||
{ETag: "7b976cc68452e003eec7cb0eb631a19a-10000", AWSETag: "7b976cc68452e003eec7cb0eb631a19a-10000"}, // 3
|
||||
{ETag: "20000f00db2d90a7b40782d4cff2b41a7799fc1e7ead25972db65150118dfbe2ba76a3c002da28f85c840cd2001a28a9", AWSETag: "ba76a3c002da28f85c840cd2001a28a9"}, // 4
|
||||
}
|
||||
|
||||
func TestFormat(t *testing.T) {
|
||||
for i, test := range formatTests {
|
||||
tag, err := Parse(test.ETag)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to parse ETag: %v", i, err)
|
||||
}
|
||||
if s := tag.Format().String(); s != test.AWSETag {
|
||||
t.Fatalf("Test %d: got '%v' - want '%v'", i, s, test.AWSETag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var fromContentMD5Tests = []struct {
|
||||
Header http.Header
|
||||
ETag ETag
|
||||
@ -246,6 +269,45 @@ func TestFromContentMD5(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
var decryptTests = []struct {
|
||||
Key []byte
|
||||
ETag ETag
|
||||
Plaintext ETag
|
||||
}{
|
||||
{ // 0
|
||||
Key: make([]byte, 32),
|
||||
ETag: must("3b83ef96387f14655fc854ddc3c6bd57"),
|
||||
Plaintext: must("3b83ef96387f14655fc854ddc3c6bd57"),
|
||||
},
|
||||
{ // 1
|
||||
Key: make([]byte, 32),
|
||||
ETag: must("7b976cc68452e003eec7cb0eb631a19a-1"),
|
||||
Plaintext: must("7b976cc68452e003eec7cb0eb631a19a-1"),
|
||||
},
|
||||
{ // 2
|
||||
Key: make([]byte, 32),
|
||||
ETag: must("7b976cc68452e003eec7cb0eb631a19a-10000"),
|
||||
Plaintext: must("7b976cc68452e003eec7cb0eb631a19a-10000"),
|
||||
},
|
||||
{ // 3
|
||||
Key: make([]byte, 32),
|
||||
ETag: must("20000f00f2cc184414bc982927ec56abb7e18426faa205558982e9a8125c1370a9cf5754406e428b3343f21ee1125965"),
|
||||
Plaintext: must("6d6cdccb9a7498c871bde8eab2f49141"),
|
||||
},
|
||||
}
|
||||
|
||||
func TestDecrypt(t *testing.T) {
|
||||
for i, test := range decryptTests {
|
||||
etag, err := Decrypt(test.Key, test.ETag)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to decrypt ETag: %v", i, err)
|
||||
}
|
||||
if !Equal(etag, test.Plaintext) {
|
||||
t.Fatalf("Test %d: got '%v' - want '%v'", i, etag, test.Plaintext)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func must(s string) ETag {
|
||||
t, err := Parse(s)
|
||||
if err != nil {
|
||||
|
Loading…
Reference in New Issue
Block a user