MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

listing: decrypt only SSE-S3 single-part ETags (#14638)

Browse Source 
This commit optimises the ETag decryption when
listing objects.

When MinIO lists objects, it has to decrypt the
ETags of single-part SSE-S3 objects.

It does not need to decrypt ETags of
 - plaintext objects => Their ETag is not encrypted
 - SSE-C objects     => Their ETag is not the content MD5
 - SSE-KMS objects   => Their ETag is not the content MD5
 - multipart objects => Their ETag is not encrypted

Hence, MinIO only needs to make a call to the KMS
when it needs to decrypt a single-part SSE-S3 object.
It can resolve the ETags off all other object types
locally.

This commit implements the above semantics by
processing an object listing in batches.
If the batch contains no single-part SSE-S3 object,
then no KMS calls will be made.

If the batch contains at least one single-part
SSE-S3 object we have to make at least one KMS call.
No we first filter all single-part SSE-S3 objects
such that we only request the decryption keys for
these objects.
Once we know which objects resp. ETags require a
decryption key, MinIO either uses the KES bulk
decryption API (if supported) or decrypts each
ETag serially.

This commit is a significant improvement compared
to the previous listing code. Before, a single
non-SSE-S3 object caused MinIO to fall-back to
a serial ETag decryption.
For example, if a batch consisted of 249 SSE-S3
objects and one single SSE-KMS object, MinIO would
send 249 requests to the KMS.
Now, MinIO will send a single request for exactly
those 249 objects and skip the one SSE-KMS object
since it can handle its ETag locally.

Further, MinIO would request decryption keys
for SSE-S3 multipart objects in the past - even
though multipart ETags are not encrypted.
So, if a bucket contained only multipart SSE-S3
objects, MinIO would make totally unnecessary
requests to the KMS.
Now, MinIO simply skips these multipart objects
since it can handle the ETags locally.

Signed-off-by: Andreas Auernhammer <hi@aead.dev>
This commit is contained in:
Andreas Auernhammer 2022-03-28 03:34:11 +02:00 committed by GitHub
parent 908eb57795
commit 04df69f633
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 53 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
cmd/encryption-v1.go
View File

@ -99,9 +99,9 @@ func kmsKeyIDFromMetadata(metadata map[string]string) string {
// DecryptETags dectypts all ObjectInfo ETags, if encrypted, using the KMS.
func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchSize int) error {
var (
metadata []map[string]string
buckets []string
names []string
metadata = make([]map[string]string, 0, batchSize)
buckets = make([]string, 0, batchSize)
names = make([]string, 0, batchSize)
)
for len(objects) > 0 {
var N int
@ -111,14 +111,20 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchS
N = batchSize
}
SSES3Batch := true
// We have to conntect the KMS only if there is at least
// one SSE-S3 single-part object. SSE-C and SSE-KMS objects
// don't return the plaintext MD5 ETag and the ETag of
// SSE-S3 multipart objects is not encrypted.
// Therefore, we can skip the expensive KMS calls whenever
// there is no single-part SSE-S3 object entirely.
var containsSSES3SinglePart bool
for _, object := range objects[:N] {
if kind, ok := crypto.IsEncrypted(object.UserDefined); !ok || kind != crypto.S3 {
SSES3Batch = false
if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) {
containsSSES3SinglePart = true
break
}
}
if !SSES3Batch {
if !containsSSES3SinglePart {
for i := range objects[:N] {
size, err := objects[i].GetActualSize()
if err != nil {
@ -131,33 +137,38 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchS
continue
}
// Now, decrypt all ETags using the a specialized bulk decryption API, if available.
// We check the cap of all slices first, to avoid allocating them over and over again.
if cap(metadata) >= N {
metadata = metadata[:0:N]
} else {
metadata = make([]map[string]string, 0, N)
}
if cap(buckets) >= N {
buckets = buckets[:0:N]
} else {
buckets = make([]string, 0, N)
}
if cap(names) >= N {
names = names[:0:N]
} else {
names = make([]string, 0, N)
}
for _, object := range objects[:N] {
metadata = append(metadata, object.UserDefined)
buckets = append(buckets, object.Bucket)
names = append(names, object.Name)
// Now, there are some SSE-S3 single-part objects.
// We only request the decryption keys for them.
// We don't want to get the decryption keys for multipart
// or non-SSE-S3 objects.
//
// Therefore, we keep a map of indicies to remember which
// object was an SSE-S3 single-part object.
// Then we request the decryption keys for these objects.
// Finally, we decrypt the ETags of these objects using
// the decryption keys.
// However, we must also adjust the size and ETags of all
// objects (not just the SSE-S3 single part objects).
// For example, the ETag of SSE-KMS objects are random values
// and the size of an SSE-KMS object must be adjusted as well.
SSES3Objects := make(map[int]bool, 10)
metadata = metadata[:0:N]
buckets = buckets[:0:N]
names = names[:0:N]
for i, object := range objects[:N] {
if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) {
metadata = append(metadata, object.UserDefined)
buckets = append(buckets, object.Bucket)
names = append(names, object.Name)
SSES3Objects[i] = true
}
}
keys, err := crypto.S3.UnsealObjectKeys(KMS, metadata, buckets, names)
if err != nil {
return err
}
var keyIndex int
for i := range objects[:N] {
size, err := objects[i].GetActualSize()
if err != nil {
@ -165,17 +176,22 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchS
}
objects[i].Size = size
ETag, err := etag.Parse(objects[i].ETag)
if err != nil {
return err
}
if ETag.IsEncrypted() {
tag, err := keys[i].UnsealETag(ETag)
if !SSES3Objects[i] {
objects[i].ETag = objects[i].GetActualETag(nil)
} else {
ETag, err := etag.Parse(objects[i].ETag)
if err != nil {
return err
}
ETag = etag.ETag(tag)
objects[i].ETag = ETag.String()
if ETag.IsEncrypted() {
tag, err := keys[keyIndex].UnsealETag(ETag)
if err != nil {
return err
}
ETag = etag.ETag(tag)
objects[i].ETag = ETag.String()
}
keyIndex++
}
}
objects = objects[N:]