MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 22:25:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

listing: decrypt only SSE-S3 single-part ETags (#14638)

Browse Source 
This commit optimises the ETag decryption when
listing objects.

When MinIO lists objects, it has to decrypt the
ETags of single-part SSE-S3 objects.

It does not need to decrypt ETags of
 - plaintext objects => Their ETag is not encrypted
 - SSE-C objects     => Their ETag is not the content MD5
 - SSE-KMS objects   => Their ETag is not the content MD5
 - multipart objects => Their ETag is not encrypted

Hence, MinIO only needs to make a call to the KMS
when it needs to decrypt a single-part SSE-S3 object.
It can resolve the ETags off all other object types
locally.

This commit implements the above semantics by
processing an object listing in batches.
If the batch contains no single-part SSE-S3 object,
then no KMS calls will be made.

If the batch contains at least one single-part
SSE-S3 object we have to make at least one KMS call.
No we first filter all single-part SSE-S3 objects
such that we only request the decryption keys for
these objects.
Once we know which objects resp. ETags require a
decryption key, MinIO either uses the KES bulk
decryption API (if supported) or decrypts each
ETag serially.

This commit is a significant improvement compared
to the previous listing code. Before, a single
non-SSE-S3 object caused MinIO to fall-back to
a serial ETag decryption.
For example, if a batch consisted of 249 SSE-S3
objects and one single SSE-KMS object, MinIO would
send 249 requests to the KMS.
Now, MinIO will send a single request for exactly
those 249 objects and skip the one SSE-KMS object
since it can handle its ETag locally.

Further, MinIO would request decryption keys
for SSE-S3 multipart objects in the past - even
though multipart ETags are not encrypted.
So, if a bucket contained only multipart SSE-S3
objects, MinIO would make totally unnecessary
requests to the KMS.
Now, MinIO simply skips these multipart objects
since it can handle the ETags locally.

Signed-off-by: Andreas Auernhammer <hi@aead.dev>
This commit is contained in:
Andreas Auernhammer 2022-03-28 03:34:11 +02:00 committed by GitHub
parent 908eb57795
commit 04df69f633
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 53 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
cmd/encryption-v1.go
View File

@ -99,9 +99,9 @@ func kmsKeyIDFromMetadata(metadata map[string]string) string {
// DecryptETags dectypts all ObjectInfo ETags, if encrypted, using the KMS. // DecryptETags dectypts all ObjectInfo ETags, if encrypted, using the KMS.
func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchSize int) error { func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchSize int) error {
var ( var (
metadata []map[string]string metadata = make([]map[string]string, 0, batchSize)
buckets []string buckets = make([]string, 0, batchSize)
names []string names = make([]string, 0, batchSize)
) )
for len(objects) > 0 { for len(objects) > 0 {
var N int var N int
@ -111,14 +111,20 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchS
N = batchSize N = batchSize
} }
SSES3Batch := true // We have to conntect the KMS only if there is at least
// one SSE-S3 single-part object. SSE-C and SSE-KMS objects
// don't return the plaintext MD5 ETag and the ETag of
// SSE-S3 multipart objects is not encrypted.
// Therefore, we can skip the expensive KMS calls whenever
// there is no single-part SSE-S3 object entirely.
var containsSSES3SinglePart bool
for _, object := range objects[:N] { for _, object := range objects[:N] {
if kind, ok := crypto.IsEncrypted(object.UserDefined); !ok || kind != crypto.S3 { if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) {
SSES3Batch = false containsSSES3SinglePart = true
break break
} }
} }
if !SSES3Batch { if !containsSSES3SinglePart {
for i := range objects[:N] { for i := range objects[:N] {
size, err := objects[i].GetActualSize() size, err := objects[i].GetActualSize()
if err != nil { if err != nil {
@ -131,33 +137,38 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchS
continue continue
} }
// Now, decrypt all ETags using the a specialized bulk decryption API, if available. // Now, there are some SSE-S3 single-part objects.
// We check the cap of all slices first, to avoid allocating them over and over again. // We only request the decryption keys for them.
if cap(metadata) >= N { // We don't want to get the decryption keys for multipart
metadata = metadata[:0:N] // or non-SSE-S3 objects.
} else { //
metadata = make([]map[string]string, 0, N) // Therefore, we keep a map of indicies to remember which
} // object was an SSE-S3 single-part object.
if cap(buckets) >= N { // Then we request the decryption keys for these objects.
buckets = buckets[:0:N] // Finally, we decrypt the ETags of these objects using
} else { // the decryption keys.
buckets = make([]string, 0, N) // However, we must also adjust the size and ETags of all
} // objects (not just the SSE-S3 single part objects).
if cap(names) >= N { // For example, the ETag of SSE-KMS objects are random values
names = names[:0:N] // and the size of an SSE-KMS object must be adjusted as well.
} else { SSES3Objects := make(map[int]bool, 10)
names = make([]string, 0, N) metadata = metadata[:0:N]
} buckets = buckets[:0:N]
for _, object := range objects[:N] { names = names[:0:N]
metadata = append(metadata, object.UserDefined) for i, object := range objects[:N] {
buckets = append(buckets, object.Bucket) if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) {
names = append(names, object.Name) metadata = append(metadata, object.UserDefined)
buckets = append(buckets, object.Bucket)
names = append(names, object.Name)
SSES3Objects[i] = true
}
} }
keys, err := crypto.S3.UnsealObjectKeys(KMS, metadata, buckets, names) keys, err := crypto.S3.UnsealObjectKeys(KMS, metadata, buckets, names)
if err != nil { if err != nil {
return err return err
} }
var keyIndex int
for i := range objects[:N] { for i := range objects[:N] {
size, err := objects[i].GetActualSize() size, err := objects[i].GetActualSize()
if err != nil { if err != nil {
@ -165,17 +176,22 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchS
} }
objects[i].Size = size objects[i].Size = size
ETag, err := etag.Parse(objects[i].ETag) if !SSES3Objects[i] {
if err != nil { objects[i].ETag = objects[i].GetActualETag(nil)
return err } else {
} ETag, err := etag.Parse(objects[i].ETag)
if ETag.IsEncrypted() {
tag, err := keys[i].UnsealETag(ETag)
if err != nil { if err != nil {
return err return err
} }
ETag = etag.ETag(tag) if ETag.IsEncrypted() {
objects[i].ETag = ETag.String() tag, err := keys[keyIndex].UnsealETag(ETag)
if err != nil {
return err
}
ETag = etag.ETag(tag)
objects[i].ETag = ETag.String()
}
keyIndex++
} }
} }
objects = objects[N:] objects = objects[N:]