diff --git a/cmd/bucket-listobjects-handlers.go b/cmd/bucket-listobjects-handlers.go index 3ffe118df..c3472a72d 100644 --- a/cmd/bucket-listobjects-handlers.go +++ b/cmd/bucket-listobjects-handlers.go @@ -24,28 +24,12 @@ import ( "strings" "github.com/gorilla/mux" + "github.com/minio/minio/internal/kms" "github.com/minio/minio/internal/logger" - "github.com/minio/minio/internal/sync/errgroup" "github.com/minio/pkg/bucket/policy" ) -func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) { - g := errgroup.WithNErrs(len(objects)).WithConcurrency(500) - for index := range objects { - index := index - g.Go(func() error { - size, err := objects[index].GetActualSize() - if err == nil { - objects[index].Size = size - } - objects[index].ETag = objects[index].GetActualETag(nil) - return nil - }, index) - } - g.Wait() -} - // Validate all the ListObjects query arguments, returns an APIErrorCode // if one of the args do not meet the required conditions. // Special conditions required by MinIO server are as below @@ -116,7 +100,10 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r return } - concurrentDecryptETag(ctx, listObjectVersionsInfo.Objects) + if err = DecryptETags(ctx, GlobalKMS, listObjectVersionsInfo.Objects, kms.BatchSize()); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo) @@ -178,7 +165,10 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt return } - concurrentDecryptETag(ctx, listObjectsV2Info.Objects) + if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects, kms.BatchSize()); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } // The next continuation token has id@node_index format to optimize paginated listing nextContinuationToken := listObjectsV2Info.NextContinuationToken @@ -253,7 +243,10 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http return } - concurrentDecryptETag(ctx, listObjectsV2Info.Objects) + if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects, kms.BatchSize()); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter, delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, @@ -350,7 +343,10 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http return } - concurrentDecryptETag(ctx, listObjectsInfo.Objects) + if err = DecryptETags(ctx, GlobalKMS, listObjectsInfo.Objects, kms.BatchSize()); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo) diff --git a/cmd/encryption-v1.go b/cmd/encryption-v1.go index bd7685f9d..1b50e05b8 100644 --- a/cmd/encryption-v1.go +++ b/cmd/encryption-v1.go @@ -19,6 +19,7 @@ package cmd import ( "bufio" + "context" "crypto/hmac" "crypto/rand" "crypto/sha256" @@ -35,6 +36,7 @@ import ( "github.com/minio/kes" "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/etag" "github.com/minio/minio/internal/fips" xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/kms" @@ -80,6 +82,7 @@ func (o *MultipartInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDe // metadata, if any. It returns an empty ID if no key ID is // present. func kmsKeyIDFromMetadata(metadata map[string]string) string { + const ARNPrefix = "arn:aws:kms:" if len(metadata) == 0 { return "" } @@ -87,10 +90,96 @@ func kmsKeyIDFromMetadata(metadata map[string]string) string { if !ok { return "" } - if strings.HasPrefix(kmsID, "arn:aws:kms:") { + if strings.HasPrefix(kmsID, ARNPrefix) { return kmsID } - return "arn:aws:kms:" + kmsID + return ARNPrefix + kmsID +} + +// DecryptETags dectypts all ObjectInfo ETags, if encrypted, using the KMS. +func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo, batchSize int) error { + var ( + metadata []map[string]string + buckets []string + names []string + ) + for len(objects) > 0 { + var N int + if len(objects) < batchSize { + N = len(objects) + } else { + N = batchSize + } + + SSES3Batch := true + for _, object := range objects[:N] { + if kind, ok := crypto.IsEncrypted(object.UserDefined); !ok || kind != crypto.S3 { + SSES3Batch = false + break + } + } + if !SSES3Batch { + for i := range objects[:N] { + size, err := objects[i].GetActualSize() + if err != nil { + return err + } + objects[i].Size = size + objects[i].ETag = objects[i].GetActualETag(nil) + } + objects = objects[N:] + continue + } + + // Now, decrypt all ETags using the a specialized bulk decryption API, if available. + // We check the cap of all slices first, to avoid allocating them over and over again. + if cap(metadata) >= N { + metadata = metadata[:0:N] + } else { + metadata = make([]map[string]string, 0, N) + } + if cap(buckets) >= N { + buckets = buckets[:0:N] + } else { + buckets = make([]string, 0, N) + } + if cap(names) >= N { + names = names[:0:N] + } else { + names = make([]string, 0, N) + } + for _, object := range objects[:N] { + metadata = append(metadata, object.UserDefined) + buckets = append(buckets, object.Bucket) + names = append(names, object.Name) + } + keys, err := crypto.S3.UnsealObjectKeys(KMS, metadata, buckets, names) + if err != nil { + return err + } + + for i := range objects[:N] { + size, err := objects[i].GetActualSize() + if err != nil { + return err + } + ETag, err := etag.Parse(objects[i].ETag) + if err != nil { + return err + } + if ETag.IsEncrypted() { + tag, err := keys[i].UnsealETag(ETag) + if err != nil { + return err + } + ETag = etag.ETag(tag) + objects[i].Size = size + objects[i].ETag = ETag.String() + } + } + objects = objects[N:] + } + return nil } // isMultipart returns true if the current object is diff --git a/docs/kms/kes-config.toml b/docs/kms/kes-config.toml deleted file mode 100644 index 5083d5717..000000000 --- a/docs/kms/kes-config.toml +++ /dev/null @@ -1,33 +0,0 @@ -# The address:port of the kes server - i.e. on the local machine. -address = "127.0.0.1:7373" - -[tls] -key = "./kes-tls.key" -cert = "./kes-tls.crt" - -[policy.minio] -paths = [ - "/v1/key/create/minio-*", - "/v1/key/generate/minio-*", - "/v1/key/decrypt/minio-*" - ] -identities = [ "dd46485bedc9ad2909d2e8f9017216eec4413bc5c64b236d992f7ec19c843c5f" ] - -[cache.expiry] -all = "5m" -unused = "20s" - -[keystore.vault] -address = "https://127.0.0.1:8200" # The Vault endpoint - i.e. https://127.0.0.1:8200 -name = "minio" # The domain resp. prefix at Vault's K/V backend - -[keystore.vault.approle] -id = "" # Your AppRole Role ID -secret = "" # Your AppRole Secret ID -retry = "15s" # Duration until the server tries to re-authenticate after connection loss. - -[keystore.vault.tls] -ca = "./vault-tls.crt" # Since we use self-signed certificates - -[keystore.vault.status] -ping = "10s" diff --git a/go.mod b/go.mod index 561bb2913..8e834800c 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/minio/csvparser v1.0.0 github.com/minio/dperf v0.3.4 github.com/minio/highwayhash v1.0.2 - github.com/minio/kes v0.18.0 + github.com/minio/kes v0.19.0 github.com/minio/madmin-go v1.3.5 github.com/minio/minio-go/v7 v7.0.23 github.com/minio/parquet-go v1.1.0 diff --git a/go.sum b/go.sum index 954cfa230..861d25dda 100644 --- a/go.sum +++ b/go.sum @@ -1090,8 +1090,9 @@ github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEX github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= -github.com/minio/kes v0.18.0 h1:HryN2oAXc/xCwr+LzsU+P/xnrsLuIRJBun3EU8PuJaw= github.com/minio/kes v0.18.0/go.mod h1:zrimNafasyumYOBjHonPLLIUgCuRzCef0uQ6IojITZA= +github.com/minio/kes v0.19.0 h1:rKzkDXT4ay7FBW34KgXK+y85bie4x4Oiq29ONRuMzh0= +github.com/minio/kes v0.19.0/go.mod h1:e9YGKbwFCV7LbqNPMfZBazfNUsFGJ5LG4plSeWL8mmg= github.com/minio/madmin-go v1.1.23/go.mod h1:wv8zCroSCnpjjQdmgsdJEkFH2oD4w9J40OZqbhxjiJ4= github.com/minio/madmin-go v1.3.0/go.mod h1:b+BL64YlLY/NnE/LCPGbSgIcNX6WSWHx8BOb9wrYShk= github.com/minio/madmin-go v1.3.5 h1:YbDc4Q1oAjeGCss1u4j29kVgwJDLzoohgIGebAaLBXc= diff --git a/internal/crypto/sse-s3.go b/internal/crypto/sse-s3.go index 60c3b81d4..dc7815805 100644 --- a/internal/crypto/sse-s3.go +++ b/internal/crypto/sse-s3.go @@ -84,6 +84,63 @@ func (s3 sses3) UnsealObjectKey(KMS kms.KMS, metadata map[string]string, bucket, return key, err } +// UnsealObjectsKeys extracts and decrypts all sealed object keys +// from the metadata using the KMS and returns the decrypted object +// keys. +// +// The metadata, buckets and objects slices must have the same length. +func (s3 sses3) UnsealObjectKeys(KMS kms.KMS, metadata []map[string]string, buckets, objects []string) ([]ObjectKey, error) { + if len(metadata) != len(buckets) || len(metadata) != len(objects) { + return nil, Errorf("invalid metadata/object count: %d != %d != %d", len(metadata), len(buckets), len(objects)) + } + + keyIDs := make([]string, 0, len(metadata)) + kmsKeys := make([][]byte, 0, len(metadata)) + sealedKeys := make([]SealedKey, 0, len(metadata)) + + sameKeyID := true + for i := range metadata { + keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata[i]) + if err != nil { + return nil, err + } + keyIDs = append(keyIDs, keyID) + kmsKeys = append(kmsKeys, kmsKey) + sealedKeys = append(sealedKeys, sealedKey) + + if i > 0 && keyID != keyIDs[i-1] { + sameKeyID = false + } + } + if sameKeyID { + contexts := make([]kms.Context, 0, len(keyIDs)) + for i := range buckets { + contexts = append(contexts, kms.Context{buckets[i]: path.Join(buckets[i], objects[i])}) + } + unsealKeys, err := KMS.DecryptAll(keyIDs[0], kmsKeys, contexts) + if err != nil { + return nil, err + } + keys := make([]ObjectKey, len(unsealKeys)) + for i := range keys { + if err := keys[i].Unseal(unsealKeys[i], sealedKeys[i], s3.String(), buckets[i], objects[i]); err != nil { + return nil, err + } + } + return keys, nil + } + + keys := make([]ObjectKey, 0, len(keyIDs)) + for i := range keyIDs { + key, err := s3.UnsealObjectKey(KMS, metadata[i], buckets[i], objects[i]) + if err != nil { + return nil, err + } + keys = append(keys, key) + } + return keys, nil +} + // CreateMetadata encodes the sealed object key into the metadata and returns // the modified metadata. If the keyID and the kmsKey is not empty it encodes // both into the metadata as well. It allocates a new metadata map if metadata diff --git a/internal/kms/kes.go b/internal/kms/kes.go index 3ec5b0c21..43233c701 100644 --- a/internal/kms/kes.go +++ b/internal/kms/kes.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "crypto/x509" "errors" + "strings" "time" "github.com/minio/kes" @@ -69,15 +70,30 @@ func NewWithConfig(config Config) (KMS, error) { ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize), }) client.Endpoints = endpoints + + var bulkAvailable bool + _, policy, err := client.DescribeSelf(context.Background()) + if err == nil { + const BulkAPI = "/v1/key/bulk/decrypt/" + for _, allow := range policy.Allow { + if strings.HasPrefix(allow, BulkAPI) { + bulkAvailable = true + break + } + } + } return &kesClient{ - client: client, - defaultKeyID: config.DefaultKeyID, + client: client, + defaultKeyID: config.DefaultKeyID, + bulkAvailable: bulkAvailable, }, nil } type kesClient struct { defaultKeyID string client *kes.Client + + bulkAvailable bool } var _ KMS = (*kesClient)(nil) // compiler check @@ -145,3 +161,38 @@ func (c *kesClient) DecryptKey(keyID string, ciphertext []byte, ctx Context) ([] } return c.client.Decrypt(context.Background(), keyID, ciphertext, ctxBytes) } + +func (c *kesClient) DecryptAll(keyID string, ciphertexts [][]byte, contexts []Context) ([][]byte, error) { + if c.bulkAvailable { + CCPs := make([]kes.CCP, 0, len(ciphertexts)) + for i := range ciphertexts { + bCtx, err := contexts[i].MarshalText() + if err != nil { + return nil, err + } + CCPs = append(CCPs, kes.CCP{ + Ciphertext: ciphertexts[i], + Context: bCtx, + }) + } + PCPs, err := c.client.DecryptAll(context.Background(), keyID, CCPs...) + if err != nil { + return nil, err + } + plaintexts := make([][]byte, 0, len(PCPs)) + for _, p := range PCPs { + plaintexts = append(plaintexts, p.Plaintext) + } + return plaintexts, nil + } + + plaintexts := make([][]byte, 0, len(ciphertexts)) + for i := range ciphertexts { + plaintext, err := c.DecryptKey(keyID, ciphertexts[i], contexts[i]) + if err != nil { + return nil, err + } + plaintexts = append(plaintexts, plaintext) + } + return plaintexts, nil +} diff --git a/internal/kms/kms.go b/internal/kms/kms.go index d1eb8c4b5..e37dfb886 100644 --- a/internal/kms/kms.go +++ b/internal/kms/kms.go @@ -20,8 +20,10 @@ package kms import ( "encoding" "encoding/json" + "strconv" jsoniter "github.com/json-iterator/go" + "github.com/minio/pkg/env" ) // KMS is the generic interface that abstracts over @@ -51,6 +53,23 @@ type KMS interface { // by the key ID. The context must match the context value // used to generate the ciphertext. DecryptKey(keyID string, ciphertext []byte, context Context) ([]byte, error) + + // DecryptAll decrypts all ciphertexts with the key referenced + // by the key ID. The contexts must match the context value + // used to generate the ciphertexts. + DecryptAll(keyID string, ciphertext [][]byte, context []Context) ([][]byte, error) +} + +// BatchSize returns the size of the batches that should be used during +// KES bulk decryption API calls. +func BatchSize() int { + const DefaultBatchSize = 500 + v := env.Get("MINIO_KMS_KES_BULK_API_BATCH_SIZE", strconv.Itoa(DefaultBatchSize)) + n, err := strconv.Atoi(v) + if err != nil { + return DefaultBatchSize + } + return n } // Status describes the current state of a KMS. diff --git a/internal/kms/single-key.go b/internal/kms/single-key.go index 8cafd429d..b55596320 100644 --- a/internal/kms/single-key.go +++ b/internal/kms/single-key.go @@ -224,6 +224,18 @@ func (kms secretKey) DecryptKey(keyID string, ciphertext []byte, context Context return plaintext, nil } +func (kms secretKey) DecryptAll(keyID string, ciphertexts [][]byte, contexts []Context) ([][]byte, error) { + plaintexts := make([][]byte, 0, len(ciphertexts)) + for i := range ciphertexts { + plaintext, err := kms.DecryptKey(keyID, ciphertexts[i], contexts[i]) + if err != nil { + return nil, err + } + plaintexts = append(plaintexts, plaintext) + } + return plaintexts, nil +} + type encryptedKey struct { Algorithm string `json:"aead"` IV []byte `json:"iv"`