kms: add support for KES API keys and switch to KES Go SDK (#16617)

Signed-off-by: Andreas Auernhammer <hi@aead.dev>

cmd/admin-bucket-handlers.go

@@ -31,7 +31,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"

cmd/admin-handler-utils.go

@@ -23,7 +23,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"

cmd/bucket-encryption-handlers.go

@@ -25,7 +25,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"

cmd/common-main.go

@@ -51,7 +51,7 @@ import (
 	consoleCerts "github.com/minio/console/pkg/certs"
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -795,6 +795,15 @@ func handleKMSConfig() {
 		GlobalKMS = KMS
 	}
 	if env.IsSet(config.EnvKESEndpoint) {
+		if env.IsSet(config.EnvKESAPIKey) {
+			if env.IsSet(config.EnvKESClientKey) {
+				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", config.EnvKESAPIKey, config.EnvKESClientKey))
+			}
+			if env.IsSet(config.EnvKESClientCert) {
+				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", config.EnvKESAPIKey, config.EnvKESClientCert))
+			}
+		}
+
 		var endpoints []string
 		for _, endpoint := range strings.Split(env.Get(config.EnvKESEndpoint, ""), ",") {
 			if strings.TrimSpace(endpoint) == "" {
@@ -817,6 +826,20 @@ func handleKMSConfig() {
 			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
 		}
 
+		var kmsConf kms.Config
+		if env.IsSet(config.EnvKESAPIKey) {
+			key, err := kes.ParseAPIKey(env.Get(config.EnvKESAPIKey, ""))
+			if err != nil {
+				logger.Fatal(err, fmt.Sprintf("Failed to parse KES API key from %q", env.Get(config.EnvKESAPIKey, "")))
+			}
+			kmsConf = kms.Config{
+				Endpoints:    endpoints,
+				Enclave:      env.Get(config.EnvKESEnclave, ""),
+				DefaultKeyID: env.Get(config.EnvKESKeyName, ""),
+				APIKey:       key,
+				RootCAs:      rootCAs,
+			}
+		} else {
 			loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
 				// Manually load the certificate and private key into memory.
 				// We need to check whether the private key is encrypted, and
@@ -855,24 +878,25 @@ func handleKMSConfig() {
 			certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
 			certificate.Notify(reloadCertEvents)
 
-		defaultKeyID := env.Get(config.EnvKESKeyName, "")
+			kmsConf = kms.Config{
-		KMS, err := kms.NewWithConfig(kms.Config{
 				Endpoints:        endpoints,
 				Enclave:          env.Get(config.EnvKESEnclave, ""),
-			DefaultKeyID:     defaultKeyID,
+				DefaultKeyID:     env.Get(config.EnvKESKeyName, ""),
 				Certificate:      certificate,
 				ReloadCertEvents: reloadCertEvents,
 				RootCAs:          rootCAs,
-		})
+			}
+		}
+
+		KMS, err := kms.NewWithConfig(kmsConf)
 		if err != nil {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
-
 		// We check that the default key ID exists or try to create it otherwise.
 		// This implicitly checks that we can communicate to KES. We don't treat
 		// a policy error as failure condition since MinIO may not have the permission
 		// to create keys - just to generate/decrypt data encryption keys.
-		if err = KMS.CreateKey(context.Background(), defaultKeyID); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
+		if err = KMS.CreateKey(context.Background(), env.Get(config.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
 		GlobalKMS = KMS

cmd/encryption-v1.go

@@ -34,7 +34,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"

cmd/kms-handlers.go

@@ -23,7 +23,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"

cmd/metrics-v2.go

@@ -27,7 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/logger"

go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dperf v0.4.2
 	github.com/minio/highwayhash v1.0.2
-	github.com/minio/kes v0.22.3
+	github.com/minio/kes-go v0.1.0
 	github.com/minio/madmin-go/v2 v2.0.11
 	github.com/minio/minio-go/v7 v7.0.47
 	github.com/minio/mux v1.9.0

go.sum

@@ -233,8 +233,8 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoU
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
-github.com/armon/go-metrics v0.3.9 h1:O2sNqxBdvq8Eq5xmzljcYzAORli6RWCvEym4cJf9m18=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
@@ -584,8 +584,8 @@ github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng
 github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-hclog v1.1.0 h1:QsGcniKx5/LuX2eYoeL+Np3UKYPNaN7YKpTh29h8rbw=
 github.com/hashicorp/go-hclog v1.1.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
+github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs=
@@ -772,8 +772,8 @@ github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEX
 github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
-github.com/minio/kes v0.22.3 h1:aSPW9uCMVaLax5POxvoQJxCU4MNo/KzMXA7WfmC/lRw=
+github.com/minio/kes-go v0.1.0 h1:h201DyOYP5sTqajkxFGxmXz/kPbT8HQNX1uh3Yx2PFc=
-github.com/minio/kes v0.22.3/go.mod h1:wnhmdwWX2rpurNPKn3yDFImg2wuc7j3e+IU5rVkR9UY=
+github.com/minio/kes-go v0.1.0/go.mod h1:VorHLaIYis9/MxAHAtXN4d8PUMNKhIxTIlvFt0hBOEo=
 github.com/minio/madmin-go v1.6.6/go.mod h1:ATvkBOLiP3av4D++2v1UEHC/QzsGtgXD5kYvvRYzdKs=
 github.com/minio/madmin-go/v2 v2.0.11 h1:Ct905UAMJ43EAwKCi8xy5PzWPWyYL5YCQ441E9LYXTA=
 github.com/minio/madmin-go/v2 v2.0.11/go.mod h1:5aFi/VLWBHC2DEFfGIlUmAeJhaF4ZAjuYpEWZFU14Zw=

internal/config/constants.go

@@ -69,13 +69,14 @@ const (
 
 	EnvKMSSecretKey      = "MINIO_KMS_SECRET_KEY"
 	EnvKMSSecretKeyFile  = "MINIO_KMS_SECRET_KEY_FILE"
-	EnvKESEndpoint       = "MINIO_KMS_KES_ENDPOINT"
+	EnvKESEndpoint       = "MINIO_KMS_KES_ENDPOINT"     // One or multiple KES endpoints, separated by ','
-	EnvKESEnclave        = "MINIO_KMS_KES_ENCLAVE"
+	EnvKESEnclave        = "MINIO_KMS_KES_ENCLAVE"      // Optional "namespace" within a KES cluster - not required for stateless KES
-	EnvKESKeyName        = "MINIO_KMS_KES_KEY_NAME"
+	EnvKESKeyName        = "MINIO_KMS_KES_KEY_NAME"     // The default key name used for IAM data and when no key ID is specified on a bucket
-	EnvKESClientKey      = "MINIO_KMS_KES_KEY_FILE"
+	EnvKESAPIKey         = "MINIO_KMS_KES_API_KEY"      // Access credential for KES - API keys and private key / certificate are mutually exclusive
-	EnvKESClientPassword = "MINIO_KMS_KES_KEY_PASSWORD"
+	EnvKESClientKey      = "MINIO_KMS_KES_KEY_FILE"     // Path to TLS private key for authenticating to KES with mTLS - usually prefer API keys
-	EnvKESClientCert     = "MINIO_KMS_KES_CERT_FILE"
+	EnvKESClientPassword = "MINIO_KMS_KES_KEY_PASSWORD" // Optional password to decrypt an encrypt TLS private key
-	EnvKESServerCA       = "MINIO_KMS_KES_CAPATH"
+	EnvKESClientCert     = "MINIO_KMS_KES_CERT_FILE"    // Path to TLS certificate for authenticating to KES with mTLS - usually prefer API keys
+	EnvKESServerCA       = "MINIO_KMS_KES_CAPATH"       // Path to file/directory containing CA certificates to verify the KES server certificate
 
 	EnvEndpoints  = "MINIO_ENDPOINTS"   // legacy
 	EnvWorm       = "MINIO_WORM"        // legacy

internal/kms/identity-manager.go

@@ -20,7 +20,7 @@ package kms
 import (
 	"context"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 )
 
 // IdentityManager is the generic interface that handles KMS identity operations

internal/kms/kes.go

@@ -26,7 +26,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/pkg/certs"
 )
 
@@ -52,6 +52,11 @@ type Config struct {
 	// a cryptographic operation.
 	DefaultKeyID string
 
+	// APIKey is an credential provided by env. var.
+	// to authenticate to a KES server. Either an
+	// API key or a client certificate must be specified.
+	APIKey kes.APIKey
+
 	// Certificate is the client TLS certificate
 	// to authenticate to KMS via mTLS.
 	Certificate *certs.Certificate
@@ -74,12 +79,26 @@ func NewWithConfig(config Config) (KMS, error) {
 	endpoints := make([]string, len(config.Endpoints)) // Copy => avoid being affect by any changes to the original slice
 	copy(endpoints, config.Endpoints)
 
-	client := kes.NewClientWithConfig("", &tls.Config{
+	var client *kes.Client
+	if config.APIKey != nil {
+		cert, err := kes.GenerateCertificate(config.APIKey)
+		if err != nil {
+			return nil, err
+		}
+		client = kes.NewClientWithConfig("", &tls.Config{
+			MinVersion:         tls.VersionTLS12,
+			Certificates:       []tls.Certificate{cert},
+			RootCAs:            config.RootCAs,
+			ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize),
+		})
+	} else {
+		client = kes.NewClientWithConfig("", &tls.Config{
 			MinVersion:         tls.VersionTLS12,
 			Certificates:       []tls.Certificate{config.Certificate.Get()},
 			RootCAs:            config.RootCAs,
 			ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize),
 		})
+	}
 	client.Endpoints = endpoints
 
 	var bulkAvailable bool
@@ -101,6 +120,9 @@ func NewWithConfig(config Config) (KMS, error) {
 		bulkAvailable: bulkAvailable,
 	}
 	go func() {
+		if config.Certificate == nil || config.ReloadCertEvents == nil {
+			return
+		}
 		for {
 			var prevCertificate tls.Certificate
 			certificate, ok := <-config.ReloadCertEvents

internal/kms/key-manager.go

@@ -20,7 +20,7 @@ package kms
 import (
 	"context"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 )
 
 // KeyManager is the generic interface that handles KMS key operations

internal/kms/kms.go

@@ -23,7 +23,7 @@ import (
 	"encoding/json"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 )
 
 // KMS is the generic interface that abstracts over

internal/kms/policy-manager.go

@@ -20,7 +20,7 @@ package kms
 import (
 	"context"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 )
 
 // PolicyManager is the generic interface that handles KMS policy] operations

internal/kms/single-key.go

@@ -33,7 +33,7 @@ import (
 	"golang.org/x/crypto/chacha20"
 	"golang.org/x/crypto/chacha20poly1305"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/minio/internal/hash/sha256"
 )
 

internal/kms/status-manager.go

@@ -20,7 +20,7 @@ package kms
 import (
 	"context"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 )
 
 // StatusManager is the generic interface that handles KMS status operations