mirror of
https://github.com/minio/minio.git
synced 2025-11-08 21:24:55 -05:00
kms: add support for MinKMS and remove some unused/broken code (#19368)
This commit adds support for MinKMS. Now, there are three KMS implementations in `internal/kms`: Builtin, MinIO KES and MinIO KMS. Adding another KMS integration required some cleanup. In particular: - Various KMS APIs that haven't been and are not used have been removed. A lot of the code was broken anyway. - Metrics are now monitored by the `kms.KMS` itself. For basic metrics this is simpler than collecting metrics for external servers. In particular, each KES server returns its own metrics and no cluster-level view. - The builtin KMS now uses the same en/decryption implemented by MinKMS and KES. It still supports decryption of the previous ciphertext format. It's backwards compatible. - Data encryption keys now include a master key version since MinKMS supports multiple versions (~4 billion in total and 10000 concurrent) per key name. Signed-off-by: Andreas Auernhammer <github@aead.dev>
This commit is contained in:
Andreas Auernhammer
committed by
GitHub
GitHub
parent
981497799a
commit
8b660e18f2
@@ -17,16 +17,393 @@
|
||||
|
||||
package kms
|
||||
|
||||
// Top level config constants for KMS
|
||||
const (
|
||||
EnvKMSSecretKey = "MINIO_KMS_SECRET_KEY"
|
||||
EnvKMSSecretKeyFile = "MINIO_KMS_SECRET_KEY_FILE"
|
||||
EnvKESEndpoint = "MINIO_KMS_KES_ENDPOINT" // One or multiple KES endpoints, separated by ','
|
||||
EnvKESKeyName = "MINIO_KMS_KES_KEY_NAME" // The default key name used for IAM data and when no key ID is specified on a bucket
|
||||
EnvKESAPIKey = "MINIO_KMS_KES_API_KEY" // Access credential for KES - API keys and private key / certificate are mutually exclusive
|
||||
EnvKESClientKey = "MINIO_KMS_KES_KEY_FILE" // Path to TLS private key for authenticating to KES with mTLS - usually prefer API keys
|
||||
EnvKESClientPassword = "MINIO_KMS_KES_KEY_PASSWORD" // Optional password to decrypt an encrypt TLS private key
|
||||
EnvKESClientCert = "MINIO_KMS_KES_CERT_FILE" // Path to TLS certificate for authenticating to KES with mTLS - usually prefer API keys
|
||||
EnvKESServerCA = "MINIO_KMS_KES_CAPATH" // Path to file/directory containing CA certificates to verify the KES server certificate
|
||||
EnvKESKeyCacheInterval = "MINIO_KMS_KEY_CACHE_INTERVAL" // Period between polls of the KES KMS Master Key cache, to prevent it from being unused and purged
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/minio/kms-go/kes"
|
||||
"github.com/minio/kms-go/kms"
|
||||
"github.com/minio/pkg/v2/certs"
|
||||
"github.com/minio/pkg/v2/ellipses"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
)
|
||||
|
||||
// Environment variables for MinIO KMS.
|
||||
const (
|
||||
EnvKMSEndpoint = "MINIO_KMS_SERVER" // List of MinIO KMS endpoints, separated by ','
|
||||
EnvKMSEnclave = "MINIO_KMS_ENCLAVE" // MinIO KMS enclave in which the key and identity exists
|
||||
EnvKMSDefaultKey = "MINIO_KMS_SSE_KEY" // Default key used for SSE-S3 or when no SSE-KMS key ID is specified
|
||||
EnvKMSAPIKey = "MINIO_KMS_API_KEY" // Credential to access the MinIO KMS.
|
||||
)
|
||||
|
||||
// Environment variables for MinIO KES.
|
||||
const (
|
||||
EnvKESEndpoint = "MINIO_KMS_KES_ENDPOINT" // One or multiple KES endpoints, separated by ','
|
||||
EnvKESDefaultKey = "MINIO_KMS_KES_KEY_NAME" // The default key name used for IAM data and when no key ID is specified on a bucket
|
||||
EnvKESAPIKey = "MINIO_KMS_KES_API_KEY" // Access credential for KES - API keys and private key / certificate are mutually exclusive
|
||||
EnvKESClientKey = "MINIO_KMS_KES_KEY_FILE" // Path to TLS private key for authenticating to KES with mTLS - usually prefer API keys
|
||||
EnvKESClientCert = "MINIO_KMS_KES_CERT_FILE" // Path to TLS certificate for authenticating to KES with mTLS - usually prefer API keys
|
||||
EnvKESServerCA = "MINIO_KMS_KES_CAPATH" // Path to file/directory containing CA certificates to verify the KES server certificate
|
||||
EnvKESClientPassword = "MINIO_KMS_KES_KEY_PASSWORD" // Optional password to decrypt an encrypt TLS private key
|
||||
)
|
||||
|
||||
// Environment variables for static KMS key.
|
||||
const (
|
||||
EnvKMSSecretKey = "MINIO_KMS_SECRET_KEY" // Static KMS key in the form "<key-name>:<base64-32byte-key>". Implements a subset of KMS/KES APIs
|
||||
EnvKMSSecretKeyFile = "MINIO_KMS_SECRET_KEY_FILE" // Path to a file to read the static KMS key from
|
||||
)
|
||||
|
||||
const (
|
||||
tlsClientSessionCacheSize = 100
|
||||
)
|
||||
|
||||
// ConnectionOptions is a structure containing options for connecting
|
||||
// to a KMS.
|
||||
type ConnectionOptions struct {
|
||||
CADir string // Path to directory (or file) containing CA certificates
|
||||
}
|
||||
|
||||
// Connect returns a new Conn to a KMS. It uses configuration from the
|
||||
// environment and returns a:
|
||||
//
|
||||
// - connection to MinIO KMS if the "MINIO_KMS_SERVER" variable is present.
|
||||
// - connection to MinIO KES if the "MINIO_KMS_KES_ENDPOINT" is present.
|
||||
// - connection to a "local" KMS implementation using a static key if the
|
||||
// "MINIO_KMS_SECRET_KEY" or "MINIO_KMS_SECRET_KEY_FILE" is present.
|
||||
//
|
||||
// It returns an error if connecting to the KMS implementation fails,
|
||||
// e.g. due to incomplete config, or when configurations for multiple
|
||||
// KMS implementations are present.
|
||||
func Connect(ctx context.Context, opts *ConnectionOptions) (*KMS, error) {
|
||||
if present, err := IsPresent(); !present || err != nil {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, errors.New("kms: no KMS configuration specified")
|
||||
}
|
||||
|
||||
lookup := func(key string) bool {
|
||||
_, ok := os.LookupEnv(key)
|
||||
return ok
|
||||
}
|
||||
switch {
|
||||
case lookup(EnvKMSEndpoint):
|
||||
rawEndpoint := env.Get(EnvKMSEndpoint, "")
|
||||
if rawEndpoint == "" {
|
||||
return nil, errors.New("kms: no KMS server endpoint provided")
|
||||
}
|
||||
endpoints, err := expandEndpoints(rawEndpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key, err := kms.ParseAPIKey(env.Get(EnvKMSAPIKey, ""))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var rootCAs *x509.CertPool
|
||||
if opts != nil && opts.CADir != "" {
|
||||
rootCAs, err = certs.GetRootCAs(opts.CADir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
client, err := kms.NewClient(&kms.Config{
|
||||
Endpoints: endpoints,
|
||||
APIKey: key,
|
||||
TLS: &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize),
|
||||
RootCAs: rootCAs,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &KMS{
|
||||
Type: MinKMS,
|
||||
DefaultKey: env.Get(EnvKMSDefaultKey, ""),
|
||||
conn: &kmsConn{
|
||||
enclave: env.Get(EnvKMSEnclave, ""),
|
||||
defaultKey: env.Get(EnvKMSDefaultKey, ""),
|
||||
client: client,
|
||||
},
|
||||
latencyBuckets: defaultLatencyBuckets,
|
||||
latency: make([]atomic.Uint64, len(defaultLatencyBuckets)),
|
||||
}, nil
|
||||
case lookup(EnvKESEndpoint):
|
||||
rawEndpoint := env.Get(EnvKESEndpoint, "")
|
||||
if rawEndpoint == "" {
|
||||
return nil, errors.New("kms: no KES server endpoint provided")
|
||||
}
|
||||
endpoints, err := expandEndpoints(rawEndpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conf := &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize),
|
||||
}
|
||||
if s := env.Get(EnvKESAPIKey, ""); s != "" {
|
||||
key, err := kes.ParseAPIKey(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cert, err := kes.GenerateCertificate(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
conf.Certificates = append(conf.Certificates, cert)
|
||||
} else {
|
||||
loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
|
||||
// Manually load the certificate and private key into memory.
|
||||
// We need to check whether the private key is encrypted, and
|
||||
// if so, decrypt it using the user-provided password.
|
||||
certBytes, err := os.ReadFile(certFile)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
|
||||
}
|
||||
keyBytes, err := os.ReadFile(keyFile)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
|
||||
}
|
||||
privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
|
||||
if len(rest) != 0 {
|
||||
return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
|
||||
}
|
||||
if x509.IsEncryptedPEMBlock(privateKeyPEM) {
|
||||
keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(EnvKESClientPassword, "")))
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
|
||||
}
|
||||
keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
|
||||
}
|
||||
certificate, err := tls.X509KeyPair(certBytes, keyBytes)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
|
||||
}
|
||||
return certificate, nil
|
||||
}
|
||||
|
||||
certificate, err := certs.NewCertificate(env.Get(EnvKESClientCert, ""), env.Get(EnvKESClientKey, ""), loadX509KeyPair)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
certificate.Watch(ctx, 15*time.Minute, syscall.SIGHUP)
|
||||
|
||||
conf.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
cert := certificate.Get()
|
||||
return &cert, nil
|
||||
}
|
||||
}
|
||||
|
||||
var caDir string
|
||||
if opts != nil {
|
||||
caDir = opts.CADir
|
||||
}
|
||||
conf.RootCAs, err = certs.GetRootCAs(env.Get(EnvKESServerCA, caDir))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := kes.NewClientWithConfig("", conf)
|
||||
client.Endpoints = endpoints
|
||||
|
||||
// Keep the default key in the KES cache to prevent availability issues
|
||||
// when MinIO restarts
|
||||
go func() {
|
||||
timer := time.NewTicker(10 * time.Second)
|
||||
defer timer.Stop()
|
||||
defaultKey := env.Get(EnvKESDefaultKey, "")
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
client.DescribeKey(ctx, defaultKey)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return &KMS{
|
||||
Type: MinKES,
|
||||
DefaultKey: env.Get(EnvKESDefaultKey, ""),
|
||||
conn: &kesConn{
|
||||
defaultKeyID: env.Get(EnvKESDefaultKey, ""),
|
||||
client: client,
|
||||
},
|
||||
latencyBuckets: defaultLatencyBuckets,
|
||||
latency: make([]atomic.Uint64, len(defaultLatencyBuckets)),
|
||||
}, nil
|
||||
default:
|
||||
var s string
|
||||
if lookup(EnvKMSSecretKeyFile) {
|
||||
b, err := os.ReadFile(env.Get(EnvKMSSecretKeyFile, ""))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s = string(b)
|
||||
} else {
|
||||
s = env.Get(EnvKMSSecretKey, "")
|
||||
}
|
||||
return ParseSecretKey(s)
|
||||
}
|
||||
}
|
||||
|
||||
// IsPresent reports whether a KMS configuration is present.
|
||||
// It returns an error if multiple KMS configurations are
|
||||
// present or if one configuration is incomplete.
|
||||
func IsPresent() (bool, error) {
|
||||
// isPresent reports whether at least one of the
|
||||
// given env. variables is present.
|
||||
isPresent := func(vars ...string) bool {
|
||||
for _, v := range vars {
|
||||
if _, ok := os.LookupEnv(v); ok {
|
||||
return ok
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// First, check which KMS/KES env. variables are present.
|
||||
// Only one set, either KMS, KES or static key must be
|
||||
// present.
|
||||
kmsPresent := isPresent(
|
||||
EnvKMSEndpoint,
|
||||
EnvKMSEnclave,
|
||||
EnvKMSAPIKey,
|
||||
EnvKMSDefaultKey,
|
||||
)
|
||||
kesPresent := isPresent(
|
||||
EnvKESEndpoint,
|
||||
EnvKESDefaultKey,
|
||||
EnvKESAPIKey,
|
||||
EnvKESClientKey,
|
||||
EnvKESClientCert,
|
||||
EnvKESClientPassword,
|
||||
EnvKESServerCA,
|
||||
)
|
||||
// We have to handle a special case for MINIO_KMS_SECRET_KEY and
|
||||
// MINIO_KMS_SECRET_KEY_FILE. The docker image always sets the
|
||||
// MINIO_KMS_SECRET_KEY_FILE - either to the argument passed to
|
||||
// the container or to a default string (e.g. "minio_master_key").
|
||||
//
|
||||
// We have to distinguish a explicit config from an implicit. Hence,
|
||||
// we unset the env. vars if they are set but empty or contain a path
|
||||
// which does not exist. The downside of this check is that if
|
||||
// MINIO_KMS_SECRET_KEY_FILE is set to a path that does not exist,
|
||||
// the server does not complain and start without a KMS config.
|
||||
//
|
||||
// Until the container image changes, this behavior has to be preserved.
|
||||
if isPresent(EnvKMSSecretKey) && os.Getenv(EnvKMSSecretKey) == "" {
|
||||
os.Unsetenv(EnvKMSSecretKey)
|
||||
}
|
||||
if isPresent(EnvKMSSecretKeyFile) {
|
||||
if filename := os.Getenv(EnvKMSSecretKeyFile); filename == "" {
|
||||
os.Unsetenv(EnvKMSSecretKeyFile)
|
||||
} else if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
|
||||
os.Unsetenv(EnvKMSSecretKeyFile)
|
||||
}
|
||||
}
|
||||
// Now, the static key env. vars are only present if they contain explicit
|
||||
// values.
|
||||
staticKeyPresent := isPresent(EnvKMSSecretKey, EnvKMSSecretKeyFile)
|
||||
|
||||
switch {
|
||||
case kmsPresent && kesPresent:
|
||||
return false, errors.New("kms: configuration for MinIO KMS and MinIO KES is present")
|
||||
case kmsPresent && staticKeyPresent:
|
||||
return false, errors.New("kms: configuration for MinIO KMS and static KMS key is present")
|
||||
case kesPresent && staticKeyPresent:
|
||||
return false, errors.New("kms: configuration for MinIO KES and static KMS key is present")
|
||||
}
|
||||
|
||||
// Next, we check that all required configuration for the concrete
|
||||
// KMS is present.
|
||||
// For example, the MinIO KMS requires an endpoint or a list of
|
||||
// endpoints and authentication credentials. However, a path to
|
||||
// CA certificates is optional.
|
||||
switch {
|
||||
default:
|
||||
return false, nil // No KMS config present
|
||||
case kmsPresent:
|
||||
if !isPresent(EnvKMSEndpoint) {
|
||||
return false, fmt.Errorf("kms: incomplete configuration for MinIO KMS: missing '%s'", EnvKMSEndpoint)
|
||||
}
|
||||
if !isPresent(EnvKMSEnclave) {
|
||||
return false, fmt.Errorf("kms: incomplete configuration for MinIO KMS: missing '%s'", EnvKMSEnclave)
|
||||
}
|
||||
if !isPresent(EnvKMSDefaultKey) {
|
||||
return false, fmt.Errorf("kms: incomplete configuration for MinIO KMS: missing '%s'", EnvKMSDefaultKey)
|
||||
}
|
||||
if !isPresent(EnvKMSAPIKey) {
|
||||
return false, fmt.Errorf("kms: incomplete configuration for MinIO KMS: missing '%s'", EnvKMSAPIKey)
|
||||
}
|
||||
return true, nil
|
||||
case staticKeyPresent:
|
||||
if isPresent(EnvKMSSecretKey) && isPresent(EnvKMSSecretKeyFile) {
|
||||
return false, fmt.Errorf("kms: invalid configuration for static KMS key: '%s' and '%s' are present", EnvKMSSecretKey, EnvKMSSecretKeyFile)
|
||||
}
|
||||
return true, nil
|
||||
case kesPresent:
|
||||
if !isPresent(EnvKESEndpoint) {
|
||||
return false, fmt.Errorf("kms: incomplete configuration for MinIO KES: missing '%s'", EnvKESEndpoint)
|
||||
}
|
||||
if !isPresent(EnvKESDefaultKey) {
|
||||
return false, fmt.Errorf("kms: incomplete configuration for MinIO KES: missing '%s'", EnvKESDefaultKey)
|
||||
}
|
||||
|
||||
if isPresent(EnvKESClientKey, EnvKESClientCert, EnvKESClientPassword) {
|
||||
if isPresent(EnvKESAPIKey) {
|
||||
return false, fmt.Errorf("kms: invalid configuration for MinIO KES: '%s' and client certificate is present", EnvKESAPIKey)
|
||||
}
|
||||
if !isPresent(EnvKESClientCert) {
|
||||
return false, fmt.Errorf("kms: incomplete configuration for MinIO KES: missing '%s'", EnvKESClientCert)
|
||||
}
|
||||
if !isPresent(EnvKESClientKey) {
|
||||
return false, fmt.Errorf("kms: incomplete configuration for MinIO KES: missing '%s'", EnvKESClientKey)
|
||||
}
|
||||
} else if !isPresent(EnvKESAPIKey) {
|
||||
return false, errors.New("kms: incomplete configuration for MinIO KES: missing authentication method")
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
func expandEndpoints(s string) ([]string, error) {
|
||||
var endpoints []string
|
||||
for _, endpoint := range strings.Split(s, ",") {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
continue
|
||||
}
|
||||
if !ellipses.HasEllipses(endpoint) {
|
||||
endpoints = append(endpoints, endpoint)
|
||||
continue
|
||||
}
|
||||
|
||||
pattern, err := ellipses.FindEllipsesPatterns(endpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kms: invalid endpoint '%s': %v", endpoint, err)
|
||||
}
|
||||
for _, p := range pattern.Expand() {
|
||||
endpoints = append(endpoints, strings.Join(p, ""))
|
||||
}
|
||||
}
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user