mirror of
https://github.com/minio/minio.git
synced 2025-11-07 04:42:56 -05:00
refactor vault configuration and add master-key KMS (#6488)
This refactors the vault configuration by moving the vault-related environment variables to `environment.go` (Other ENV should follow in the future to have a central place for adding / handling ENV instead of magic constants and handling across different files) Further this commit adds master-key SSE-S3 support. The operator can specify a SSE-S3 master key using `MINIO_SSE_MASTER_KEY` which will be used as master key to derive and encrypt per-object keys for SSE-S3 requests. This commit is also a pre-condition for SSE-S3 auto-encyption support. Fixes #6329
This commit is contained in:
Andreas Auernhammer
committed by
Nitish Tiwari
Nitish Tiwari
parent
79b9a9ce46
commit
21d8c0fd13
@@ -19,200 +19,133 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
vault "github.com/hashicorp/vault/api"
|
||||
)
|
||||
|
||||
const (
|
||||
// vaultEndpointEnv Vault endpoint environment variable
|
||||
vaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
|
||||
// vaultAuthTypeEnv type of vault auth to be used
|
||||
vaultAuthTypeEnv = "MINIO_SSE_VAULT_AUTH_TYPE"
|
||||
// vaultAppRoleIDEnv Vault AppRole ID environment variable
|
||||
vaultAppRoleIDEnv = "MINIO_SSE_VAULT_APPROLE_ID"
|
||||
// vaultAppSecretIDEnv Vault AppRole Secret environment variable
|
||||
vaultAppSecretIDEnv = "MINIO_SSE_VAULT_APPROLE_SECRET"
|
||||
// vaultKeyVersionEnv Vault Key Version environment variable
|
||||
vaultKeyVersionEnv = "MINIO_SSE_VAULT_KEY_VERSION"
|
||||
// vaultKeyNameEnv Vault Encryption Key Name environment variable
|
||||
vaultKeyNameEnv = "MINIO_SSE_VAULT_KEY_NAME"
|
||||
|
||||
// vaultCAPath is the path to a directory of PEM-encoded CA
|
||||
// cert files to verify the Vault server SSL certificate.
|
||||
vaultCAPath = "MINIO_SSE_VAULT_CAPATH"
|
||||
)
|
||||
|
||||
var (
|
||||
//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
|
||||
ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
|
||||
)
|
||||
|
||||
// VaultKey represents vault encryption key-ring.
|
||||
type VaultKey struct {
|
||||
Name string `json:"name"` // The name of the encryption key-ring
|
||||
Version int `json:"version"` // The key version
|
||||
}
|
||||
|
||||
// VaultAuth represents vault authentication type.
|
||||
// Currently the only supported authentication type is AppRole.
|
||||
type VaultAuth struct {
|
||||
Type string `json:"type"` // The authentication type
|
||||
AppRole VaultAppRole `json:"approle"` // The AppRole authentication credentials
|
||||
}
|
||||
|
||||
// VaultAppRole represents vault AppRole authentication credentials
|
||||
type VaultAppRole struct {
|
||||
ID string `json:"id"` // The AppRole access ID
|
||||
Secret string `json:"secret"` // The AppRole secret
|
||||
}
|
||||
|
||||
// VaultConfig represents vault configuration.
|
||||
type VaultConfig struct {
|
||||
Endpoint string `json:"endpoint"` // The vault API endpoint as URL
|
||||
CAPath string `json:"-"` // The path to PEM-encoded certificate files used for mTLS. Currently not used in config file.
|
||||
Auth VaultAuth `json:"auth"` // The vault authentication configuration
|
||||
Key VaultKey `json:"key-id"` // The named key used for key-generation / decryption.
|
||||
Namespace string `json:"-"` // The vault namespace of enterprise vault instances
|
||||
}
|
||||
|
||||
// vaultService represents a connection to a vault KMS.
|
||||
type vaultService struct {
|
||||
config *VaultConfig
|
||||
client *vault.Client
|
||||
leaseDuration time.Duration
|
||||
}
|
||||
|
||||
// return transit secret engine's path for generate data key operation
|
||||
func (v *vaultService) genDataKeyEndpoint(key string) string {
|
||||
return "/transit/datakey/plaintext/" + key
|
||||
var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
|
||||
|
||||
// empty/default vault configuration used to check whether a particular is empty.
|
||||
var emptyVaultConfig = VaultConfig{}
|
||||
|
||||
// IsEmpty returns true if the vault config struct is an
|
||||
// empty configuration.
|
||||
func (v *VaultConfig) IsEmpty() bool { return *v == emptyVaultConfig }
|
||||
|
||||
// Verify returns a nil error if the vault configuration
|
||||
// is valid. A valid configuration is either empty or
|
||||
// contains valid non-default values.
|
||||
func (v *VaultConfig) Verify() (err error) {
|
||||
if v.IsEmpty() {
|
||||
return // an empty configuration is valid
|
||||
}
|
||||
switch {
|
||||
case v.Endpoint == "":
|
||||
err = errors.New("crypto: missing hashicorp vault endpoint")
|
||||
case strings.ToLower(v.Auth.Type) != "approle":
|
||||
err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
|
||||
case v.Auth.AppRole.ID == "":
|
||||
err = errors.New("crypto: missing hashicorp vault AppRole ID")
|
||||
case v.Auth.AppRole.Secret == "":
|
||||
err = errors.New("crypto: missing hashicorp vault AppSecret ID")
|
||||
case v.Key.Name == "":
|
||||
err = errors.New("crypto: missing hashicorp vault key name")
|
||||
case v.Key.Version < 0:
|
||||
err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// return transit secret engine's path for decrypt operation
|
||||
func (v *vaultService) decryptEndpoint(key string) string {
|
||||
return "/transit/decrypt/" + key
|
||||
}
|
||||
// NewVault initializes Hashicorp Vault KMS by authenticating
|
||||
// to Vault with the credentials in config and gets a client
|
||||
// token for future api calls.
|
||||
func NewVault(config VaultConfig) (KMS, error) {
|
||||
if config.IsEmpty() {
|
||||
return nil, errors.New("crypto: the hashicorp vault configuration must not be empty")
|
||||
}
|
||||
if err := config.Verify(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// VaultKey represents vault encryption key-id name & version
|
||||
type VaultKey struct {
|
||||
Name string `json:"name"`
|
||||
Version int `json:"version"`
|
||||
}
|
||||
vaultCfg := vault.Config{Address: config.Endpoint}
|
||||
if err := vaultCfg.ConfigureTLS(&vault.TLSConfig{CAPath: config.CAPath}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := vault.NewClient(&vaultCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.Namespace != "" {
|
||||
client.SetNamespace(config.Namespace)
|
||||
}
|
||||
|
||||
// VaultAuth represents vault auth type to use. For now, AppRole is the only supported
|
||||
// auth type.
|
||||
type VaultAuth struct {
|
||||
Type string `json:"type"`
|
||||
AppRole VaultAppRole `json:"approle"`
|
||||
}
|
||||
|
||||
// VaultAppRole represents vault approle credentials
|
||||
type VaultAppRole struct {
|
||||
ID string `json:"id"`
|
||||
Secret string `json:"secret"`
|
||||
}
|
||||
|
||||
// VaultConfig holds config required to start vault service
|
||||
type VaultConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Auth VaultAuth `json:"auth"`
|
||||
Key VaultKey `json:"key-id"`
|
||||
}
|
||||
|
||||
// validate whether all required env variables needed to start vault service have
|
||||
// been set
|
||||
func validateVaultConfig(c *VaultConfig) error {
|
||||
if c.Endpoint == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", vaultEndpointEnv)
|
||||
payload := map[string]interface{}{
|
||||
"role_id": config.Auth.AppRole.ID,
|
||||
"secret_id": config.Auth.AppRole.Secret,
|
||||
}
|
||||
if strings.ToLower(c.Auth.Type) != "approle" {
|
||||
return fmt.Errorf("Unsupported hashicorp vault auth type - %s", vaultAuthTypeEnv)
|
||||
}
|
||||
if c.Auth.AppRole.ID == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault AppRole ID - %s is empty", vaultAppRoleIDEnv)
|
||||
}
|
||||
if c.Auth.AppRole.Secret == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault AppSecret ID - %s is empty", vaultAppSecretIDEnv)
|
||||
}
|
||||
if c.Key.Name == "" {
|
||||
return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyNameEnv)
|
||||
}
|
||||
if c.Key.Version < 0 {
|
||||
return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyVersionEnv)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// authenticate to vault with app role id and app role secret, and get a client access token, lease duration
|
||||
func getVaultAccessToken(client *vault.Client, appRoleID, appSecret string) (token string, duration int, err error) {
|
||||
data := map[string]interface{}{
|
||||
"role_id": appRoleID,
|
||||
"secret_id": appSecret,
|
||||
}
|
||||
resp, e := client.Logical().Write("auth/approle/login", data)
|
||||
if e != nil {
|
||||
return token, duration, e
|
||||
resp, err := client.Logical().Write("auth/approle/login", payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Auth == nil {
|
||||
return token, duration, ErrKMSAuthLogin
|
||||
return nil, ErrKMSAuthLogin
|
||||
}
|
||||
return resp.Auth.ClientToken, resp.Auth.LeaseDuration, nil
|
||||
|
||||
client.SetToken(resp.Auth.ClientToken)
|
||||
v := &vaultService{client: client, config: &config, leaseDuration: time.Duration(resp.Auth.LeaseDuration)}
|
||||
v.renewToken()
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// NewVaultConfig sets KMSConfig from environment
|
||||
// variables and performs validations.
|
||||
func NewVaultConfig() (KMSConfig, error) {
|
||||
kc := KMSConfig{}
|
||||
endpoint := os.Getenv(vaultEndpointEnv)
|
||||
roleID := os.Getenv(vaultAppRoleIDEnv)
|
||||
roleSecret := os.Getenv(vaultAppSecretIDEnv)
|
||||
keyName := os.Getenv(vaultKeyNameEnv)
|
||||
keyVersion := 0
|
||||
authType := "approle"
|
||||
if versionStr := os.Getenv(vaultKeyVersionEnv); versionStr != "" {
|
||||
version, err := strconv.Atoi(versionStr)
|
||||
if err != nil {
|
||||
return kc, fmt.Errorf("Unable to parse %s value (`%s`)", vaultKeyVersionEnv, versionStr)
|
||||
}
|
||||
keyVersion = version
|
||||
}
|
||||
// return if none of the vault env variables are configured
|
||||
if (endpoint == "") && (roleID == "") && (roleSecret == "") && (keyName == "") && (keyVersion == 0) {
|
||||
return kc, nil
|
||||
}
|
||||
c := VaultConfig{
|
||||
Endpoint: endpoint,
|
||||
Auth: VaultAuth{
|
||||
Type: authType,
|
||||
AppRole: VaultAppRole{
|
||||
ID: roleID,
|
||||
Secret: roleSecret,
|
||||
},
|
||||
},
|
||||
Key: VaultKey{
|
||||
Version: keyVersion,
|
||||
Name: keyName,
|
||||
},
|
||||
}
|
||||
if err := validateVaultConfig(&c); err != nil {
|
||||
return kc, err
|
||||
}
|
||||
kc.Vault = c
|
||||
return kc, nil
|
||||
}
|
||||
|
||||
// NewVault initializes Hashicorp Vault KMS by
|
||||
// authenticating to Vault with the credentials in KMSConfig,
|
||||
// and gets a client token for future api calls.
|
||||
func NewVault(kmsConf KMSConfig) (KMS, error) {
|
||||
config := kmsConf.Vault
|
||||
vconfig := &vault.Config{
|
||||
Address: config.Endpoint,
|
||||
}
|
||||
if err := vconfig.ConfigureTLS(&vault.TLSConfig{
|
||||
CAPath: os.Getenv(vaultCAPath),
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := vault.NewClient(vconfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if ns, ok := os.LookupEnv("MINIO_SSE_VAULT_NAMESPACE"); ok {
|
||||
c.SetNamespace(ns)
|
||||
}
|
||||
|
||||
accessToken, leaseDuration, err := getVaultAccessToken(c, config.Auth.AppRole.ID, config.Auth.AppRole.Secret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// authenticate and get the access token
|
||||
c.SetToken(accessToken)
|
||||
v := vaultService{client: c, config: &config, leaseDuration: time.Duration(leaseDuration)}
|
||||
v.renewToken(c)
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func (v *vaultService) renewToken(c *vault.Client) {
|
||||
// renewToken starts a new go-routine which renews
|
||||
// the vault authentication token periodically.
|
||||
func (v *vaultService) renewToken() {
|
||||
retryDelay := 1 * time.Minute
|
||||
go func() {
|
||||
for {
|
||||
s, err := c.Auth().Token().RenewSelf(int(v.leaseDuration))
|
||||
s, err := v.client.Auth().Token().RenewSelf(int(v.leaseDuration))
|
||||
if err != nil {
|
||||
time.Sleep(retryDelay)
|
||||
continue
|
||||
@@ -223,48 +156,54 @@ func (v *vaultService) renewToken(c *vault.Client) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Generates a random plain text key, sealed plain text key from
|
||||
// Vault. It returns the plaintext key and sealed plaintext key on success
|
||||
// GenerateKey returns a new plaintext key, generated by the KMS,
|
||||
// and a sealed version of this plaintext key encrypted using the
|
||||
// named key referenced by keyID. It also binds the generated key
|
||||
// cryptographically to the provided context.
|
||||
func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
|
||||
contextStream := new(bytes.Buffer)
|
||||
ctx.WriteTo(contextStream)
|
||||
var contextStream bytes.Buffer
|
||||
ctx.WriteTo(&contextStream)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
|
||||
}
|
||||
s, err1 := v.client.Logical().Write(v.genDataKeyEndpoint(keyID), payload)
|
||||
|
||||
if err1 != nil {
|
||||
return key, sealedKey, err1
|
||||
s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
|
||||
if err != nil {
|
||||
return key, sealedKey, err
|
||||
}
|
||||
sealKey := s.Data["ciphertext"].(string)
|
||||
plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
|
||||
if err != nil {
|
||||
return key, sealedKey, err1
|
||||
return key, sealedKey, err
|
||||
}
|
||||
copy(key[:], []byte(plainKey))
|
||||
return key, []byte(sealKey), nil
|
||||
}
|
||||
|
||||
// unsealKMSKey unseals the sealedKey using the Vault master key
|
||||
// referenced by the keyID. The plain text key is returned on success.
|
||||
// UnsealKey returns the decrypted sealedKey as plaintext key.
|
||||
// Therefore it sends the sealedKey to the KMS which decrypts
|
||||
// it using the named key referenced by keyID and responses with
|
||||
// the plaintext key.
|
||||
//
|
||||
// The context must be same context as the one provided while
|
||||
// generating the plaintext key / sealedKey.
|
||||
func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
|
||||
contextStream := new(bytes.Buffer)
|
||||
ctx.WriteTo(contextStream)
|
||||
var contextStream bytes.Buffer
|
||||
ctx.WriteTo(&contextStream)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"ciphertext": string(sealedKey),
|
||||
"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
|
||||
}
|
||||
s, err1 := v.client.Logical().Write(v.decryptEndpoint(keyID), payload)
|
||||
if err1 != nil {
|
||||
return key, err1
|
||||
s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
base64Key := s.Data["plaintext"].(string)
|
||||
plainKey, err1 := base64.StdEncoding.DecodeString(base64Key)
|
||||
if err1 != nil {
|
||||
return key, err1
|
||||
plainKey, err := base64.StdEncoding.DecodeString(base64Key)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
copy(key[:], []byte(plainKey))
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user