remove github.com/minio/kes as a dependency (#8665)

This commit removes github.com/minio/kes as
a dependency and implements the necessary
client-side functionality without relying
on the KES project.

This resolves the licensing issue since
KES is licensed under AGPL while MinIO
is licensed under Apache.

cmd/config-current.go

@@ -251,7 +251,7 @@ func validateConfig(s config.Config) error {
 		}
 	}
 	{
-		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get())
+		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewCustomHTTPTransport())
 		if err != nil {
 			return err
 		}
@@ -355,7 +355,7 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
-	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get())
+	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewCustomHTTPTransport())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to setup KMS config: %w", err))
 	}

cmd/crypto/config.go

@@ -17,6 +17,7 @@ package crypto
 import (
 	"errors"
 	"fmt"
+	"net/http"
 	"reflect"
 	"strconv"
 
@@ -259,7 +260,7 @@ func lookupAutoEncryption() (bool, error) {
 
 // LookupConfig lookup vault or kes config, returns KMSConfig
 // to configure KMS object for object encryption
-func LookupConfig(c config.Config, defaultRootCAsDir string) (KMSConfig, error) {
+func LookupConfig(c config.Config, defaultRootCAsDir string, transport *http.Transport) (KMSConfig, error) {
 	vcfg, err := LookupVaultConfig(c[config.KmsVaultSubSys][config.Default])
 	if err != nil {
 		return KMSConfig{}, err
@@ -268,6 +269,7 @@ func LookupConfig(c config.Config, defaultRootCAsDir string) (KMSConfig, error)
 	if err != nil {
 		return KMSConfig{}, err
 	}
+	kesCfg.Transport = transport
 	if kesCfg.Enabled && kesCfg.CAPath == "" {
 		kesCfg.CAPath = defaultRootCAsDir
 	}

cmd/crypto/kes.go

@@ -18,13 +18,16 @@ import (
 	"bytes"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
-
-	"github.com/minio/kes"
+	"strings"
 )
 
 // KesConfig contains the configuration required
@@ -63,6 +66,10 @@ type KesConfig struct {
 
 	// The default key ID returned by KMS.KeyID().
 	DefaultKeyID string
+
+	// The HTTP transport configuration for
+	// the KES client.
+	Transport *http.Transport
 }
 
 // Verify verifies if the kes configuration is correct
@@ -81,7 +88,7 @@ func (k KesConfig) Verify() (err error) {
 }
 
 type kesService struct {
-	client *kes.Client
+	client *kesClient
 
 	endpoint     string
 	defaultKeyID string
@@ -102,11 +109,17 @@ func NewKes(cfg KesConfig) (KMS, error) {
 	if err != nil {
 		return nil, err
 	}
+	cfg.Transport.TLSClientConfig = &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      certPool,
+	}
 	return &kesService{
-		client: kes.NewClient(cfg.Endpoint, &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			RootCAs:      certPool,
-		}),
+		client: &kesClient{
+			addr: cfg.Endpoint,
+			httpClient: http.Client{
+				Transport: cfg.Transport,
+			},
+		},
 		endpoint:     cfg.Endpoint,
 		defaultKeyID: cfg.DefaultKeyID,
 	}, nil
@@ -187,6 +200,110 @@ func (kes *kesService) UpdateKey(keyID string, sealedKey []byte, ctx Context) ([
 	return sealedKey, nil
 }
 
+// kesClient implements the bare minimum functionality needed for
+// MinIO to talk to a KES server. In particular, it implements
+// GenerateDataKey (API: /v1/key/generate/) and
+// DecryptDataKey  (API: /v1/key/decrypt/).
+type kesClient struct {
+	addr       string
+	httpClient http.Client
+}
+
+// GenerateDataKey requests a new data key from the KES server.
+// On success, the KES server will respond with the plaintext key
+// and the ciphertext key as the plaintext key encrypted with
+// the key specified by name.
+//
+// The optional context is crytpo. bound to the generated data key
+// such that you have to provide the same context when decrypting
+// the data key.
+func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte, error) {
+	type Request struct {
+		Context []byte `json:"context"`
+	}
+	body, err := json.Marshal(Request{
+		Context: context,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
+	resp, err := c.httpClient.Post(url, "application/json", bytes.NewReader(body))
+	if err != nil {
+		return nil, nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, nil, c.parseErrorResponse(resp)
+	}
+	defer resp.Body.Close()
+
+	type Response struct {
+		Plaintext  []byte `json:"plaintext"`
+		Ciphertext []byte `json:"ciphertext"`
+	}
+	const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
+	var response Response
+	if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(&response); err != nil {
+		return nil, nil, err
+	}
+	return response.Plaintext, response.Ciphertext, nil
+}
+
+// GenerateDataKey decrypts an encrypted data key with the key
+// specified by name by talking to the KES server.
+// On success, the KES server will respond with the plaintext key.
+//
+// The optional context must match the value you provided when
+// generating the data key.
+func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]byte, error) {
+	type Request struct {
+		Ciphertext []byte `json:"ciphertext"`
+		Context    []byte `json:"context"`
+	}
+	body, err := json.Marshal(Request{
+		Ciphertext: ciphertext,
+		Context:    context,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
+	resp, err := c.httpClient.Post(url, "application/json", bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, c.parseErrorResponse(resp)
+	}
+	defer resp.Body.Close()
+
+	type Response struct {
+		Plaintext []byte `json:"plaintext"`
+	}
+	const limit = 32 * 1024 // A data key will never be larger than 32 KB
+	var response Response
+	if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(&response); err != nil {
+		return nil, err
+	}
+	return response.Plaintext, nil
+}
+
+func (c *kesClient) parseErrorResponse(resp *http.Response) error {
+	if resp.Body == nil {
+		return nil
+	}
+	defer resp.Body.Close()
+
+	const limit = 32 * 1024 // A (valid) error response will not be greater than 32 KB
+	var errMsg strings.Builder
+	if _, err := io.Copy(&errMsg, io.LimitReader(resp.Body, limit)); err != nil {
+		return err
+	}
+	return fmt.Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
+}
+
 // loadCACertificates returns a new CertPool
 // that contains all system root CA certificates
 // and any PEM-encoded certificate(s) found at

go.mod

@@ -44,7 +44,6 @@ require (
 	github.com/minio/gokrb5/v7 v7.2.5
 	github.com/minio/hdfs/v3 v3.0.1
 	github.com/minio/highwayhash v1.0.0
-	github.com/minio/kes v0.4.0
 	github.com/minio/lsync v1.0.1
 	github.com/minio/mc v0.0.0-20191012041914-735aa139b19c
 	github.com/minio/minio-go v0.0.0-20190327203652-5325257a208f

go.sum

@@ -563,8 +563,6 @@ github.com/minio/hdfs/v3 v3.0.1/go.mod h1:6ALh9HsAwG9xAXdpdrZJcSY0vR6z3K+9XIz6Y9
 github.com/minio/highwayhash v0.0.0-20181220011308-93ed73d64169/go.mod h1:NL8wme5P5MoscwAkXfGroz3VgpCdhBw3KYOu5mEsvpU=
 github.com/minio/highwayhash v1.0.0 h1:iMSDhgUILCr0TNm8LWlSjF8N0ZIj2qbO8WHp6Q/J2BA=
 github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc=
-github.com/minio/kes v0.4.0 h1:N96zVEH/RnY4XfGf/gVz+j8DRyrYWDM2f24lF/K3LhA=
-github.com/minio/kes v0.4.0/go.mod h1:Y1hLsM+dhq0azrJ+7nxG1QaD8gliTUthFnvOk99FasQ=
 github.com/minio/lsync v0.0.0-20190207022115-a4e43e3d0887/go.mod h1:ni10+iSX7FO8N2rv41XM444V6w4rYO0dZo5KIkbn/YA=
 github.com/minio/lsync v1.0.1 h1:AVvILxA976xc27hstd1oR+X9PQG0sPSom1MNb1ImfUs=
 github.com/minio/lsync v1.0.1/go.mod h1:tCFzfo0dlvdGl70IT4IAK/5Wtgb0/BrTmo/jE8pArKA=
@@ -702,8 +700,6 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
-github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
-github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4 v2.2.6+incompatible h1:6aCX4/YZ9v8q69hTyiR7dNLnTA3fgtKHVVW5BCd5Znw=