crypto: add a basic KMS implementation (#6161)

This commit adds a basic KMS implementation for an
operator-specified SSE-S3 master key. The master key
is wrapped as KMS such that using SSE-S3 with master key
and SSE-S3 with KMS can use the same code.

Bindings for a remote / true KMS (like hashicorp vault)
will be added later on.

cmd/crypto/error.go

@@ -28,3 +28,9 @@ var (
 	// is not supported.
 	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
 )
+
+var (
+	// errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able
+	// to produce enough random data. This is fatal error and should cause a panic.
+	errOutOfEntropy = errors.New("Unable to read enough randomness from the system")
+)

cmd/crypto/key.go

@@ -43,13 +43,13 @@ func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
 	}
 	var nonce [32]byte
 	if _, err := io.ReadFull(random, nonce[:]); err != nil {
-		logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
 	}
 	sha := sha256.New()
 	sha.Write(extKey[:])
 	sha.Write(nonce[:])
 	sha.Sum(key[:0])
-	return
+	return key
 }
 
 // SealedKey represents a sealed object key. It can be stored
@@ -126,5 +126,5 @@ func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
 	mac := hmac.New(sha256.New, key[:])
 	mac.Write(bin[:])
 	mac.Sum(partKey[:0])
-	return
+	return partKey
 }

cmd/crypto/kms.go

@@ -0,0 +1,139 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+// Context is a list of key-value pairs cryptographically
+// associated with a certain object.
+type Context map[string]string
+
+// WriteTo writes the context in a canonical from to w.
+// It returns the number of bytes and the first error
+// encounter during writing to w, if any.
+//
+// WriteTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+func (c Context) WriteTo(w io.Writer) (n int64, err error) {
+	sortedKeys := make(sort.StringSlice, 0, len(c))
+	for k := range c {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Sort(sortedKeys)
+
+	nn, err := io.WriteString(w, "{")
+	if err != nil {
+		return n + int64(nn), err
+	}
+	n += int64(nn)
+	for i, k := range sortedKeys {
+		s := fmt.Sprintf("\"%s\":\"%s\",", k, c[k])
+		if i == len(sortedKeys)-1 {
+			s = s[:len(s)-1] // remove last ','
+		}
+
+		nn, err = io.WriteString(w, s)
+		if err != nil {
+			return n + int64(nn), err
+		}
+		n += int64(nn)
+	}
+	nn, err = io.WriteString(w, "}")
+	return n + int64(nn), err
+}
+
+// KMS represents an active and authenticted connection
+// to a Key-Management-Service. It supports generating
+// data key generation and unsealing of KMS-generated
+// data keys.
+type KMS interface {
+	// GenerateKey generates a new random data key using
+	// the master key referenced by the keyID. It returns
+	// the plaintext key and the sealed plaintext key
+	// on success.
+	//
+	// The context is cryptographically bound to the
+	// generated key. The same context must be provided
+	// again to unseal the generated key.
+	GenerateKey(keyID string, context Context) (key, sealedKey []byte, err error)
+
+	// UnsealKey unseals the sealedKey using the master key
+	// referenced by the keyID. The provided context must
+	// match the context used to generate the sealed key.
+	UnsealKey(keyID string, sealedKey []byte, context Context) (key []byte, err error)
+}
+
+type masterKeyKMS struct {
+	masterKey [32]byte
+}
+
+// NewKMS returns a basic KMS implementation from a single 256 bit master key.
+//
+// The KMS accepts any keyID but binds the keyID and context cryptographically
+// to the generated keys.
+func NewKMS(key [32]byte) KMS { return &masterKeyKMS{masterKey: key} }
+
+func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key, sealedKey []byte, err error) {
+	key = make([]byte, 32)
+	if _, err = io.ReadFull(rand.Reader, key); err != nil {
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
+	}
+
+	var (
+		buffer     bytes.Buffer
+		derivedKey = kms.deriveKey(keyID, ctx)
+	)
+	if n, err := sio.Encrypt(&buffer, bytes.NewReader(key), sio.Config{Key: derivedKey[:]}); err != nil || n != 64 {
+		logger.CriticalIf(context.Background(), errors.New("KMS: unable to encrypt data key"))
+	}
+	sealedKey = buffer.Bytes()
+	return key, sealedKey, nil
+}
+
+func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key []byte, err error) {
+	var (
+		buffer     bytes.Buffer
+		derivedKey = kms.deriveKey(keyID, ctx)
+	)
+	if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
+		return nil, err // TODO(aead): upgrade sio to use sio.Error
+	}
+	key = buffer.Bytes()
+	return key, nil
+}
+
+func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte) {
+	if context == nil {
+		context = Context{}
+	}
+	mac := hmac.New(sha256.New, kms.masterKey[:])
+	mac.Write([]byte(keyID))
+	context.WriteTo(mac)
+	mac.Sum(key[:0])
+	return key
+}

cmd/crypto/kms_test.go

@@ -0,0 +1,84 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"path"
+	"strings"
+	"testing"
+)
+
+var masterKeyKMSTests = []struct {
+	GenKeyID, UnsealKeyID     string
+	GenContext, UnsealContext Context
+
+	ShouldFail bool
+}{
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{}, UnsealContext: nil, ShouldFail: false},                                                                                     // 0
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: false},                                                               // 1
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"bucket": "object"}, ShouldFail: false},                           // 2
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: false}, // 3
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "0", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: false},                       // 4
+
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7e", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: true},                                                               // 5
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"Bucket": "object"}, ShouldFail: true},                           // 6
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "Object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: true}, // 7
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "1", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: true},                       // 8
+}
+
+func TestMasterKeyKMS(t *testing.T) {
+	kms := NewKMS([32]byte{})
+	for i, test := range masterKeyKMSTests {
+		key, sealedKey, err := kms.GenerateKey(test.GenKeyID, test.GenContext)
+		if err != nil {
+			t.Errorf("Test %d: KMS failed to generate key: %v", i, err)
+		}
+		unsealedKey, err := kms.UnsealKey(test.UnsealKeyID, sealedKey, test.UnsealContext)
+		if err != nil && !test.ShouldFail {
+			t.Errorf("Test %d: KMS failed to unseal the generated key: %v", i, err)
+		}
+		if err == nil && test.ShouldFail {
+			t.Errorf("Test %d: KMS unsealed the generated successfully but should have failed", i)
+		}
+		if !test.ShouldFail && !bytes.Equal(key, unsealedKey) {
+			t.Errorf("Test %d: The generated and unsealed key differ", i)
+		}
+	}
+}
+
+var contextWriteToTests = []struct {
+	Context      Context
+	ExpectedJSON string
+}{
+	{Context: Context{}, ExpectedJSON: "{}"},                                                    // 0
+	{Context: Context{"a": "b"}, ExpectedJSON: `{"a":"b"}`},                                     // 1
+	{Context: Context{"a": "b", "c": "d"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 2
+	{Context: Context{"c": "d", "a": "b"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 3
+	{Context: Context{"0": "1", "-": "2", ".": "#"}, ExpectedJSON: `{"-":"2",".":"#","0":"1"}`}, // 4
+}
+
+func TestContextWriteTo(t *testing.T) {
+	for i, test := range contextWriteToTests {
+		var jsonContext strings.Builder
+		if _, err := test.Context.WriteTo(&jsonContext); err != nil {
+			t.Errorf("Test %d: Failed to encode context: %v", i, err)
+			continue
+		}
+		if s := jsonContext.String(); s != test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
+		}
+	}
+}