Implement auto cert reloading (#5963)

cmd/certs.go

@@ -25,6 +25,8 @@ import (
 	"encoding/pem"
 	"io/ioutil"
 	"os"
+
+	"github.com/minio/minio/pkg/certs"
 )
 
 // TLSPrivateKeyPassword is the environment variable which contains the password used
@@ -135,10 +137,19 @@ func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	if err != nil {
 		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg(err.Error())
 	}
+	// Ensure that the private key is not a P-384 or P-521 EC key.
+	// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
+	if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
+		if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
+			if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check
+				return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
+			}
+		}
+	}
 	return cert, nil
 }
 
-func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
+func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, c *certs.Certs, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
 		return nil, nil, nil, false, nil
 	}
@@ -147,27 +158,15 @@ func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsC
 		return nil, nil, nil, false, err
 	}
 
-	var cert tls.Certificate
-	if cert, err = loadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
+	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
+	if err != nil {
 		return nil, nil, nil, false, err
 	}
-	// Ensure that the private key is not a P-384 or P-521 EC key.
-	// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
-	if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
-		if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
-			if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check
-				return nil, nil, nil, false, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
-			}
-		}
-
-	}
-
-	tlsCert = &cert
 
 	if rootCAs, err = getRootCAs(getCADir()); err != nil {
 		return nil, nil, nil, false, err
 	}
 
 	secureConn = true
-	return x509Certs, rootCAs, tlsCert, secureConn, nil
+	return x509Certs, rootCAs, c, secureConn, nil
 }

cmd/gateway-main.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/cli"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/certs"
 )
 
 func init() {
@@ -164,7 +165,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// Check and load SSL certificates.
 	var err error
-	globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig()
+	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
 	logger.FatalIf(err, "Invalid SSL certificate file")
 
 	// Set system resources to maximum.
@@ -179,9 +180,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
 
-	newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
-	logger.FatalIf(err, "Unable to initialize gateway layer")
-
 	router := mux.NewRouter().SkipClean(true)
 
 	// Add healthcheck router
@@ -198,17 +196,31 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Add API router.
 	registerAPIRouter(router)
 
-	globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, registerHandlers(router, globalHandlers...), globalTLSCertificate)
+	var getCert certs.GetCertificateFunc
+	if globalTLSCerts != nil {
+		getCert = globalTLSCerts.GetCertificate
+	}
+
+	globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, registerHandlers(router, globalHandlers...), getCert)
+	globalHTTPServer.ReadTimeout = globalConnReadTimeout
+	globalHTTPServer.WriteTimeout = globalConnWriteTimeout
 	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
 	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
-
-	// Start server, automatically configures TLS if certs are available.
 	go func() {
 		globalHTTPServerErrorCh <- globalHTTPServer.Start()
 	}()
 
 	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
 
+	newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
+	if err != nil {
+		// Stop watching for any certificate changes.
+		globalTLSCerts.Stop()
+
+		globalHTTPServer.Shutdown()
+		logger.FatalIf(err, "Unable to initialize gateway backend")
+	}
+
 	// Once endpoints are finalized, initialize the new object api.
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject

cmd/globals.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"crypto/tls"
 	"crypto/x509"
 	"os"
 	"runtime"
@@ -27,6 +26,7 @@ import (
 	"github.com/fatih/color"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/certs"
 )
 
 // minio configuration related constants.
@@ -131,7 +131,7 @@ var (
 	// IsSSL indicates if the server is configured with SSL.
 	globalIsSSL bool
 
-	globalTLSCertificate *tls.Certificate
+	globalTLSCerts *certs.Certs
 
 	globalHTTPServer        *xhttp.Server
 	globalHTTPServerErrorCh = make(chan error)

cmd/http/listener_test.go

@@ -49,6 +49,14 @@ func getNextPort() string {
 	return strconv.Itoa(int(atomic.AddUint32(&serverPort, 1)))
 }
 
+var getCert = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	certificate, err := getTLSCert()
+	if err != nil {
+		return nil, err
+	}
+	return &certificate, nil
+}
+
 func getTLSCert() (tls.Certificate, error) {
 	keyPEMBlock := []byte(`-----BEGIN RSA PRIVATE KEY-----
 MIIEpAIBAAKCAQEApEkbPrT6wzcWK1W5atQiGptvuBsRdf8MCg4u6SN10QbslA5k

cmd/http/server.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,6 +26,7 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/certs"
 )
 
 const (
@@ -176,17 +177,18 @@ var defaultCipherSuites = []uint16{
 var secureCurves = []tls.CurveID{tls.X25519, tls.CurveP256}
 
 // NewServer - creates new HTTP server using given arguments.
-func NewServer(addrs []string, handler http.Handler, certificate *tls.Certificate) *Server {
+func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificateFunc) *Server {
 	var tlsConfig *tls.Config
-	if certificate != nil {
+	if getCert != nil {
 		tlsConfig = &tls.Config{
+			// TLS hardening
 			PreferServerCipherSuites: true,
 			CipherSuites:             defaultCipherSuites,
 			CurvePreferences:         secureCurves,
 			MinVersion:               tls.VersionTLS12,
 			NextProtos:               []string{"http/1.1", "h2"},
 		}
-		tlsConfig.Certificates = append(tlsConfig.Certificates, *certificate)
+		tlsConfig.GetCertificate = getCert
 	}
 
 	httpServer := &Server{

cmd/http/server_test.go

@@ -23,33 +23,31 @@ import (
 	"reflect"
 	"testing"
 	"time"
+
+	"github.com/minio/minio/pkg/certs"
 )
 
 func TestNewServer(t *testing.T) {
 	nonLoopBackIP := getNonLoopBackIP(t)
-	certificate, err := getTLSCert()
-	if err != nil {
-		t.Fatalf("Unable to parse private/certificate data. %v\n", err)
-	}
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(w, "Hello, world")
 	})
 
 	testCases := []struct {
-		addrs       []string
-		handler     http.Handler
-		certificate *tls.Certificate
+		addrs   []string
+		handler http.Handler
+		certFn  certs.GetCertificateFunc
 	}{
 		{[]string{"127.0.0.1:9000"}, handler, nil},
 		{[]string{nonLoopBackIP + ":9000"}, handler, nil},
 		{[]string{"127.0.0.1:9000", nonLoopBackIP + ":9000"}, handler, nil},
-		{[]string{"127.0.0.1:9000"}, handler, &certificate},
-		{[]string{nonLoopBackIP + ":9000"}, handler, &certificate},
-		{[]string{"127.0.0.1:9000", nonLoopBackIP + ":9000"}, handler, &certificate},
+		{[]string{"127.0.0.1:9000"}, handler, getCert},
+		{[]string{nonLoopBackIP + ":9000"}, handler, getCert},
+		{[]string{"127.0.0.1:9000", nonLoopBackIP + ":9000"}, handler, getCert},
 	}
 
 	for i, testCase := range testCases {
-		server := NewServer(testCase.addrs, testCase.handler, testCase.certificate)
+		server := NewServer(testCase.addrs, testCase.handler, testCase.certFn)
 		if server == nil {
 			t.Fatalf("Case %v: server: expected: <non-nil>, got: <nil>", (i + 1))
 		}
@@ -63,7 +61,7 @@ func TestNewServer(t *testing.T) {
 		// 	t.Fatalf("Case %v: server.Handler: expected: %v, got: %v", (i + 1), testCase.handler, server.Handler)
 		// }
 
-		if testCase.certificate == nil {
+		if testCase.certFn == nil {
 			if server.TLSConfig != nil {
 				t.Fatalf("Case %v: server.TLSConfig: expected: <nil>, got: %v", (i + 1), server.TLSConfig)
 			}
@@ -122,11 +120,6 @@ func TestServerTLSCiphers(t *testing.T) {
 		tls.TLS_RSA_WITH_AES_256_GCM_SHA384, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure.
 	}
 
-	certificate, err := getTLSCert()
-	if err != nil {
-		t.Fatalf("Unable to parse private/certificate data. %v\n", err)
-	}
-
 	testCases := []struct {
 		ciphers            []uint16
 		resetServerCiphers bool
@@ -145,8 +138,7 @@ func TestServerTLSCiphers(t *testing.T) {
 			server := NewServer([]string{addr},
 				http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 					fmt.Fprintf(w, "Hello, world")
-				}),
-				&certificate)
+				}), getCert)
 			if testCase.resetServerCiphers {
 				// Use Go default ciphers.
 				server.TLSConfig.CipherSuites = nil

cmd/server-main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/minio/dsync"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/certs"
 )
 
 func init() {
@@ -215,7 +216,7 @@ func serverMain(ctx *cli.Context) {
 
 	// Check and load SSL certificates.
 	var err error
-	globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig()
+	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
 	logger.FatalIf(err, "Unable to load the TLS configuration")
 
 	// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
@@ -275,7 +276,12 @@ func serverMain(ctx *cli.Context) {
 	// Initialize Admin Peers inter-node communication only in distributed setup.
 	initGlobalAdminPeers(globalEndpoints)
 
-	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, handler, globalTLSCertificate)
+	var getCert certs.GetCertificateFunc
+	if globalTLSCerts != nil {
+		getCert = globalTLSCerts.GetCertificate
+	}
+
+	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, handler, getCert)
 	globalHTTPServer.ReadTimeout = globalConnReadTimeout
 	globalHTTPServer.WriteTimeout = globalConnWriteTimeout
 	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
@@ -288,6 +294,9 @@ func serverMain(ctx *cli.Context) {
 
 	newObject, err := newObjectLayer(globalEndpoints)
 	if err != nil {
+		// Stop watching for any certificate changes.
+		globalTLSCerts.Stop()
+
 		globalHTTPServer.Shutdown()
 		logger.FatalIf(err, "Unable to initialize backend")
 	}

cmd/signals.go

@@ -45,6 +45,9 @@ func handleSignals() {
 			globalNotificationSys.RemoveAllRemoteTargets()
 		}
 
+		// Stop watching for any certificate changes.
+		globalTLSCerts.Stop()
+
 		err = globalHTTPServer.Shutdown()
 		logger.LogIf(context.Background(), err)
 
@@ -76,13 +79,11 @@ func handleSignals() {
 				// Ignore this at the moment.
 			case serviceRestart:
 				logger.Info("Restarting on service signal")
-				err := globalHTTPServer.Shutdown()
-				logger.LogIf(context.Background(), err)
 				stopHTTPTrace()
+				stop := stopProcess()
 				rerr := restartProcess()
 				logger.LogIf(context.Background(), rerr)
-
-				exit(err == nil && rerr == nil)
+				exit(stop && rerr == nil)
 			case serviceStop:
 				logger.Info("Stopping on service signal")
 				stopHTTPTrace()