MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-11 15:03:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

Move etcd, logger, crypto into their own packages (#8366)

Browse Source 
- Deprecates _MINIO_PROFILER, `mc admin profile` does the job
- Move ENVs to common location in cmd/config/
This commit is contained in:
Harshavardhana 2019-10-07 22:47:56 -07:00 committed by kannappanr
parent bffc378a4f
commit 290ad0996f
36 changed files with 735 additions and 533 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -35,7 +35,6 @@ matrix:
- make verifiers
- make crosscompile
- make verify
- make coverage
- cd browser && yarn && yarn test && cd ..
- bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'
@ -47,7 +46,7 @@ matrix:
go: 1.13.x
script:
- go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
- bash buildscripts/go-coverage.sh
- CGO_ENABLED=1 go test -v --timeout 20m ./...
before_script:
# Add an IPv6 config - see the corresponding Travis issue

4
Makefile
View File

@ -64,8 +64,10 @@ test: verifiers build
@echo "Running unit tests"
@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
verify: build
# Verify minio binary, enable races as well
verify:
@echo "Verifying build"
@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
@(env bash $(PWD)/buildscripts/verify-build.sh)
coverage: build

2
cmd/admin-handlers.go
View File

@ -1722,7 +1722,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
keyID := r.URL.Query().Get("key-id")
if keyID == "" {
keyID = globalKMSKeyID
keyID = GlobalKMS.KeyID()
}
var response = madmin.KMSKeyStatus{
KeyID: keyID,

7
cmd/api-errors_test.go
View File

@ -19,6 +19,8 @@ package cmd
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"github.com/minio/minio/cmd/crypto"
@ -65,6 +67,11 @@ var toAPIErrorTests = []struct {
}
func TestAPIErrCode(t *testing.T) {
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
defer os.RemoveAll(disk)
initFSObjects(disk, t)
ctx := context.Background()
for i, testCase := range toAPIErrorTests {
errCode := toAPIErrorCode(ctx, testCase.err)

136
cmd/common-main.go
View File

@ -1,5 +1,5 @@
/*
* MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
* MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,24 +17,23 @@
package cmd
import (
"crypto/tls"
"crypto/x509"
"errors"
"net"
"path/filepath"
"strings"
"time"
etcd "github.com/coreos/etcd/clientv3"
dns2 "github.com/miekg/dns"
"github.com/minio/cli"
"github.com/minio/minio-go/v6/pkg/set"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/etcd"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/cmd/logger/target/http"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/certs"
"github.com/minio/minio/pkg/dns"
"github.com/minio/minio/pkg/env"
xnet "github.com/minio/minio/pkg/net"
)
func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
@ -71,36 +70,6 @@ func checkUpdate(mode string) {
}
}
// Load logger targets based on user's configuration
func loadLoggers() {
loggerUserAgent := getUserAgent(getMinioMode())
auditEndpoint, ok := env.Lookup("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
if ok {
// Enable audit HTTP logging through ENV.
logger.AddAuditTarget(http.New(auditEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
}
loggerEndpoint, ok := env.Lookup("MINIO_LOGGER_HTTP_ENDPOINT")
if ok {
// Enable HTTP logging through ENV.
logger.AddTarget(http.New(loggerEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
} else {
for _, l := range globalServerConfig.Logger.HTTP {
if l.Enabled {
// Enable http logging
logger.AddTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
}
}
}
if globalServerConfig.Logger.Console.Enabled {
// Enable console logging
logger.AddTarget(globalConsoleSys.Console())
}
}
func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
var dir string
var dirSet bool
@ -190,15 +159,8 @@ func handleCommonCmdArgs(ctx *cli.Context) {
}
func handleCommonEnvVars() {
// Start profiler if env is set.
if profiler := env.Get("_MINIO_PROFILER", ""); profiler != "" {
var err error
globalProfiler, err = startProfiler(profiler, "")
logger.FatalIf(err, "Unable to setup a profiler")
}
accessKey := env.Get("MINIO_ACCESS_KEY", "")
secretKey := env.Get("MINIO_SECRET_KEY", "")
accessKey := env.Get(config.EnvAccessKey, "")
secretKey := env.Get(config.EnvSecretKey, "")
if accessKey != "" && secretKey != "" {
cred, err := auth.CreateCredentials(accessKey, secretKey)
if err != nil {
@ -211,8 +173,8 @@ func handleCommonEnvVars() {
globalActiveCred = cred
}
if browser := env.Get("MINIO_BROWSER", "on"); browser != "" {
browserFlag, err := ParseBoolFlag(browser)
if browser := env.Get(config.EnvBrowser, "on"); browser != "" {
browserFlag, err := config.ParseBoolFlag(browser)
if err != nil {
logger.Fatal(config.ErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
}
@ -223,57 +185,15 @@ func handleCommonEnvVars() {
globalIsBrowserEnabled = bool(browserFlag)
}
etcdEndpointsEnv, ok := env.Lookup("MINIO_ETCD_ENDPOINTS")
if ok {
etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
var etcdSecure bool
for _, endpoint := range etcdEndpoints {
u, err := xnet.ParseURL(endpoint)
if err != nil {
logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
}
// If one of the endpoint is https, we will use https directly.
etcdSecure = etcdSecure || u.Scheme == "https"
}
var err error
if etcdSecure {
// This is only to support client side certificate authentication
// https://coreos.com/etcd/docs/latest/op-guide/security.html
etcdClientCertFile, ok1 := env.Lookup("MINIO_ETCD_CLIENT_CERT")
etcdClientCertKey, ok2 := env.Lookup("MINIO_ETCD_CLIENT_CERT_KEY")
var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
if ok1 && ok2 {
getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
return &cert, terr
}
}
globalEtcdClient, err = etcd.New(etcd.Config{
Endpoints: etcdEndpoints,
DialTimeout: defaultDialTimeout,
DialKeepAliveTime: defaultDialKeepAlive,
TLS: &tls.Config{
RootCAs: globalRootCAs,
GetClientCertificate: getClientCertificate,
},
})
} else {
globalEtcdClient, err = etcd.New(etcd.Config{
Endpoints: etcdEndpoints,
DialTimeout: defaultDialTimeout,
DialKeepAliveTime: defaultDialKeepAlive,
})
}
logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
var err error
globalEtcdClient, err = etcd.New(globalRootCAs)
if err != nil {
logger.FatalIf(err, "Unable to initialize etcd config")
}
v, ok := env.Lookup("MINIO_DOMAIN")
if ok {
for _, domainName := range strings.Split(v, ",") {
if _, ok = dns2.IsDomainName(domainName); !ok {
for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), ",") {
if domainName != "" {
if _, ok := dns2.IsDomainName(domainName); !ok {
logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
"Invalid MINIO_DOMAIN value in environment variable")
}
@ -281,7 +201,7 @@ func handleCommonEnvVars() {
}
}
minioEndpointsEnv, ok := env.Lookup("MINIO_PUBLIC_IPS")
minioEndpointsEnv, ok := env.Lookup(config.EnvPublicIPs)
if ok {
minioEndpoints := strings.Split(minioEndpointsEnv, ",")
var domainIPs = set.NewStringSet()
@ -315,11 +235,11 @@ func handleCommonEnvVars() {
// In place update is true by default if the MINIO_UPDATE is not set
// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
// in-place update is off.
globalInplaceUpdateDisabled = strings.EqualFold(env.Get("MINIO_UPDATE", "off"), "off")
globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, "off"), "off")
// Get WORM environment variable.
if worm := env.Get("MINIO_WORM", "off"); worm != "" {
wormFlag, err := ParseBoolFlag(worm)
if worm := env.Get(config.EnvWorm, "off"); worm != "" {
wormFlag, err := config.ParseBoolFlag(worm)
if err != nil {
logger.Fatal(config.ErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
}
@ -338,3 +258,21 @@ func logStartupMessage(msg string, data ...interface{}) {
}
logger.StartupMessage(msg, data...)
}
func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
return nil, nil, false, nil
}
if x509Certs, err = config.ParsePublicCertFile(getPublicCertFile()); err != nil {
return nil, nil, false, err
}
c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), config.LoadX509KeyPair)
if err != nil {
return nil, nil, false, err
}
secureConn = true
return x509Certs, c, secureConn, nil
}

53
cmd/config-current.go
View File

@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"reflect"
"strings"
"sync"
"github.com/minio/minio/cmd/config"
@ -31,6 +32,7 @@ import (
"github.com/minio/minio/cmd/crypto"
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/cmd/logger/target/http"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/env"
"github.com/minio/minio/pkg/event"
@ -111,7 +113,7 @@ func (s *serverConfig) GetCredential() auth.Credentials {
// SetWorm set if worm is enabled.
func (s *serverConfig) SetWorm(b bool) {
// Set the new value.
s.Worm = BoolFlag(b)
s.Worm = config.BoolFlag(b)
}
// GetStorageClass reads storage class fields from current config.
@ -271,7 +273,7 @@ func (s *serverConfig) lookupConfigs() {
globalCacheMaxUse = s.Cache.MaxUse
if cacheEncKey := env.Get(cache.EnvCacheEncryptionMasterKey, ""); cacheEncKey != "" {
globalCacheKMSKeyID, globalCacheKMS, err = parseKMSMasterKey(cacheEncKey)
globalCacheKMS, err = crypto.ParseMasterKey(cacheEncKey)
if err != nil {
logger.FatalIf(config.ErrInvalidCacheEncryptionKey(err),
"Unable to setup encryption cache")
@ -279,8 +281,19 @@ func (s *serverConfig) lookupConfigs() {
}
}
if err = LookupKMSConfig(s.KMS); err != nil {
logger.FatalIf(err, "Unable to setup KMS")
s.KMS, err = crypto.LookupConfig(s.KMS)
if err != nil {
logger.FatalIf(err, "Unable to setup KMS config")
}
GlobalKMS, err = crypto.NewKMS(s.KMS)
if err != nil {
logger.FatalIf(err, "Unable to setup KMS with current KMS config")
}
globalAutoEncryption = strings.EqualFold(env.Get(crypto.EnvAutoEncryption, "off"), "on")
if globalAutoEncryption && GlobalKMS == nil {
logger.FatalIf(errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present"), "")
}
s.Compression, err = compress.LookupConfig(s.Compression)
@ -311,6 +324,34 @@ func (s *serverConfig) lookupConfigs() {
if err != nil {
logger.FatalIf(err, "Unable to parse LDAP configuration from env")
}
// Load logger targets based on user's configuration
loggerUserAgent := getUserAgent(getMinioMode())
s.Logger, err = logger.LookupConfig(s.Logger)
if err != nil {
logger.FatalIf(err, "Unable to initialize logger")
}
for _, l := range s.Logger.HTTP {
if l.Enabled {
// Enable http logging
logger.AddTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
}
}
for _, l := range s.Logger.Audit {
if l.Enabled {
// Enable http audit logging
logger.AddAuditTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
}
}
if s.Logger.Console.Enabled {
// Enable console logging
logger.AddTarget(globalConsoleSys.Console())
}
}
// TestNotificationTargets tries to establish connections to all notification
@ -531,8 +572,8 @@ func newServerConfig() *serverConfig {
// Console logging is on by default
srvCfg.Logger.Console.Enabled = true
// Create an example of HTTP logger
srvCfg.Logger.HTTP = make(map[string]loggerHTTP)
srvCfg.Logger.HTTP["target1"] = loggerHTTP{Endpoint: "https://username:password@example.com/api"}
srvCfg.Logger.HTTP = make(map[string]logger.HTTP)
srvCfg.Logger.HTTP["target1"] = logger.HTTP{Endpoint: "https://username:password@example.com/api"}
return srvCfg
}

86
cmd/config-current_test.go
View File

@ -23,6 +23,7 @@ import (
"testing"
"github.com/minio/minio/cmd/config/storageclass"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event/target"
)
@ -63,79 +64,6 @@ func TestServerConfig(t *testing.T) {
}
}
func TestServerConfigWithEnvs(t *testing.T) {
os.Setenv("MINIO_BROWSER", "off")
defer os.Unsetenv("MINIO_BROWSER")
os.Setenv("MINIO_WORM", "on")
defer os.Unsetenv("MINIO_WORM")
os.Setenv("MINIO_ACCESS_KEY", "minio")
defer os.Unsetenv("MINIO_ACCESS_KEY")
os.Setenv("MINIO_SECRET_KEY", "minio123")
defer os.Unsetenv("MINIO_SECRET_KEY")
os.Setenv("MINIO_REGION", "us-west-1")
defer os.Unsetenv("MINIO_REGION")
os.Setenv("MINIO_DOMAIN", "domain.com")
defer os.Unsetenv("MINIO_DOMAIN")
defer resetGlobalIsEnvs()
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("Init Test config failed")
}
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
serverHandleEnvVars()
// Init config
initConfig(objLayer)
// Check if serverConfig has browser disabled
if globalIsBrowserEnabled {
t.Error("Expected browser to be disabled but it is not")
}
// Check if serverConfig returns WORM config from the env
if !globalServerConfig.GetWorm() {
t.Error("Expected WORM to be enabled but it is not")
}
// Check if serverConfig has region from the environment
if globalServerConfig.GetRegion() != "us-west-1" {
t.Errorf("Expected region to be \"us-west-1\", found %v", globalServerConfig.GetRegion())
}
// Check if serverConfig has credentials from the environment
cred := globalServerConfig.GetCredential()
if cred.AccessKey != "minio" {
t.Errorf("Expected access key to be `minio`, found %s", cred.AccessKey)
}
if cred.SecretKey != "minio123" {
t.Errorf("Expected access key to be `minio123`, found %s", cred.SecretKey)
}
// Check if serverConfig has the correct domain
if globalDomainNames[0] != "domain.com" {
t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainNames[0])
}
}
// Tests config validator..
func TestValidateConfig(t *testing.T) {
objLayer, fsDir, err := prepareFS()
@ -360,13 +288,13 @@ func TestConfigDiff(t *testing.T) {
},
// 15
{
&serverConfig{Logger: loggerConfig{
Console: loggerConsole{Enabled: true},
HTTP: map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
&serverConfig{Logger: logger.Config{
Console: logger.Console{Enabled: false},
HTTP: map[string]logger.HTTP{"1": {Endpoint: "http://address1"}},
}},
&serverConfig{Logger: loggerConfig{
Console: loggerConsole{Enabled: true},
HTTP: map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
&serverConfig{Logger: logger.Config{
Console: logger.Console{Enabled: false},
HTTP: map[string]logger.HTTP{"1": {Endpoint: "http://address2"}},
}},
"Logger configuration differs",
},

67
cmd/config-migrate.go
View File

@ -65,7 +65,7 @@ func migrateConfig() error {
// Load only config version information.
version, err := GetVersion(getConfigFile())
if err != nil {
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
}
return err
@ -234,7 +234,7 @@ func purgeV1() error {
cv1 := &configV1{}
_, err := Load(configFile, cv1)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@ -255,7 +255,7 @@ func migrateV2ToV3() error {
cv2 := &configV2{}
_, err := Load(configFile, cv2)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘2’. %v", err)
@ -314,7 +314,7 @@ func migrateV3ToV4() error {
cv3 := &configV3{}
_, err := Load(configFile, cv3)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘3’. %v", err)
@ -352,7 +352,7 @@ func migrateV4ToV5() error {
cv4 := &configV4{}
_, err := Load(configFile, cv4)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘4’. %v", err)
@ -393,7 +393,7 @@ func migrateV5ToV6() error {
cv5 := &configV5{}
_, err := Load(configFile, cv5)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘5’. %v", err)
@ -482,7 +482,7 @@ func migrateV6ToV7() error {
cv6 := &configV6{}
_, err := Load(configFile, cv6)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘6’. %v", err)
@ -538,7 +538,7 @@ func migrateV7ToV8() error {
cv7 := &serverConfigV7{}
_, err := Load(configFile, cv7)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘7’. %v", err)
@ -600,7 +600,7 @@ func migrateV8ToV9() error {
cv8 := &serverConfigV8{}
_, err := Load(configFile, cv8)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘8’. %v", err)
@ -670,7 +670,7 @@ func migrateV9ToV10() error {
cv9 := &serverConfigV9{}
_, err := Load(configFile, cv9)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘9’. %v", err)
@ -738,7 +738,7 @@ func migrateV10ToV11() error {
cv10 := &serverConfigV10{}
_, err := Load(configFile, cv10)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘10’. %v", err)
@ -809,7 +809,7 @@ func migrateV11ToV12() error {
cv11 := &serverConfigV11{}
_, err := Load(configFile, cv11)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘11’. %v", err)
@ -906,7 +906,7 @@ func migrateV12ToV13() error {
cv12 := &serverConfigV12{}
_, err := Load(configFile, cv12)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘12’. %v", err)
@ -986,7 +986,7 @@ func migrateV13ToV14() error {
cv13 := &serverConfigV13{}
_, err := Load(configFile, cv13)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘13’. %v", err)
@ -1071,7 +1071,7 @@ func migrateV14ToV15() error {
cv14 := &serverConfigV14{}
_, err := Load(configFile, cv14)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘14’. %v", err)
@ -1161,7 +1161,7 @@ func migrateV15ToV16() error {
cv15 := &serverConfigV15{}
_, err := Load(configFile, cv15)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘15’. %v", err)
@ -1251,7 +1251,7 @@ func migrateV16ToV17() error {
cv16 := &serverConfigV16{}
_, err := Load(configFile, cv16)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘16’. %v", err)
@ -1372,7 +1372,7 @@ func migrateV17ToV18() error {
cv17 := &serverConfigV17{}
_, err := Load(configFile, cv17)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘17’. %v", err)
@ -1474,7 +1474,7 @@ func migrateV18ToV19() error {
cv18 := &serverConfigV18{}
_, err := Load(configFile, cv18)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘18’. %v", err)
@ -1580,7 +1580,7 @@ func migrateV19ToV20() error {
cv19 := &serverConfigV19{}
_, err := Load(configFile, cv19)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘18’. %v", err)
@ -1685,7 +1685,7 @@ func migrateV20ToV21() error {
cv20 := &serverConfigV20{}
_, err := Load(configFile, cv20)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘20’. %v", err)
@ -1789,7 +1789,7 @@ func migrateV21ToV22() error {
cv21 := &serverConfigV21{}
_, err := Load(configFile, cv21)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘21’. %v", err)
@ -1893,7 +1893,7 @@ func migrateV22ToV23() error {
cv22 := &serverConfigV22{}
_, err := Load(configFile, cv22)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘22’. %v", err)
@ -2006,7 +2006,7 @@ func migrateV23ToV24() error {
cv23 := &serverConfigV23{}
_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘23’. %v", err)
@ -2119,7 +2119,7 @@ func migrateV24ToV25() error {
cv24 := &serverConfigV24{}
_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘24’. %v", err)
@ -2237,7 +2237,7 @@ func migrateV25ToV26() error {
cv25 := &serverConfigV25{}
_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version ‘25’. %v", err)
@ -2359,7 +2359,7 @@ func migrateV26ToV27() error {
// in the new `logger` field
srvConfig := &serverConfigV27{}
_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config file. %v", err)
@ -2373,8 +2373,8 @@ func migrateV26ToV27() error {
// Enable console logging by default to avoid breaking users
// current deployments
srvConfig.Logger.Console.Enabled = true
srvConfig.Logger.HTTP = make(map[string]loggerHTTP)
srvConfig.Logger.HTTP["1"] = loggerHTTP{}
srvConfig.Logger.HTTP = make(map[string]logger.HTTP)
srvConfig.Logger.HTTP["1"] = logger.HTTP{}
if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %v", err)
@ -2392,7 +2392,7 @@ func migrateV27ToV28() error {
srvConfig := &serverConfigV28{}
_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
if os.IsNotExist(err) {
if os.IsNotExist(err) || os.IsPermission(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config file. %v", err)
@ -2459,14 +2459,17 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
var config = &serverConfig{}
for _, cfgFile := range configFiles {
if _, err = Load(cfgFile, config); err != nil {
if !os.IsNotExist(err) {
if !os.IsNotExist(err) && !os.IsPermission(err) {
return err
}
continue
}
break
}
if os.IsNotExist(err) {
if os.IsPermission(err) {
logger.Info("Older config found but not readable %s, proceeding to initialize new config anyways", err)
}
if os.IsNotExist(err) || os.IsPermission(err) {
// Initialize the server config, if no config exists.
return newSrvConfig(objAPI)
}

72
cmd/config-versions.go
View File

@ -19,11 +19,13 @@ package cmd
import (
"sync"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/cache"
"github.com/minio/minio/cmd/config/compress"
xldap "github.com/minio/minio/cmd/config/ldap"
"github.com/minio/minio/cmd/config/storageclass"
"github.com/minio/minio/cmd/crypto"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event/target"
"github.com/minio/minio/pkg/iam/openid"
@ -404,7 +406,7 @@ type serverConfigV14 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
// Additional error logging configuration.
Logger *loggerV7 `json:"logger"`
@ -421,7 +423,7 @@ type serverConfigV15 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
// Additional error logging configuration.
Logger *loggerV7 `json:"logger"`
@ -459,7 +461,7 @@ type serverConfigV16 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
// Additional error logging configuration.
Logger *loggers `json:"logger"`
@ -478,7 +480,7 @@ type serverConfigV17 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
// Additional error logging configuration.
Logger *loggers `json:"logger"`
@ -497,7 +499,7 @@ type serverConfigV18 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
// Additional error logging configuration.
Logger *loggers `json:"logger"`
@ -515,7 +517,7 @@ type serverConfigV19 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
// Additional error logging configuration.
Logger *loggers `json:"logger"`
@ -533,7 +535,7 @@ type serverConfigV20 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
Domain string `json:"domain"`
// Additional error logging configuration.
@ -551,7 +553,7 @@ type serverConfigV21 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
Domain string `json:"domain"`
// Notification queue configuration.
@ -569,7 +571,7 @@ type serverConfigV22 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
Domain string `json:"domain"`
// Storage class configuration
@ -589,7 +591,7 @@ type serverConfigV23 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
Domain string `json:"domain"`
// Storage class configuration
@ -613,7 +615,7 @@ type serverConfigV24 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Browser config.BoolFlag `json:"browser"`
Domain string `json:"domain"`
// Storage class configuration
@ -639,8 +641,8 @@ type serverConfigV25 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Worm BoolFlag `json:"worm"`
Browser config.BoolFlag `json:"browser"`
Worm config.BoolFlag `json:"worm"`
Domain string `json:"domain"`
// Storage class configuration
@ -663,8 +665,8 @@ type serverConfigV26 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Worm BoolFlag `json:"worm"`
Browser config.BoolFlag `json:"browser"`
Worm config.BoolFlag `json:"worm"`
Domain string `json:"domain"`
// Storage class configuration
@ -677,20 +679,6 @@ type serverConfigV26 struct {
Notify notifierV3 `json:"notify"`
}
type loggerConsole struct {
Enabled bool `json:"enabled"`
}
type loggerHTTP struct {
Enabled bool `json:"enabled"`
Endpoint string `json:"endpoint"`
}
type loggerConfig struct {
Console loggerConsole `json:"console"`
HTTP map[string]loggerHTTP `json:"http"`
}
// serverConfigV27 is just like version '26', stores additionally
// the logger field
//
@ -704,8 +692,8 @@ type serverConfigV27 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Browser BoolFlag `json:"browser"`
Worm BoolFlag `json:"worm"`
Browser config.BoolFlag `json:"browser"`
Worm config.BoolFlag `json:"worm"`
Domain string `json:"domain"`
// Storage class configuration
@ -718,7 +706,7 @@ type serverConfigV27 struct {
Notify notifierV3 `json:"notify"`
// Logger configuration
Logger loggerConfig `json:"logger"`
Logger logger.Config `json:"logger"`
}
// serverConfigV28 is just like version '27', additionally
@ -734,7 +722,7 @@ type serverConfigV28 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Worm BoolFlag `json:"worm"`
Worm config.BoolFlag `json:"worm"`
// Storage class configuration
StorageClass storageclass.Config `json:"storageclass"`
@ -749,7 +737,7 @@ type serverConfigV28 struct {
Notify notifierV3 `json:"notify"`
// Logger configuration
Logger loggerConfig `json:"logger"`
Logger logger.Config `json:"logger"`
}
// serverConfigV29 is just like version '28'.
@ -763,7 +751,7 @@ type serverConfigV30 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Worm BoolFlag `json:"worm"`
Worm config.BoolFlag `json:"worm"`
// Storage class configuration
StorageClass storageclass.Config `json:"storageclass"`
@ -778,7 +766,7 @@ type serverConfigV30 struct {
Notify notifierV3 `json:"notify"`
// Logger configuration
Logger loggerConfig `json:"logger"`
Logger logger.Config `json:"logger"`
// Compression configuration
Compression compress.Config `json:"compress"`
@ -791,7 +779,7 @@ type serverConfigV31 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Worm BoolFlag `json:"worm"`
Worm config.BoolFlag `json:"worm"`
// Storage class configuration
StorageClass storageclass.Config `json:"storageclass"`
@ -806,7 +794,7 @@ type serverConfigV31 struct {
Notify notifierV3 `json:"notify"`
// Logger configuration
Logger loggerConfig `json:"logger"`
Logger logger.Config `json:"logger"`
// Compression configuration
Compression compress.Config `json:"compress"`
@ -846,7 +834,7 @@ type serverConfigV32 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Worm BoolFlag `json:"worm"`
Worm config.BoolFlag `json:"worm"`
// Storage class configuration
StorageClass storageclass.Config `json:"storageclass"`
@ -861,7 +849,7 @@ type serverConfigV32 struct {
Notify notifier `json:"notify"`
// Logger configuration
Logger loggerConfig `json:"logger"`
Logger logger.Config `json:"logger"`
// Compression configuration
Compression compress.Config `json:"compress"`
@ -890,7 +878,7 @@ type serverConfigV33 struct {
// S3 API configuration.
Credential auth.Credentials `json:"credential"`
Region string `json:"region"`
Worm BoolFlag `json:"worm"`
Worm config.BoolFlag `json:"worm"`
// Storage class configuration
StorageClass storageclass.Config `json:"storageclass"`
@ -905,7 +893,7 @@ type serverConfigV33 struct {
Notify notifier `json:"notify"`
// Logger configuration
Logger loggerConfig `json:"logger"`
Logger logger.Config `json:"logger"`
// Compression configuration
Compression compress.Config `json:"compress"`

4
cmd/bool-flag.go → cmd/config/bool-flag.go
View File

@ -1,5 +1,5 @@
/*
* MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
* MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -14,7 +14,7 @@
* limitations under the License.
*/
package cmd
package config
import (
"encoding/json"

4
cmd/bool-flag_test.go → cmd/config/bool-flag_test.go
View File

@ -1,5 +1,5 @@
/*
* MinIO Cloud Storage, (C) 2017 MinIO, Inc.
* MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -14,7 +14,7 @@
* limitations under the License.
*/
package cmd
package config
import (
"errors"

89
cmd/certs.go → cmd/config/certs.go
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package cmd
package config
import (
"bytes"
@ -24,18 +24,19 @@ import (
"crypto/x509"
"encoding/pem"
"io/ioutil"
"os"
"path/filepath"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/pkg/certs"
"github.com/minio/minio/pkg/env"
)
// TLSPrivateKeyPassword is the environment variable which contains the password used
// EnvCertPassword is the environment variable which contains the password used
// to decrypt the TLS private key. It must be set if the TLS private key is
// password protected.
const TLSPrivateKeyPassword = "MINIO_CERT_PASSWD"
const EnvCertPassword = "MINIO_CERT_PASSWD"
func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
// ParsePublicCertFile - parses public cert into its *x509.Certificate equivalent.
func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
// Read certificate file.
var data []byte
if data, err = ioutil.ReadFile(certFile); err != nil {
@ -50,25 +51,27 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
for len(current) > 0 {
var pemBlock *pem.Block
if pemBlock, current = pem.Decode(current); pemBlock == nil {
return nil, config.ErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
return nil, ErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
}
var x509Cert *x509.Certificate
if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
return nil, config.ErrSSLUnexpectedData(err)
return nil, ErrSSLUnexpectedData(err)
}
x509Certs = append(x509Certs, x509Cert)
}
if len(x509Certs) == 0 {
return nil, config.ErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
return nil, ErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
}
return x509Certs, nil
}
func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
// GetRootCAs - returns all the root CAs into certPool
// at the input certsCADir
func GetRootCAs(certsCAsDir string) (*x509.CertPool, error) {
rootCAs, _ := x509.SystemCertPool()
if rootCAs == nil {
// In some systems (like Windows) system cert pool is
@ -77,9 +80,9 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
rootCAs = x509.NewCertPool()
}
fis, err := readDir(certsCAsDir)
fis, err := ioutil.ReadDir(certsCAsDir)
if err != nil {
if err == errFileNotFound {
if os.IsNotExist(err) {
err = nil // Return success if CA's directory is missing.
}
return rootCAs, err
@ -87,77 +90,61 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
// Load all custom CA files.
for _, fi := range fis {
// Skip all directories.
if hasSuffix(fi, SlashSeparator) {
continue
// Only load regular files as public cert.
if fi.Mode().IsRegular() {
caCert, err := ioutil.ReadFile(filepath.Join(certsCAsDir, fi.Name()))
if err != nil {
return rootCAs, err
}
rootCAs.AppendCertsFromPEM(caCert)
}
caCert, err := ioutil.ReadFile(pathJoin(certsCAsDir, fi))
if err != nil {
return rootCAs, err
}
rootCAs.AppendCertsFromPEM(caCert)
}
return rootCAs, nil
}
// load an X509 key pair (private key , certificate) from the provided
// paths. The private key may be encrypted and is decrypted using the
// ENV_VAR: MINIO_CERT_PASSWD.
func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
// LoadX509KeyPair - load an X509 key pair (private key , certificate)
// from the provided paths. The private key may be encrypted and is
// decrypted using the ENV_VAR: MINIO_CERT_PASSWD.
func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
certPEMBlock, err := ioutil.ReadFile(certFile)
if err != nil {
return tls.Certificate{}, config.ErrSSLUnexpectedError(err)
return tls.Certificate{}, ErrSSLUnexpectedError(err)
}
keyPEMBlock, err := ioutil.ReadFile(keyFile)
if err != nil {
return tls.Certificate{}, config.ErrSSLUnexpectedError(err)
return tls.Certificate{}, ErrSSLUnexpectedError(err)
}
key, rest := pem.Decode(keyPEMBlock)
if len(rest) > 0 {
return tls.Certificate{}, config.ErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
}
if x509.IsEncryptedPEMBlock(key) {
password, ok := env.Lookup(TLSPrivateKeyPassword)
password, ok := env.Lookup(EnvCertPassword)
if !ok {
return tls.Certificate{}, config.ErrSSLNoPassword(nil)
return tls.Certificate{}, ErrSSLNoPassword(nil)
}
decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
if decErr != nil {
return tls.Certificate{}, config.ErrSSLWrongPassword(decErr)
return tls.Certificate{}, ErrSSLWrongPassword(decErr)
}
keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
}
cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
if err != nil {
return tls.Certificate{}, config.ErrSSLUnexpectedData(nil).Msg(err.Error())
return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg(err.Error())
}
// Ensure that the private key is not a P-384 or P-521 EC key.
// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
if name := pub.Params().Name; name == "P-384" || name == "P-521" {
switch pub.Params().Name {
case "P-384":
fallthrough
case "P-521":
// unfortunately there is no cleaner way to check
return tls.Certificate{}, config.ErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", pub.Params().Name)
}
}
}
return cert, nil
}
func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
return nil, nil, false, nil
}
if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
return nil, nil, false, err
}
c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
if err != nil {
return nil, nil, false, err
}
secureConn = true
return x509Certs, c, secureConn, nil
}

12
cmd/certs_test.go → cmd/config/certs_test.go
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package cmd
package config
import (
"fmt"
@ -176,7 +176,7 @@ M9ofSEt/bdRD
}
for _, testCase := range testCases {
certs, err := parsePublicCertFile(testCase.certFile)
certs, err := ParsePublicCertFile(testCase.certFile)
if testCase.expectedErr == nil {
if err != nil {
@ -234,7 +234,7 @@ func TestGetRootCAs(t *testing.T) {
}
for _, testCase := range testCases {
_, err := getRootCAs(testCase.certCAsDir)
_, err := GetRootCAs(testCase.certCAsDir)
if testCase.expectedErr == nil {
if err != nil {
@ -260,11 +260,11 @@ func TestLoadX509KeyPair(t *testing.T) {
t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
}
os.Unsetenv(TLSPrivateKeyPassword)
os.Unsetenv(EnvCertPassword)
if testCase.password != "" {
os.Setenv(TLSPrivateKeyPassword, testCase.password)
os.Setenv(EnvCertPassword, testCase.password)
}
_, err = loadX509KeyPair(certificate, privateKey)
_, err = LoadX509KeyPair(certificate, privateKey)
if err != nil && !testCase.shouldFail {
t.Errorf("Test %d: test should succeed but it failed: %v", i, err)
}

17
cmd/config/compress/compress.go
View File

@ -34,9 +34,9 @@ type Config struct {
// Compression environment variables
const (
EnvMinioCompress = "MINIO_COMPRESS"
EnvMinioCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
EnvMinioCompressMimeTypes = "MINIO_COMPRESS_MIMETYPES"
EnvCompress = "MINIO_COMPRESS"
EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
EnvCompressMimeTypes = "MINIO_COMPRESS_MIMETYPES"
)
// Parses the given compression exclude list `extensions` or `content-types`.
@ -51,23 +51,22 @@ func parseCompressIncludes(includes []string) ([]string, error) {
// LookupConfig - lookup compression config.
func LookupConfig(cfg Config) (Config, error) {
const compressEnvDelimiter = ","
if compress := env.Get(EnvMinioCompress, strconv.FormatBool(cfg.Enabled)); compress != "" {
if compress := env.Get(EnvCompress, strconv.FormatBool(cfg.Enabled)); compress != "" {
cfg.Enabled = strings.EqualFold(compress, "true")
}
compressExtensions := env.Get(EnvMinioCompressExtensions, strings.Join(cfg.Extensions, ","))
compressMimeTypes := env.Get(EnvMinioCompressMimeTypes, strings.Join(cfg.MimeTypes, ","))
compressExtensions := env.Get(EnvCompressExtensions, strings.Join(cfg.Extensions, ","))
compressMimeTypes := env.Get(EnvCompressMimeTypes, strings.Join(cfg.MimeTypes, ","))
if compressExtensions != "" || compressMimeTypes != "" {
if compressExtensions != "" {
extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter))
extensions, err := parseCompressIncludes(strings.Split(compressExtensions, config.ValueSeparator))
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
}
cfg.Extensions = extensions
}
if compressMimeTypes != "" {
contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter))
contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, config.ValueSeparator))
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, contenttypes)
}

35
cmd/config/constants.go Normal file
View File

@ -0,0 +1,35 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package config
// Config value separator
const (
ValueSeparator = ","
)
// Top level common ENVs
const (
EnvAccessKey = "MINIO_ACCESS_KEY"
EnvSecretKey = "MINIO_SECRET_KEY"
EnvBrowser = "MINIO_BROWSER"
EnvDomain = "MINIO_DOMAIN"
EnvPublicIPs = "MINIO_PUBLIC_IPS"
EnvEndpoints = "MINIO_ENDPOINTS"
EnvUpdate = "MINIO_UPDATE"
EnvWorm = "MINIO_WORM"
)

96
cmd/config/etcd/etcd.go Normal file
View File

@ -0,0 +1,96 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package etcd
import (
"crypto/tls"
"crypto/x509"
"strings"
"time"
"github.com/coreos/etcd/clientv3"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/pkg/env"
xnet "github.com/minio/minio/pkg/net"
)
const (
// Default values used while communicating with etcd.
defaultDialTimeout = 30 * time.Second
defaultDialKeepAlive = 30 * time.Second
)
// etcd environment values
const (
EnvEtcdEndpoints = "MINIO_ETCD_ENDPOINTS"
EnvEtcdClientCert = "MINIO_ETCD_CLIENT_CERT"
EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
)
// New - Initialize new etcd client
func New(rootCAs *x509.CertPool) (*clientv3.Client, error) {
envEndpoints := env.Get(EnvEtcdEndpoints, "")
if envEndpoints == "" {
// etcd is not configured, nothing to do.
return nil, nil
}
etcdEndpoints := strings.Split(envEndpoints, config.ValueSeparator)
var etcdSecure bool
for _, endpoint := range etcdEndpoints {
u, err := xnet.ParseURL(endpoint)
if err != nil {
return nil, err
}
// If one of the endpoint is https, we will use https directly.
etcdSecure = etcdSecure || u.Scheme == "https"
}
var err error
var etcdClnt *clientv3.Client
if etcdSecure {
// This is only to support client side certificate authentication
// https://coreos.com/etcd/docs/latest/op-guide/security.html
etcdClientCertFile, ok1 := env.Lookup(EnvEtcdClientCert)
etcdClientCertKey, ok2 := env.Lookup(EnvEtcdClientCertKey)
var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
if ok1 && ok2 {
getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
return &cert, terr
}
}
etcdClnt, err = clientv3.New(clientv3.Config{
Endpoints: etcdEndpoints,
DialTimeout: defaultDialTimeout,
DialKeepAliveTime: defaultDialKeepAlive,
TLS: &tls.Config{
RootCAs: rootCAs,
GetClientCertificate: getClientCertificate,
},
})
} else {
etcdClnt, err = clientv3.New(clientv3.Config{
Endpoints: etcdEndpoints,
DialTimeout: defaultDialTimeout,
DialKeepAliveTime: defaultDialKeepAlive,
})
}
return etcdClnt, err
}

17
cmd/consolelogger.go
View File

@ -19,6 +19,7 @@ package cmd
import (
ring "container/ring"
"context"
"sync"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/cmd/logger/message/log"
@ -36,6 +37,8 @@ type HTTPConsoleLoggerSys struct {
pubsub *pubsub.PubSub
console *console.Target
nodeName string
// To protect ring buffer.
logBufLk sync.RWMutex
logBuf *ring.Ring
}
@ -52,7 +55,7 @@ func NewConsoleLogger(ctx context.Context, endpoints EndpointList) *HTTPConsoleL
}
ps := pubsub.New()
return &HTTPConsoleLoggerSys{
ps, nil, nodeName, ring.New(defaultLogBufferCount),
ps, nil, nodeName, sync.RWMutex{}, ring.New(defaultLogBufferCount),
}
}
@ -78,13 +81,14 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan s
}
lastN = make([]madmin.LogInfo, last)
r := sys.logBuf
r.Do(func(p interface{}) {
sys.logBufLk.RLock()
sys.logBuf.Do(func(p interface{}) {
if p != nil && (p.(madmin.LogInfo)).SendLog(node) {
lastN[cnt%last] = p.(madmin.LogInfo)
cnt++
}
})
sys.logBufLk.RUnlock()
// send last n console log messages in order filtered by node
if cnt > 0 {
for i := 0; i < last; i++ {
@ -102,8 +106,11 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan s
sys.pubsub.Subscribe(subCh, doneCh, filter)
}
// Console returns a console target
// Console returns a console target
func (sys *HTTPConsoleLoggerSys) Console() *HTTPConsoleLoggerSys {
if sys == nil {
return sys
}
if sys.console == nil {
sys.console = console.New()
}
@ -122,9 +129,11 @@ func (sys *HTTPConsoleLoggerSys) Send(e interface{}) error {
}
sys.pubsub.Publish(lg)
sys.logBufLk.Lock()
// add log to ring buffer
sys.logBuf.Value = lg
sys.logBuf = sys.logBuf.Next()
sys.logBufLk.Unlock()
if globalServerConfig.Logger.Console.Enabled {
return sys.console.Send(e)

123
cmd/crypto/config.go
View File

@ -1,4 +1,4 @@
// MinIO Cloud Storage, (C) 2015, 2016, 2017, 2018 MinIO, Inc.
// MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -14,8 +14,129 @@
package crypto
import (
"errors"
"fmt"
"strconv"
"strings"
"github.com/minio/minio/pkg/env"
)
const (
// EnvKMSMasterKey is the environment variable used to specify
// a KMS master key used to protect SSE-S3 per-object keys.
// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
// EnvAutoEncryption is the environment variable used to en/disable
// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
// requires a valid KMS configuration and turns any non-SSE-C
// request into an SSE-S3 request.
// If present EnvAutoEncryption must be either "on" or "off".
EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
)
const (
// EnvVaultEndpoint is the environment variable used to specify
// the vault HTTPS endpoint.
EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
// EnvVaultAuthType is the environment variable used to specify
// the authentication type for vault.
EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
// EnvVaultAppRoleID is the environment variable used to specify
// the vault AppRole ID.
EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
// EnvVaultAppSecretID is the environment variable used to specify
// the vault AppRole secret corresponding to the AppRole ID.
EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
// EnvVaultKeyVersion is the environment variable used to specify
// the vault key version.
EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
// EnvVaultKeyName is the environment variable used to specify
// the vault named key-ring. In the S3 context it's referred as
// customer master key ID (CMK-ID).
EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
// EnvVaultCAPath is the environment variable used to specify the
// path to a directory of PEM-encoded CA cert files. These CA cert
// files are used to authenticate MinIO to Vault over mTLS.
EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
// EnvVaultNamespace is the environment variable used to specify
// vault namespace. The vault namespace is used if the enterprise
// version of Hashicorp Vault is used.
EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
)
// KMSConfig has the KMS config for hashicorp vault
type KMSConfig struct {
AutoEncryption bool `json:"-"`
Vault VaultConfig `json:"vault"`
}
// LookupConfig extracts the KMS configuration provided by environment
// variables and merge them with the provided KMS configuration. The
// merging follows the following rules:
//
// 1. A valid value provided as environment variable is higher prioritized
// than the provided configuration and overwrites the value from the
// configuration file.
//
// 2. A value specified as environment variable never changes the configuration
// file. So it is never made a persistent setting.
//
// It sets the global KMS configuration according to the merged configuration
// on succes.
func LookupConfig(config KMSConfig) (KMSConfig, error) {
var err error
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
if err != nil {
return config, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
}
if err = config.Vault.Verify(); err != nil {
return config, err
}
return config, nil
}
// NewKMS - initialize a new KMS.
func NewKMS(config KMSConfig) (kms KMS, err error) {
// Lookup KMS master keys - only available through ENV.
if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
if !config.Vault.IsEmpty() { // Vault and KMS master key provided
return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
}
kms, err = ParseMasterKey(masterKey)
if err != nil {
return kms, err
}
}
if !config.Vault.IsEmpty() {
kms, err = NewVault(config.Vault)
if err != nil {
return kms, err
}
}
autoEncryption := strings.EqualFold(env.Get(EnvAutoEncryption, "off"), "on")
if autoEncryption && kms == nil {
return kms, errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
}
return kms, nil
}

12
cmd/crypto/kms.go
View File

@ -72,6 +72,9 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
// data key generation and unsealing of KMS-generated
// data keys.
type KMS interface {
// KeyID - returns configured KMS key id.
KeyID() string
// GenerateKey generates a new random data key using
// the master key referenced by the keyID. It returns
// the plaintext key and the sealed plaintext key
@ -102,14 +105,19 @@ type KMS interface {
}
type masterKeyKMS struct {
keyID string
masterKey [32]byte
}
// NewKMS returns a basic KMS implementation from a single 256 bit master key.
// NewMasterKey returns a basic KMS implementation from a single 256 bit master key.
//
// The KMS accepts any keyID but binds the keyID and context cryptographically
// to the generated keys.
func NewKMS(key [32]byte) KMS { return &masterKeyKMS{masterKey: key} }
func NewMasterKey(keyID string, key [32]byte) KMS { return &masterKeyKMS{keyID: keyID, masterKey: key} }
func (kms *masterKeyKMS) KeyID() string {
return kms.keyID
}
func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
if _, err = io.ReadFull(rand.Reader, key[:]); err != nil {

3
cmd/crypto/kms_test.go
View File

@ -40,8 +40,9 @@ var masterKeyKMSTests = []struct {
}
func TestMasterKeyKMS(t *testing.T) {
kms := NewKMS([32]byte{})
for i, test := range masterKeyKMSTests {
kms := NewMasterKey(test.GenKeyID, [32]byte{})
key, sealedKey, err := kms.GenerateKey(test.GenKeyID, test.GenContext)
if err != nil {
t.Errorf("Test %d: KMS failed to generate key: %v", i, err)

42
cmd/crypto/parse.go Normal file
View File

@ -0,0 +1,42 @@
// MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import (
"encoding/hex"
"fmt"
"strings"
)
// ParseMasterKey parses the value of the environment variable
// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
func ParseMasterKey(envArg string) (KMS, error) {
values := strings.SplitN(envArg, ":", 2)
if len(values) != 2 {
return nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
}
var (
keyID = values[0]
hexKey = values[1]
)
if len(hexKey) != 64 { // 2 hex bytes = 1 byte
return nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
}
var masterKey [32]byte
if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
return nil, err
}
return NewMasterKey(keyID, masterKey), nil
}

59
cmd/crypto/parse_test.go Normal file
View File

@ -0,0 +1,59 @@
// MinIO Cloud Storage, (C) 2019 MinIO, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package crypto
import "testing"
func TestParseMasterKey(t *testing.T) {
tests := []struct {
envValue string
expectedKeyID string
success bool
}{
{
envValue: "invalid-value",
success: false,
},
{
envValue: "too:many:colons",
success: false,
},
{
envValue: "myminio-key:not-a-hex",
success: false,
},
{
envValue: "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574",
expectedKeyID: "my-minio-key",
success: true,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.envValue, func(t *testing.T) {
kms, err := ParseMasterKey(tt.envValue)
if tt.success && err != nil {
t.Error(err)
}
if !tt.success && err == nil {
t.Error("Unexpected failure")
}
if err == nil && kms.KeyID() != tt.expectedKeyID {
t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, kms.KeyID())
}
})
}
}

4
cmd/crypto/sse_test.go
View File

@ -191,7 +191,7 @@ var s3UnsealObjectKeyTests = []struct {
ExpectedErr error
}{
{ // 0 - Valid KMS key-ID and valid metadata entries for bucket/object
KMS: NewKMS([32]byte{}),
KMS: NewMasterKey("my-minio-key", [32]byte{}),
Bucket: "bucket",
Object: "object",
Metadata: map[string]string{
@ -204,7 +204,7 @@ var s3UnsealObjectKeyTests = []struct {
ExpectedErr: nil,
},
{ // 1 - Valid KMS key-ID for invalid metadata entries for bucket/object
KMS: NewKMS([32]byte{}),
KMS: NewMasterKey("my-minio-key", [32]byte{}),
Bucket: "bucket",
Object: "object",
Metadata: map[string]string{

5
cmd/crypto/vault.go
View File

@ -199,6 +199,11 @@ func (v *vaultService) authenticate() (err error) {
return
}
// KeyID - vault configured keyID
func (v *vaultService) KeyID() string {
return v.config.Key.Name
}
// GenerateKey returns a new plaintext key, generated by the KMS,
// and a sealed version of this plaintext key encrypted using the
// named key referenced by keyID. It also binds the generated key

4
cmd/disk-cache-backend.go
View File

@ -442,14 +442,14 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
if globalCacheKMS == nil {
return nil, errKMSNotConfigured
}
key, encKey, err := globalCacheKMS.GenerateKey(globalCacheKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
key, encKey, err := globalCacheKMS.GenerateKey(globalCacheKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return nil, err
}
objectKey := crypto.GenerateKey(key, rand.Reader)
sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
crypto.S3.CreateMetadata(metadata, globalCacheKMSKeyID, encKey, sealedKey)
crypto.S3.CreateMetadata(metadata, globalCacheKMS.KeyID(), encKey, sealedKey)
if etag, ok := metadata["etag"]; ok {
metadata["etag"] = hex.EncodeToString(objectKey.SealETag([]byte(etag)))

8
cmd/encryption-v1.go
View File

@ -157,12 +157,12 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
return err
}
newKey, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
newKey, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return err
}
sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
return nil
}
}
@ -173,14 +173,14 @@ func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]s
if GlobalKMS == nil {
return nil, errKMSNotConfigured
}
key, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
key, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
if err != nil {
return nil, err
}
objectKey := crypto.GenerateKey(key, rand.Reader)
sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
return objectKey[:], nil
}
var extKey [32]byte

7
cmd/endpoint.go
View File

@ -32,6 +32,7 @@ import (
"github.com/minio/cli"
"github.com/minio/minio-go/v6/pkg/set"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/cmd/config/etcd"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/env"
"github.com/minio/minio/pkg/mountinfo"
@ -572,9 +573,9 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
return serverAddr, endpoints, setupType, err
}
_, dok := env.Lookup("MINIO_DOMAIN")
_, eok := env.Lookup("MINIO_ETCD_ENDPOINTS")
_, iok := env.Lookup("MINIO_PUBLIC_IPS")
_, dok := env.Lookup(config.EnvDomain)
_, eok := env.Lookup(etcd.EnvEtcdEndpoints)
_, iok := env.Lookup(config.EnvPublicIPs)
if dok && eok && !iok {
updateDomainIPs(uniqueArgs)
}

158
cmd/environment.go
View File

@ -1,158 +0,0 @@
// MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"encoding/hex"
"errors"
"fmt"
"strconv"
"strings"
"github.com/minio/minio/cmd/crypto"
"github.com/minio/minio/pkg/env"
)
const (
// EnvKMSMasterKey is the environment variable used to specify
// a KMS master key used to protect SSE-S3 per-object keys.
// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
// EnvAutoEncryption is the environment variable used to en/disable
// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
// requires a valid KMS configuration and turns any non-SSE-C
// request into an SSE-S3 request.
// If present EnvAutoEncryption must be either "on" or "off".
EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
)
const (
// EnvVaultEndpoint is the environment variable used to specify
// the vault HTTPS endpoint.
EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
// EnvVaultAuthType is the environment variable used to specify
// the authentication type for vault.
EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
// EnvVaultAppRoleID is the environment variable used to specify
// the vault AppRole ID.
EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
// EnvVaultAppSecretID is the environment variable used to specify
// the vault AppRole secret corresponding to the AppRole ID.
EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
// EnvVaultKeyVersion is the environment variable used to specify
// the vault key version.
EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
// EnvVaultKeyName is the environment variable used to specify
// the vault named key-ring. In the S3 context it's referred as
// customer master key ID (CMK-ID).
EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
// EnvVaultCAPath is the environment variable used to specify the
// path to a directory of PEM-encoded CA cert files. These CA cert
// files are used to authenticate MinIO to Vault over mTLS.
EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
// EnvVaultNamespace is the environment variable used to specify
// vault namespace. The vault namespace is used if the enterprise
// version of Hashicorp Vault is used.
EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
)
// LookupKMSConfig extracts the KMS configuration provided by environment
// variables and merge them with the provided KMS configuration. The
// merging follows the following rules:
//
// 1. A valid value provided as environment variable is higher prioritized
// than the provided configuration and overwrites the value from the
// configuration file.
//
// 2. A value specified as environment variable never changes the configuration
// file. So it is never made a persistent setting.
//
// It sets the global KMS configuration according to the merged configuration
// on success.
func LookupKMSConfig(config crypto.KMSConfig) (err error) {
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
if err != nil {
return fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
}
if err = config.Vault.Verify(); err != nil {
return err
}
// Lookup KMS master keys - only available through ENV.
if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
if !config.Vault.IsEmpty() { // Vault and KMS master key provided
return errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
}
globalKMSKeyID, GlobalKMS, err = parseKMSMasterKey(masterKey)
if err != nil {
return err
}
}
if !config.Vault.IsEmpty() {
GlobalKMS, err = crypto.NewVault(config.Vault)
if err != nil {
return err
}
globalKMSKeyID = config.Vault.Key.Name
}
autoEncryption, err := ParseBoolFlag(env.Get(EnvAutoEncryption, "off"))
if err != nil {
return err
}
globalAutoEncryption = bool(autoEncryption)
if globalAutoEncryption && GlobalKMS == nil { // auto-encryption enabled but no KMS
return errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
}
return nil
}
// parseKMSMasterKey parses the value of the environment variable
// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
func parseKMSMasterKey(envArg string) (string, crypto.KMS, error) {
values := strings.SplitN(envArg, ":", 2)
if len(values) != 2 {
return "", nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
}
var (
keyID = values[0]
hexKey = values[1]
)
if len(hexKey) != 64 { // 2 hex bytes = 1 byte
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
}
var masterKey [32]byte
if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
}
return keyID, crypto.NewKMS(masterKey), nil
}

7
cmd/gateway-main.go
View File

@ -133,7 +133,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
logger.FatalIf(err, "Invalid TLS certificate file")
// Check and load Root CAs.
globalRootCAs, err = getRootCAs(globalCertsCADir.Get())
globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
logger.FatalIf(err, "Failed to read root CAs (%v)", err)
// Handle common env vars.
@ -162,7 +162,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
registerSTSRouter(router)
}
// initialize globalConsoleSys system
// Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(context.Background(), globalEndpoints)
enableConfigOps := gatewayName == "nas"
@ -246,9 +246,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
globalConfigSys.WatchConfigNASDisk(newObject)
}
// Load logger subsystem
loadLoggers()
// This is only to uniquely identify each gateway deployments.
globalDeploymentID = env.Get("MINIO_GATEWAY_DEPLOYMENT_ID", mustGetUUID())
logger.SetDeploymentID(globalDeploymentID)

5
cmd/globals.go
View File

@ -215,8 +215,6 @@ var (
globalCacheExpiry = 90
// Max allowed disk cache percentage
globalCacheMaxUse = 80
// Disk cache KMS Key
globalCacheKMSKeyID string
// Initialized KMS configuration for disk cache
globalCacheKMS crypto.KMS
// Allocated etcd endpoint for config and bucket DNS.
@ -230,9 +228,6 @@ var (
// Usage check interval value.
globalUsageCheckInterval = globalDefaultUsageCheckInterval
// KMS key id
globalKMSKeyID string
// GlobalKMS initialized KMS configuration
GlobalKMS crypto.KMS

86
cmd/logger/config.go Normal file
View File

@ -0,0 +1,86 @@
/*
* MinIO Cloud Storage, (C) 2019 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package logger
import (
"strings"
"github.com/minio/minio/pkg/env"
)
// Console logger target
type Console struct {
Enabled bool `json:"enabled"`
}
// HTTP logger target
type HTTP struct {
Enabled bool `json:"enabled"`
Endpoint string `json:"endpoint"`
}
// Config console and http logger targets
type Config struct {
Console Console `json:"console"`
HTTP map[string]HTTP `json:"http"`
Audit map[string]HTTP `json:"audit"`
}
// HTTP endpoint logger
const (
EnvLoggerHTTPEndpoint = "MINIO_LOGGER_HTTP_ENDPOINT"
EnvAuditLoggerHTTPEndpoint = "MINIO_AUDIT_LOGGER_HTTP_ENDPOINT"
)
// Default target name when no targets are found
const (
defaultTarget = "_"
)
// LookupConfig - lookup logger config, override with ENVs if set.
func LookupConfig(cfg Config) (Config, error) {
envs := env.List(EnvLoggerHTTPEndpoint)
for _, e := range envs {
target := strings.TrimPrefix(e, EnvLoggerHTTPEndpoint)
if target == "" {
target = defaultTarget
}
_, ok := cfg.HTTP[target]
if ok {
cfg.HTTP[target] = HTTP{
Enabled: true,
Endpoint: env.Get(e, cfg.HTTP[target].Endpoint),
}
}
}
aenvs := env.List(EnvAuditLoggerHTTPEndpoint)
for _, e := range aenvs {
target := strings.TrimPrefix(e, EnvAuditLoggerHTTPEndpoint)
if target == "" {
target = defaultTarget
}
_, ok := cfg.Audit[target]
if ok {
cfg.Audit[target] = HTTP{
Enabled: true,
Endpoint: env.Get(e, cfg.Audit[target].Endpoint),
}
}
}
return cfg, nil
}

6
cmd/logger/console.go
View File

@ -28,14 +28,14 @@ import (
"github.com/minio/minio/pkg/color"
)
// Console interface describes the methods that need to be implemented to satisfy the interface requirements.
type Console interface {
// Logger interface describes the methods that need to be implemented to satisfy the interface requirements.
type Logger interface {
json(msg string, args ...interface{})
quiet(msg string, args ...interface{})
pretty(msg string, args ...interface{})
}
func consoleLog(console Console, msg string, args ...interface{}) {
func consoleLog(console Logger, msg string, args ...interface{}) {
switch {
case jsonFlag:
// Strip escape control characters from json message

12
cmd/server-main.go
View File

@ -136,7 +136,7 @@ EXAMPLES:
// Checks if endpoints are either available through environment
// or command line, returns false if both fails.
func endpointsPresent(ctx *cli.Context) bool {
_, ok := env.Lookup("MINIO_ENDPOINTS")
_, ok := env.Lookup(config.EnvEndpoints)
if !ok {
ok = ctx.Args().Present()
}
@ -158,7 +158,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
logger.FatalIf(uErr, "Unable to validate passed endpoints")
}
endpoints := strings.Fields(env.Get("MINIO_ENDPOINTS", ""))
endpoints := strings.Fields(env.Get(config.EnvEndpoints, ""))
if len(endpoints) > 0 {
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(globalCLIContext.Addr, endpoints...)
} else {
@ -216,7 +216,7 @@ func serverMain(ctx *cli.Context) {
logger.FatalIf(err, "Unable to load the TLS configuration")
// Check and load Root CAs.
globalRootCAs, err = getRootCAs(globalCertsCADir.Get())
globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
logger.FatalIf(err, "Failed to read root CAs (%v)", err)
// Handle all server environment vars.
@ -291,8 +291,9 @@ func serverMain(ctx *cli.Context) {
globalSweepHealState = initHealState()
}
// initialize globalConsoleSys system
// Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(context.Background(), globalEndpoints)
// Configure server.
var handler http.Handler
handler, err = configureServerHandler(globalEndpoints)
@ -338,9 +339,6 @@ func serverMain(ctx *cli.Context) {
logger.Fatal(err, "Unable to initialize config system")
}
// Load logger subsystem
loadLoggers()
// Create new IAM system.
globalIAMSys = NewIAMSys()
if err = globalIAMSys.Init(newObject); err != nil {

5
cmd/test-utils_test.go
View File

@ -529,11 +529,16 @@ func resetTestGlobals() {
// Configure the server for the test run.
func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) {
// Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(context.Background(), globalEndpoints)
// Initialize server config.
if err = newSrvConfig(obj); err != nil {
return err
}
globalServerConfig.Logger.Console.Enabled = false
// Set a default region.
globalServerConfig.SetRegion(bucketLocation)

15
pkg/env/env.go vendored
View File

@ -1,6 +1,9 @@
package env
import "os"
import (
"os"
"strings"
)
// Get retrieves the value of the environment variable named
// by the key. If the variable is present in the environment the
@ -19,3 +22,13 @@ func Get(key, defaultValue string) string {
// Otherwise the returned value will be empty and the boolean will
// be false.
func Lookup(key string) (string, bool) { return os.LookupEnv(key) }
// List all envs with a given prefix.
func List(prefix string) (envs []string) {
for _, env := range os.Environ() {
if strings.HasPrefix(env, prefix) {
envs = append(envs, env)
}
}
return envs
}

3
pkg/quick/encoding.go
View File

@ -179,9 +179,6 @@ func loadFileConfigEtcd(filename string, clnt *etcd.Client, v interface{}) error
// decoder format according to the filename extension. If no
// extension is provided, json will be selected by default.
func loadFileConfig(filename string, v interface{}) error {
if _, err := os.Stat(filename); err != nil {
return err
}
fileData, err := ioutil.ReadFile(filename)
if err != nil {
return err