MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 21:02:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

rename all remaining packages to internal/ (#12418)

Browse Source 
This is to ensure that there are no projects
that try to import `minio/minio/pkg` into
their own repo. Any such common packages should
go to `https://github.com/minio/pkg`
This commit is contained in:
Harshavardhana
2021-06-01 14:59:40 -07:00
committed by GitHub
parent bf87c4b1e4
commit 1f262daf6f
540 changed files with 757 additions and 778 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
internal/config/api/api.go Normal file
View File

@@ -0,0 +1,220 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"encoding/json"
"errors"
"strconv"
"strings"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/env"
)
// API sub-system constants
const (
apiRequestsMax = "requests_max"
apiRequestsDeadline = "requests_deadline"
apiClusterDeadline = "cluster_deadline"
apiCorsAllowOrigin = "cors_allow_origin"
apiRemoteTransportDeadline = "remote_transport_deadline"
apiListQuorum = "list_quorum"
apiExtendListCacheLife = "extend_list_cache_life"
apiReplicationWorkers = "replication_workers"
apiReplicationFailedWorkers = "replication_failed_workers"
EnvAPIRequestsMax = "MINIO_API_REQUESTS_MAX"
EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
EnvAPIClusterDeadline = "MINIO_API_CLUSTER_DEADLINE"
EnvAPICorsAllowOrigin = "MINIO_API_CORS_ALLOW_ORIGIN"
EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
EnvAPIListQuorum = "MINIO_API_LIST_QUORUM"
EnvAPIExtendListCacheLife = "MINIO_API_EXTEND_LIST_CACHE_LIFE"
EnvAPISecureCiphers = "MINIO_API_SECURE_CIPHERS"
EnvAPIReplicationWorkers = "MINIO_API_REPLICATION_WORKERS"
EnvAPIReplicationFailedWorkers = "MINIO_API_REPLICATION_FAILED_WORKERS"
)
// Deprecated key and ENVs
const (
apiReadyDeadline = "ready_deadline"
EnvAPIReadyDeadline = "MINIO_API_READY_DEADLINE"
)
// DefaultKVS - default storage class config
var (
DefaultKVS = config.KVS{
config.KV{
Key: apiRequestsMax,
Value: "0",
},
config.KV{
Key: apiRequestsDeadline,
Value: "10s",
},
config.KV{
Key: apiClusterDeadline,
Value: "10s",
},
config.KV{
Key: apiCorsAllowOrigin,
Value: "*",
},
config.KV{
Key: apiRemoteTransportDeadline,
Value: "2h",
},
config.KV{
Key: apiListQuorum,
Value: "optimal",
},
config.KV{
Key: apiExtendListCacheLife,
Value: "0s",
},
config.KV{
Key: apiReplicationWorkers,
Value: "250",
},
config.KV{
Key: apiReplicationFailedWorkers,
Value: "8",
},
}
)
// Config storage class configuration
type Config struct {
RequestsMax int `json:"requests_max"`
RequestsDeadline time.Duration `json:"requests_deadline"`
ClusterDeadline time.Duration `json:"cluster_deadline"`
CorsAllowOrigin []string `json:"cors_allow_origin"`
RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
ListQuorum string `json:"list_quorum"`
ExtendListLife time.Duration `json:"extend_list_cache_life"`
ReplicationWorkers int `json:"replication_workers"`
ReplicationFailedWorkers int `json:"replication_failed_workers"`
}
// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
func (sCfg *Config) UnmarshalJSON(data []byte) error {
type Alias Config
aux := &struct {
*Alias
}{
Alias: (*Alias)(sCfg),
}
return json.Unmarshal(data, &aux)
}
// GetListQuorum interprets list quorum values and returns appropriate
// acceptable quorum expected for list operations
func (sCfg Config) GetListQuorum() int {
switch sCfg.ListQuorum {
case "reduced":
return 2
case "disk":
// smallest possible value, generally meant for testing.
return 1
case "strict":
return -1
}
// Defaults to 3 drives per set, defaults to "optimal" value
return 3
}
// LookupConfig - lookup api config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
// remove this since we have removed this already.
kvs.Delete(apiReadyDeadline)
if err = config.CheckValidKeys(config.APISubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
// Check environment variables parameters
requestsMax, err := strconv.Atoi(env.Get(EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
if err != nil {
return cfg, err
}
if requestsMax < 0 {
return cfg, errors.New("invalid API max requests value")
}
requestsDeadline, err := time.ParseDuration(env.Get(EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
if err != nil {
return cfg, err
}
clusterDeadline, err := time.ParseDuration(env.Get(EnvAPIClusterDeadline, kvs.Get(apiClusterDeadline)))
if err != nil {
return cfg, err
}
corsAllowOrigin := strings.Split(env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)), ",")
remoteTransportDeadline, err := time.ParseDuration(env.Get(EnvAPIRemoteTransportDeadline, kvs.Get(apiRemoteTransportDeadline)))
if err != nil {
return cfg, err
}
listQuorum := env.Get(EnvAPIListQuorum, kvs.Get(apiListQuorum))
switch listQuorum {
case "strict", "optimal", "reduced", "disk":
default:
return cfg, errors.New("invalid value for list strict quorum")
}
listLife, err := time.ParseDuration(env.Get(EnvAPIExtendListCacheLife, kvs.Get(apiExtendListCacheLife)))
if err != nil {
return cfg, err
}
replicationWorkers, err := strconv.Atoi(env.Get(EnvAPIReplicationWorkers, kvs.Get(apiReplicationWorkers)))
if err != nil {
return cfg, err
}
if replicationWorkers <= 0 {
return cfg, config.ErrInvalidReplicationWorkersValue(nil).Msg("Minimum number of replication workers should be 1")
}
replicationFailedWorkers, err := strconv.Atoi(env.Get(EnvAPIReplicationFailedWorkers, kvs.Get(apiReplicationFailedWorkers)))
if err != nil {
return cfg, err
}
if replicationFailedWorkers <= 0 {
return cfg, config.ErrInvalidReplicationWorkersValue(nil).Msg("Minimum number of replication failed workers should be 1")
}
return Config{
RequestsMax: requestsMax,
RequestsDeadline: requestsDeadline,
ClusterDeadline: clusterDeadline,
CorsAllowOrigin: corsAllowOrigin,
RemoteTransportDeadline: remoteTransportDeadline,
ListQuorum: listQuorum,
ExtendListLife: listLife,
ReplicationWorkers: replicationWorkers,
ReplicationFailedWorkers: replicationFailedWorkers,
}, nil
}

62
internal/config/api/help.go Normal file
View File

@@ -0,0 +1,62 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package api
import "github.com/minio/minio/internal/config"
// Help template for storageclass feature.
var (
Help = config.HelpKVS{
config.HelpKV{
Key: apiRequestsMax,
Description: `set the maximum number of concurrent requests, e.g. "1600"`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: apiRequestsDeadline,
Description: `set the deadline for API requests waiting to be processed e.g. "1m"`,
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: apiCorsAllowOrigin,
Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`,
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: apiRemoteTransportDeadline,
Description: `set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h"`,
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: apiReplicationWorkers,
Description: `set the number of replication workers, defaults to 100`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: apiReplicationFailedWorkers,
Description: `set the number of replication workers for recently failed replicas, defaults to 4`,
Optional: true,
Type: "number",
},
}
)

91
internal/config/bool-flag.go Normal file
View File

@@ -0,0 +1,91 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"encoding/json"
"fmt"
"strconv"
"strings"
)
// BoolFlag - wrapper bool type.
type BoolFlag bool
// String - returns string of BoolFlag.
func (bf BoolFlag) String() string {
if bf {
return "on"
}
return "off"
}
// MarshalJSON - converts BoolFlag into JSON data.
func (bf BoolFlag) MarshalJSON() ([]byte, error) {
return json.Marshal(bf.String())
}
// UnmarshalJSON - parses given data into BoolFlag.
func (bf *BoolFlag) UnmarshalJSON(data []byte) (err error) {
var s string
if err = json.Unmarshal(data, &s); err == nil {
b := BoolFlag(true)
if s == "" {
// Empty string is treated as valid.
*bf = b
} else if b, err = ParseBoolFlag(s); err == nil {
*bf = b
}
}
return err
}
// FormatBool prints stringified version of boolean.
func FormatBool(b bool) string {
if b {
return "on"
}
return "off"
}
// ParseBool returns the boolean value represented by the string.
// It accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False.
// Any other value returns an error.
func ParseBool(str string) (bool, error) {
switch str {
case "1", "t", "T", "true", "TRUE", "True", "on", "ON", "On":
return true, nil
case "0", "f", "F", "false", "FALSE", "False", "off", "OFF", "Off":
return false, nil
}
if strings.EqualFold(str, "enabled") {
return true, nil
}
if strings.EqualFold(str, "disabled") {
return false, nil
}
return false, fmt.Errorf("ParseBool: parsing '%s': %s", str, strconv.ErrSyntax)
}
// ParseBoolFlag - parses string into BoolFlag.
func ParseBoolFlag(s string) (bf BoolFlag, err error) {
b, err := ParseBool(s)
return BoolFlag(b), err
}

129
internal/config/bool-flag_test.go Normal file
View File

@@ -0,0 +1,129 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"testing"
)
// Test BoolFlag.String()
func TestBoolFlagString(t *testing.T) {
var bf BoolFlag
testCases := []struct {
flag BoolFlag
expectedResult string
}{
{bf, "off"},
{BoolFlag(true), "on"},
{BoolFlag(false), "off"},
}
for _, testCase := range testCases {
str := testCase.flag.String()
if testCase.expectedResult != str {
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, str)
}
}
}
// Test BoolFlag.MarshalJSON()
func TestBoolFlagMarshalJSON(t *testing.T) {
var bf BoolFlag
testCases := []struct {
flag BoolFlag
expectedResult string
}{
{bf, `"off"`},
{BoolFlag(true), `"on"`},
{BoolFlag(false), `"off"`},
}
for _, testCase := range testCases {
data, _ := testCase.flag.MarshalJSON()
if testCase.expectedResult != string(data) {
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, string(data))
}
}
}
// Test BoolFlag.UnmarshalJSON()
func TestBoolFlagUnmarshalJSON(t *testing.T) {
testCases := []struct {
data []byte
expectedResult BoolFlag
expectedErr bool
}{
{[]byte(`{}`), BoolFlag(false), true},
{[]byte(`["on"]`), BoolFlag(false), true},
{[]byte(`"junk"`), BoolFlag(false), true},
{[]byte(`""`), BoolFlag(true), false},
{[]byte(`"on"`), BoolFlag(true), false},
{[]byte(`"off"`), BoolFlag(false), false},
{[]byte(`"true"`), BoolFlag(true), false},
{[]byte(`"false"`), BoolFlag(false), false},
{[]byte(`"ON"`), BoolFlag(true), false},
{[]byte(`"OFF"`), BoolFlag(false), false},
}
for _, testCase := range testCases {
var flag BoolFlag
err := (&flag).UnmarshalJSON(testCase.data)
if !testCase.expectedErr && err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
if testCase.expectedErr && err == nil {
t.Fatalf("error: expected error, got = <nil>")
}
if err == nil && testCase.expectedResult != flag {
t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, flag)
}
}
}
// Test ParseBoolFlag()
func TestParseBoolFlag(t *testing.T) {
testCases := []struct {
flagStr string
expectedResult BoolFlag
expectedErr bool
}{
{"", BoolFlag(false), true},
{"junk", BoolFlag(false), true},
{"true", BoolFlag(true), false},
{"false", BoolFlag(false), false},
{"ON", BoolFlag(true), false},
{"OFF", BoolFlag(false), false},
{"on", BoolFlag(true), false},
{"off", BoolFlag(false), false},
}
for _, testCase := range testCases {
bf, err := ParseBoolFlag(testCase.flagStr)
if !testCase.expectedErr && err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
if testCase.expectedErr && err == nil {
t.Fatalf("error: expected error, got = <nil>")
}
if err == nil && testCase.expectedResult != bf {
t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, bf)
}
}
}

166
internal/config/cache/config.go vendored Normal file
View File

@@ -0,0 +1,166 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cache
import (
"encoding/json"
"errors"
"path/filepath"
"strings"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/ellipses"
)
// Config represents cache config settings
type Config struct {
Enabled bool `json:"-"`
Drives []string `json:"drives"`
Expiry int `json:"expiry"`
MaxUse int `json:"maxuse"`
Quota int `json:"quota"`
Exclude []string `json:"exclude"`
After int `json:"after"`
WatermarkLow int `json:"watermark_low"`
WatermarkHigh int `json:"watermark_high"`
Range bool `json:"range"`
CommitWriteback bool `json:"-"`
}
// UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
// json entries for CacheConfig.
func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
type Alias Config
var _cfg = &struct {
*Alias
}{
Alias: (*Alias)(cfg),
}
if err = json.Unmarshal(data, _cfg); err != nil {
return err
}
if _cfg.Expiry < 0 {
return errors.New("config expiry value should not be negative")
}
if _cfg.MaxUse < 0 {
return errors.New("config max use value should not be null or negative")
}
if _cfg.Quota < 0 {
return errors.New("config quota value should not be null or negative")
}
if _cfg.After < 0 {
return errors.New("cache after value should not be less than 0")
}
if _cfg.WatermarkLow < 0 || _cfg.WatermarkLow > 100 {
return errors.New("config low watermark value should be between 0 and 100")
}
if _cfg.WatermarkHigh < 0 || _cfg.WatermarkHigh > 100 {
return errors.New("config high watermark value should be between 0 and 100")
}
if _cfg.WatermarkLow > 0 && (_cfg.WatermarkLow >= _cfg.WatermarkHigh) {
return errors.New("config low watermark value should be less than high watermark")
}
return nil
}
// Parses given cacheDrivesEnv and returns a list of cache drives.
func parseCacheDrives(drives string) ([]string, error) {
var drivesSlice []string
if len(drives) == 0 {
return drivesSlice, nil
}
drivesSlice = strings.Split(drives, cacheDelimiterLegacy)
if len(drivesSlice) == 1 && drivesSlice[0] == drives {
drivesSlice = strings.Split(drives, cacheDelimiter)
}
var endpoints []string
for _, d := range drivesSlice {
if len(d) == 0 {
return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir cannot be an empty path")
}
if ellipses.HasEllipses(d) {
s, err := parseCacheDrivePaths(d)
if err != nil {
return nil, err
}
endpoints = append(endpoints, s...)
} else {
endpoints = append(endpoints, d)
}
}
for _, d := range endpoints {
if !filepath.IsAbs(d) {
return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
}
}
return endpoints, nil
}
// Parses all arguments and returns a slice of drive paths following the ellipses pattern.
func parseCacheDrivePaths(arg string) (ep []string, err error) {
patterns, perr := ellipses.FindEllipsesPatterns(arg)
if perr != nil {
return []string{}, config.ErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
}
for _, lbls := range patterns.Expand() {
ep = append(ep, strings.Join(lbls, ""))
}
return ep, nil
}
// Parses given cacheExcludesEnv and returns a list of cache exclude patterns.
func parseCacheExcludes(excludes string) ([]string, error) {
var excludesSlice []string
if len(excludes) == 0 {
return excludesSlice, nil
}
excludesSlice = strings.Split(excludes, cacheDelimiterLegacy)
if len(excludesSlice) == 1 && excludesSlice[0] == excludes {
excludesSlice = strings.Split(excludes, cacheDelimiter)
}
for _, e := range excludesSlice {
if len(e) == 0 {
return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
}
if strings.HasPrefix(e, "/") {
return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
}
}
return excludesSlice, nil
}
func parseCacheCommitMode(commitStr string) (bool, error) {
switch strings.ToLower(commitStr) {
case "writeback":
return true, nil
case "writethrough":
return false, nil
}
return false, config.ErrInvalidCacheCommitValue(nil).Msg("cache commit value must be `writeback` or `writethrough`")
}

128
internal/config/cache/config_test.go vendored Normal file
View File

@@ -0,0 +1,128 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cache
import (
"reflect"
"runtime"
"testing"
)
// Tests cache drive parsing.
func TestParseCacheDrives(t *testing.T) {
testCases := []struct {
driveStr string
expectedPatterns []string
success bool
}{
// Invalid input
{"bucket1/*;*.png;images/trip/barcelona/*", []string{}, false},
{"bucket1", []string{}, false},
{";;;", []string{}, false},
{",;,;,;", []string{}, false},
}
// Valid inputs
if runtime.GOOS == "windows" {
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"C:/home/drive1;C:/home/drive2;C:/home/drive3", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"C:/home/drive{1...3}", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"C:/home/drive{1..3}", []string{}, false})
} else {
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"/home/drive1,/home/drive2,/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"/home/drive{1...3}", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"/home/drive{1..3}", []string{}, false})
}
for i, testCase := range testCases {
drives, err := parseCacheDrives(testCase.driveStr)
if err != nil && testCase.success {
t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
}
if err == nil && !testCase.success {
t.Errorf("Test %d: Expected failure but passed instead", i+1)
}
if err == nil {
if !reflect.DeepEqual(drives, testCase.expectedPatterns) {
t.Errorf("Test %d: Expected %v, got %v", i+1, testCase.expectedPatterns, drives)
}
}
}
}
// Tests cache exclude parsing.
func TestParseCacheExclude(t *testing.T) {
testCases := []struct {
excludeStr string
expectedPatterns []string
success bool
}{
// Invalid input
{"/home/drive1;/home/drive2;/home/drive3", []string{}, false},
{"/", []string{}, false},
{";;;", []string{}, false},
// valid input
{"bucket1/*;*.png;images/trip/barcelona/*", []string{"bucket1/*", "*.png", "images/trip/barcelona/*"}, true},
{"bucket1/*,*.png,images/trip/barcelona/*", []string{"bucket1/*", "*.png", "images/trip/barcelona/*"}, true},
{"bucket1", []string{"bucket1"}, true},
}
for i, testCase := range testCases {
excludes, err := parseCacheExcludes(testCase.excludeStr)
if err != nil && testCase.success {
t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
}
if err == nil && !testCase.success {
t.Errorf("Test %d: Expected failure but passed instead", i+1)
}
if err == nil {
if !reflect.DeepEqual(excludes, testCase.expectedPatterns) {
t.Errorf("Test %d: Expected %v, got %v", i+1, testCase.expectedPatterns, excludes)
}
}
}
}

85
internal/config/cache/help.go vendored Normal file
View File

@@ -0,0 +1,85 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cache
import "github.com/minio/minio/internal/config"
// Help template for caching feature.
var (
Help = config.HelpKVS{
config.HelpKV{
Key: Drives,
Description: `comma separated mountpoints e.g. "/optane1,/optane2"`,
Type: "csv",
},
config.HelpKV{
Key: Expiry,
Description: `cache expiry duration in days e.g. "90"`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: Quota,
Description: `limit cache drive usage in percentage e.g. "90"`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: Exclude,
Description: `exclude cache for following patterns e.g. "bucket/*.tmp,*.exe"`,
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: After,
Description: `minimum number of access before caching an object`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: WatermarkLow,
Description: `% of cache use at which to stop cache eviction`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: WatermarkHigh,
Description: `% of cache use at which to start cache eviction`,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: Range,
Description: `set to "on" or "off" caching of independent range requests per object, defaults to "on"`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: Commit,
Description: `set to control cache commit behavior, defaults to "writethrough"`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

55
internal/config/cache/legacy.go vendored Normal file
View File

@@ -0,0 +1,55 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cache
import (
"fmt"
"strings"
"github.com/minio/minio/internal/config"
)
const (
cacheDelimiterLegacy = ";"
)
// SetCacheConfig - One time migration code needed, for migrating from older config to new for Cache.
func SetCacheConfig(s config.Config, cfg Config) {
if len(cfg.Drives) == 0 {
// Do not save cache if no settings available.
return
}
s[config.CacheSubSys][config.Default] = config.KVS{
config.KV{
Key: Drives,
Value: strings.Join(cfg.Drives, cacheDelimiter),
},
config.KV{
Key: Exclude,
Value: strings.Join(cfg.Exclude, cacheDelimiter),
},
config.KV{
Key: Expiry,
Value: fmt.Sprintf("%d", cfg.Expiry),
},
config.KV{
Key: Quota,
Value: fmt.Sprintf("%d", cfg.MaxUse),
},
}
}

233
internal/config/cache/lookup.go vendored Normal file
View File

@@ -0,0 +1,233 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cache
import (
"errors"
"strconv"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/env"
)
// Cache ENVs
const (
Drives = "drives"
Exclude = "exclude"
Expiry = "expiry"
MaxUse = "maxuse"
Quota = "quota"
After = "after"
WatermarkLow = "watermark_low"
WatermarkHigh = "watermark_high"
Range = "range"
Commit = "commit"
EnvCacheDrives = "MINIO_CACHE_DRIVES"
EnvCacheExclude = "MINIO_CACHE_EXCLUDE"
EnvCacheExpiry = "MINIO_CACHE_EXPIRY"
EnvCacheMaxUse = "MINIO_CACHE_MAXUSE"
EnvCacheQuota = "MINIO_CACHE_QUOTA"
EnvCacheAfter = "MINIO_CACHE_AFTER"
EnvCacheWatermarkLow = "MINIO_CACHE_WATERMARK_LOW"
EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
EnvCacheRange = "MINIO_CACHE_RANGE"
EnvCacheCommit = "MINIO_CACHE_COMMIT"
EnvCacheEncryptionKey = "MINIO_CACHE_ENCRYPTION_SECRET_KEY"
DefaultExpiry = "90"
DefaultQuota = "80"
DefaultAfter = "0"
DefaultWaterMarkLow = "70"
DefaultWaterMarkHigh = "80"
DefaultCacheCommit = "writethrough"
)
// DefaultKVS - default KV settings for caching.
var (
DefaultKVS = config.KVS{
config.KV{
Key: Drives,
Value: "",
},
config.KV{
Key: Exclude,
Value: "",
},
config.KV{
Key: Expiry,
Value: DefaultExpiry,
},
config.KV{
Key: Quota,
Value: DefaultQuota,
},
config.KV{
Key: After,
Value: DefaultAfter,
},
config.KV{
Key: WatermarkLow,
Value: DefaultWaterMarkLow,
},
config.KV{
Key: WatermarkHigh,
Value: DefaultWaterMarkHigh,
},
config.KV{
Key: Range,
Value: config.EnableOn,
},
config.KV{
Key: Commit,
Value: DefaultCacheCommit,
},
}
)
const (
cacheDelimiter = ","
)
// Enabled returns if cache is enabled.
func Enabled(kvs config.KVS) bool {
drives := kvs.Get(Drives)
return drives != ""
}
// LookupConfig - extracts cache configuration provided by environment
// variables and merge them with provided CacheConfiguration.
func LookupConfig(kvs config.KVS) (Config, error) {
cfg := Config{}
if err := config.CheckValidKeys(config.CacheSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
drives := env.Get(EnvCacheDrives, kvs.Get(Drives))
if len(drives) == 0 {
return cfg, nil
}
var err error
cfg.Drives, err = parseCacheDrives(drives)
if err != nil {
return cfg, err
}
cfg.Enabled = true
if excludes := env.Get(EnvCacheExclude, kvs.Get(Exclude)); excludes != "" {
cfg.Exclude, err = parseCacheExcludes(excludes)
if err != nil {
return cfg, err
}
}
if expiryStr := env.Get(EnvCacheExpiry, kvs.Get(Expiry)); expiryStr != "" {
cfg.Expiry, err = strconv.Atoi(expiryStr)
if err != nil {
return cfg, config.ErrInvalidCacheExpiryValue(err)
}
}
if maxUseStr := env.Get(EnvCacheMaxUse, kvs.Get(MaxUse)); maxUseStr != "" {
cfg.MaxUse, err = strconv.Atoi(maxUseStr)
if err != nil {
return cfg, config.ErrInvalidCacheQuota(err)
}
// maxUse should be a valid percentage.
if cfg.MaxUse < 0 || cfg.MaxUse > 100 {
err := errors.New("config max use value should not be null or negative")
return cfg, config.ErrInvalidCacheQuota(err)
}
cfg.Quota = cfg.MaxUse
} else if quotaStr := env.Get(EnvCacheQuota, kvs.Get(Quota)); quotaStr != "" {
cfg.Quota, err = strconv.Atoi(quotaStr)
if err != nil {
return cfg, config.ErrInvalidCacheQuota(err)
}
// quota should be a valid percentage.
if cfg.Quota < 0 || cfg.Quota > 100 {
err := errors.New("config quota value should not be null or negative")
return cfg, config.ErrInvalidCacheQuota(err)
}
cfg.MaxUse = cfg.Quota
}
if afterStr := env.Get(EnvCacheAfter, kvs.Get(After)); afterStr != "" {
cfg.After, err = strconv.Atoi(afterStr)
if err != nil {
return cfg, config.ErrInvalidCacheAfter(err)
}
// after should be a valid value >= 0.
if cfg.After < 0 {
err := errors.New("cache after value cannot be less than 0")
return cfg, config.ErrInvalidCacheAfter(err)
}
}
if lowWMStr := env.Get(EnvCacheWatermarkLow, kvs.Get(WatermarkLow)); lowWMStr != "" {
cfg.WatermarkLow, err = strconv.Atoi(lowWMStr)
if err != nil {
return cfg, config.ErrInvalidCacheWatermarkLow(err)
}
// WatermarkLow should be a valid percentage.
if cfg.WatermarkLow < 0 || cfg.WatermarkLow > 100 {
err := errors.New("config min watermark value should be between 0 and 100")
return cfg, config.ErrInvalidCacheWatermarkLow(err)
}
}
if highWMStr := env.Get(EnvCacheWatermarkHigh, kvs.Get(WatermarkHigh)); highWMStr != "" {
cfg.WatermarkHigh, err = strconv.Atoi(highWMStr)
if err != nil {
return cfg, config.ErrInvalidCacheWatermarkHigh(err)
}
// MaxWatermark should be a valid percentage.
if cfg.WatermarkHigh < 0 || cfg.WatermarkHigh > 100 {
err := errors.New("config high watermark value should be between 0 and 100")
return cfg, config.ErrInvalidCacheWatermarkHigh(err)
}
}
if cfg.WatermarkLow > cfg.WatermarkHigh {
err := errors.New("config high watermark value should be greater than low watermark value")
return cfg, config.ErrInvalidCacheWatermarkHigh(err)
}
cfg.Range = true // by default range caching is enabled.
if rangeStr := env.Get(EnvCacheRange, kvs.Get(Range)); rangeStr != "" {
rng, err := config.ParseBool(rangeStr)
if err != nil {
return cfg, config.ErrInvalidCacheRange(err)
}
cfg.Range = rng
}
if commit := env.Get(EnvCacheCommit, kvs.Get(Commit)); commit != "" {
cfg.CommitWriteback, err = parseCacheCommitMode(commit)
if err != nil {
return cfg, err
}
if cfg.After > 0 && cfg.CommitWriteback {
err := errors.New("cache after cannot be used with commit writeback")
return cfg, config.ErrInvalidCacheSetting(err)
}
}
return cfg, nil
}

126
internal/config/certs.go Normal file
View File

@@ -0,0 +1,126 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"bytes"
"crypto"
"crypto/ecdsa"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"io/ioutil"
"github.com/minio/pkg/env"
)
// EnvCertPassword is the environment variable which contains the password used
// to decrypt the TLS private key. It must be set if the TLS private key is
// password protected.
const EnvCertPassword = "MINIO_CERT_PASSWD"
// ParsePublicCertFile - parses public cert into its *x509.Certificate equivalent.
func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
// Read certificate file.
var data []byte
if data, err = ioutil.ReadFile(certFile); err != nil {
return nil, err
}
// Trimming leading and tailing white spaces.
data = bytes.TrimSpace(data)
// Parse all certs in the chain.
current := data
for len(current) > 0 {
var pemBlock *pem.Block
if pemBlock, current = pem.Decode(current); pemBlock == nil {
return nil, ErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
}
var x509Cert *x509.Certificate
if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
return nil, ErrSSLUnexpectedData(err)
}
x509Certs = append(x509Certs, x509Cert)
}
if len(x509Certs) == 0 {
return nil, ErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
}
return x509Certs, nil
}
// LoadX509KeyPair - load an X509 key pair (private key , certificate)
// from the provided paths. The private key may be encrypted and is
// decrypted using the ENV_VAR: MINIO_CERT_PASSWD.
func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
certPEMBlock, err := ioutil.ReadFile(certFile)
if err != nil {
return tls.Certificate{}, ErrSSLUnexpectedError(err)
}
keyPEMBlock, err := ioutil.ReadFile(keyFile)
if err != nil {
return tls.Certificate{}, ErrSSLUnexpectedError(err)
}
key, rest := pem.Decode(keyPEMBlock)
if len(rest) > 0 {
return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
}
if x509.IsEncryptedPEMBlock(key) {
password := env.Get(EnvCertPassword, "")
if len(password) == 0 {
return tls.Certificate{}, ErrSSLNoPassword(nil)
}
decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
if decErr != nil {
return tls.Certificate{}, ErrSSLWrongPassword(decErr)
}
keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
}
cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
if err != nil {
return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg(err.Error())
}
// Ensure that the private key is not a P-384 or P-521 EC key.
// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
switch pub.Params().Name {
case "P-384":
fallthrough
case "P-521":
// unfortunately there is no cleaner way to check
return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", pub.Params().Name)
}
}
}
return cert, nil
}
// EnsureCertAndKey checks if both client certificate and key paths are provided
func EnsureCertAndKey(ClientCert, ClientKey string) error {
if (ClientCert != "" && ClientKey == "") ||
(ClientCert == "" && ClientKey != "") {
return errors.New("cert and key must be specified as a pair")
}
return nil
}

444
internal/config/certs_test.go Normal file
View File

@@ -0,0 +1,444 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"fmt"
"io/ioutil"
"os"
"runtime"
"testing"
)
func createTempFile(prefix, content string) (tempFile string, err error) {
var tmpfile *os.File
if tmpfile, err = ioutil.TempFile("", prefix); err != nil {
return tempFile, err
}
if _, err = tmpfile.Write([]byte(content)); err != nil {
return tempFile, err
}
if err = tmpfile.Close(); err != nil {
return tempFile, err
}
tempFile = tmpfile.Name()
return tempFile, err
}
func TestParsePublicCertFile(t *testing.T) {
tempFile1, err := createTempFile("public-cert-file", "")
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile1)
tempFile2, err := createTempFile("public-cert-file",
`-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBA-some-junk-Q4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----`)
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile2)
tempFile3, err := createTempFile("public-cert-file",
`-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBAabababababaQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----`)
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile3)
tempFile4, err := createTempFile("public-cert-file",
`-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----`)
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile4)
tempFile5, err := createTempFile("public-cert-file",
`-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
M9ofSEt/bdRD
-----END CERTIFICATE-----`)
if err != nil {
t.Fatalf("Unable to create temporary file. %v", err)
}
defer os.Remove(tempFile5)
nonexistentErr := fmt.Errorf("open nonexistent-file: no such file or directory")
if runtime.GOOS == "windows" {
// Below concatenation is done to get rid of goline error
// "error strings should not be capitalized or end with punctuation or a newline"
nonexistentErr = fmt.Errorf("open nonexistent-file:" + " The system cannot find the file specified.")
}
testCases := []struct {
certFile string
expectedResultLen int
expectedErr error
}{
{"nonexistent-file", 0, nonexistentErr},
{tempFile1, 0, fmt.Errorf("Empty public certificate file %s", tempFile1)},
{tempFile2, 0, fmt.Errorf("Could not read PEM block from file %s", tempFile2)},
{tempFile3, 0, fmt.Errorf("asn1: structure error: sequence tag mismatch")},
{tempFile4, 1, nil},
{tempFile5, 2, nil},
}
for _, testCase := range testCases {
certs, err := ParsePublicCertFile(testCase.certFile)
if testCase.expectedErr == nil {
if err != nil {
t.Fatalf("error: expected = <nil>, got = %v", err)
}
} else if err == nil {
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
} else if testCase.expectedErr.Error() != err.Error() {
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
}
if len(certs) != testCase.expectedResultLen {
t.Fatalf("certs: expected = %v, got = %v", testCase.expectedResultLen, len(certs))
}
}
}
func TestLoadX509KeyPair(t *testing.T) {
for i, testCase := range loadX509KeyPairTests {
privateKey, err := createTempFile("private.key", testCase.privateKey)
if err != nil {
t.Fatalf("Test %d: failed to create tmp private key file: %v", i, err)
}
certificate, err := createTempFile("public.crt", testCase.certificate)
if err != nil {
os.Remove(privateKey)
t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
}
os.Unsetenv(EnvCertPassword)
if testCase.password != "" {
os.Setenv(EnvCertPassword, testCase.password)
}
_, err = LoadX509KeyPair(certificate, privateKey)
if err != nil && !testCase.shouldFail {
t.Errorf("Test %d: test should succeed but it failed: %v", i, err)
}
if err == nil && testCase.shouldFail {
t.Errorf("Test %d: test should fail but it succeed", i)
}
os.Remove(privateKey)
os.Remove(certificate)
}
}
var loadX509KeyPairTests = []struct {
password string
privateKey, certificate string
shouldFail bool
}{
{
password: "foobar",
privateKey: `-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285
nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb
AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h
uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j
cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj
q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog
lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB
bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU
IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO
i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1
r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ
ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh
dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA
ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8
piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3
6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL
GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy
HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz
KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg
UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL
ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo
37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi
zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx
btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx
9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B
aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k
-----END RSA PRIVATE KEY-----`,
certificate: `-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6
HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR
GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc
VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla
Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd
cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD
0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy
iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ
b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj
KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb
bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl
kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR
-----END CERTIFICATE-----`,
shouldFail: false,
},
{
password: "password",
privateKey: `-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285
nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb
AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h
uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j
cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj
q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog
lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB
bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU
IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO
i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1
r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ
ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh
dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA
ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8
piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3
6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL
GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy
HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz
KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg
UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL
ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo
37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi
zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx
btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx
9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B
aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k
-----END RSA PRIVATE KEY-----`,
certificate: `-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6
HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR
GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc
VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla
Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd
cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD
0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy
iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ
b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj
KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb
bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl
kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR
-----END CERTIFICATE-----`,
shouldFail: true,
},
{
password: "",
privateKey: `-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw
WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6
Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM
6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg
QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP
mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ
+PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd
qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4
1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3
VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa
Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI
hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T
FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz
92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO
hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF
yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna
eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM
M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i
ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX
8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1
qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K
wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5
JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk
+jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW
0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg==
-----END RSA PRIVATE KEY-----`,
certificate: `-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9
ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz
EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr
z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq
zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI
r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0
d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+
Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw
SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd
5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt
1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO
OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD
-----END CERTIFICATE-----`,
shouldFail: false,
},
{
password: "foobar",
privateKey: `-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw
WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6
Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM
6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg
QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP
mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ
+PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd
qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4
1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3
VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa
Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI
hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T
FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz
92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO
hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF
yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna
eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM
M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i
ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX
8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1
qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K
wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5
JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk
+jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW
0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg==
-----END RSA PRIVATE KEY-----`,
certificate: `-----BEGIN CERTIFICATE-----
MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9
ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz
EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr
z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq
zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI
r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0
d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+
Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw
SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd
5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt
1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO
OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD
-----END CERTIFICATE-----`,
shouldFail: false,
},
}

93
internal/config/certsinfo.go Normal file
View File

@@ -0,0 +1,93 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"net/http"
"strings"
color "github.com/minio/minio/internal/color"
)
// Extra ASN1 OIDs that we may need to handle
var (
oidEmailAddress = []int{1, 2, 840, 113549, 1, 9, 1}
)
// printName prints the fields of a distinguished name, which include such
// things as its common name and locality.
func printName(names []pkix.AttributeTypeAndValue, buf *strings.Builder) []string {
values := []string{}
for _, name := range names {
oid := name.Type
if len(oid) == 4 && oid[0] == 2 && oid[1] == 5 && oid[2] == 4 {
switch oid[3] {
case 3:
values = append(values, fmt.Sprintf("CN=%s", name.Value))
case 6:
values = append(values, fmt.Sprintf("C=%s", name.Value))
case 8:
values = append(values, fmt.Sprintf("ST=%s", name.Value))
case 10:
values = append(values, fmt.Sprintf("O=%s", name.Value))
case 11:
values = append(values, fmt.Sprintf("OU=%s", name.Value))
default:
values = append(values, fmt.Sprintf("UnknownOID=%s", name.Type.String()))
}
} else if oid.Equal(oidEmailAddress) {
values = append(values, fmt.Sprintf("emailAddress=%s", name.Value))
} else {
values = append(values, fmt.Sprintf("UnknownOID=%s", name.Type.String()))
}
}
if len(values) > 0 {
buf.WriteString(values[0])
for i := 1; i < len(values); i++ {
buf.WriteString(", " + values[i])
}
buf.WriteString("\n")
}
return values
}
// CertificateText returns a human-readable string representation
// of the certificate cert. The format is similar to the OpenSSL
// way of printing certificates (not identical).
func CertificateText(cert *x509.Certificate) string {
var buf strings.Builder
buf.WriteString(color.Blue("\nCertificate:\n"))
if cert.SignatureAlgorithm != x509.UnknownSignatureAlgorithm {
buf.WriteString(color.Blue("%4sSignature Algorithm: ", "") + color.Bold(fmt.Sprintf("%s\n", cert.SignatureAlgorithm)))
}
// Issuer information
buf.WriteString(color.Blue("%4sIssuer: ", ""))
printName(cert.Issuer.Names, &buf)
// Validity information
buf.WriteString(color.Blue("%4sValidity\n", ""))
buf.WriteString(color.Bold(fmt.Sprintf("%8sNot Before: %s\n", "", cert.NotBefore.Format(http.TimeFormat))))
buf.WriteString(color.Bold(fmt.Sprintf("%8sNot After : %s\n", "", cert.NotAfter.Format(http.TimeFormat))))
return buf.String()
}

146
internal/config/compress/compress.go Normal file
View File

@@ -0,0 +1,146 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package compress
import (
"fmt"
"strings"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/env"
)
// Config represents the compression settings.
type Config struct {
Enabled bool `json:"enabled"`
AllowEncrypted bool `json:"allow_encryption"`
Extensions []string `json:"extensions"`
MimeTypes []string `json:"mime-types"`
}
// Compression environment variables
const (
Extensions = "extensions"
AllowEncrypted = "allow_encryption"
MimeTypes = "mime_types"
EnvCompressState = "MINIO_COMPRESS_ENABLE"
EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
EnvCompressMimeTypes = "MINIO_COMPRESS_MIME_TYPES"
// Include-list for compression.
DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
DefaultMimeTypes = "text/*,application/json,application/xml,binary/octet-stream"
)
// DefaultKVS - default KV config for compression settings
var (
DefaultKVS = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOff,
},
config.KV{
Key: AllowEncrypted,
Value: config.EnableOff,
},
config.KV{
Key: Extensions,
Value: DefaultExtensions,
},
config.KV{
Key: MimeTypes,
Value: DefaultMimeTypes,
},
}
)
// Parses the given compression exclude list `extensions` or `content-types`.
func parseCompressIncludes(include string) ([]string, error) {
includes := strings.Split(include, config.ValueSeparator)
for _, e := range includes {
if len(e) == 0 {
return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be empty")
}
if e == "/" {
return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be '/'")
}
}
return includes, nil
}
// LookupConfig - lookup compression config.
func LookupConfig(kvs config.KVS) (Config, error) {
var err error
cfg := Config{}
if err = config.CheckValidKeys(config.CompressionSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
compress := env.Get(EnvCompress, "")
if compress == "" {
compress = env.Get(EnvCompressState, kvs.Get(config.Enable))
}
cfg.Enabled, err = config.ParseBool(compress)
if err != nil {
// Parsing failures happen due to empty KVS, ignore it.
if kvs.Empty() {
return cfg, nil
}
return cfg, err
}
if !cfg.Enabled {
return cfg, nil
}
allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted))
cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
if err != nil {
return cfg, err
}
compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))
if compressExtensions != "" || compressMimeTypes != "" || compressMimeTypesLegacy != "" {
if compressExtensions != "" {
extensions, err := parseCompressIncludes(compressExtensions)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
}
cfg.Extensions = extensions
}
if compressMimeTypes != "" {
mimeTypes, err := parseCompressIncludes(compressMimeTypes)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
}
cfg.MimeTypes = mimeTypes
}
if compressMimeTypesLegacy != "" {
mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy)
if err != nil {
return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
}
cfg.MimeTypes = mimeTypes
}
}
return cfg, nil
}

58
internal/config/compress/compress_test.go Normal file
View File

@@ -0,0 +1,58 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package compress
import (
"reflect"
"testing"
)
func TestParseCompressIncludes(t *testing.T) {
testCases := []struct {
str string
expectedPatterns []string
success bool
}{
// invalid input
{",,,", []string{}, false},
{"", []string{}, false},
{",", []string{}, false},
{"/", []string{}, false},
{"text/*,/", []string{}, false},
// valid input
{".txt,.log", []string{".txt", ".log"}, true},
{"text/*,application/json", []string{"text/*", "application/json"}, true},
}
for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.str, func(t *testing.T) {
gotPatterns, err := parseCompressIncludes(testCase.str)
if !testCase.success && err == nil {
t.Error("expected failure but success instead")
}
if testCase.success && err != nil {
t.Errorf("expected success but failed instead %s", err)
}
if testCase.success && !reflect.DeepEqual(testCase.expectedPatterns, gotPatterns) {
t.Errorf("expected patterns %s but got %s", testCase.expectedPatterns, gotPatterns)
}
})
}
}

44
internal/config/compress/help.go Normal file
View File

@@ -0,0 +1,44 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package compress
import "github.com/minio/minio/internal/config"
// Help template for compress feature.
var (
Help = config.HelpKVS{
config.HelpKV{
Key: Extensions,
Description: `comma separated file extensions e.g. ".txt,.log,.csv"`,
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: MimeTypes,
Description: `comma separated wildcard mime-types e.g. "text/*,application/json,application/xml"`,
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

52
internal/config/compress/legacy.go Normal file
View File

@@ -0,0 +1,52 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package compress
import (
"strings"
"github.com/minio/minio/internal/config"
)
// Legacy envs.
const (
EnvCompress = "MINIO_COMPRESS"
EnvCompressMimeTypesLegacy = "MINIO_COMPRESS_MIMETYPES"
)
// SetCompressionConfig - One time migration code needed, for migrating from older config to new for Compression.
func SetCompressionConfig(s config.Config, cfg Config) {
if !cfg.Enabled {
// No need to save disabled settings in new config.
return
}
s[config.CompressionSubSys][config.Default] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: Extensions,
Value: strings.Join(cfg.Extensions, config.ValueSeparator),
},
config.KV{
Key: MimeTypes,
Value: strings.Join(cfg.MimeTypes, config.ValueSeparator),
},
}
}

763
internal/config/config.go Normal file
View File

@@ -0,0 +1,763 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"bufio"
"fmt"
"io"
"regexp"
"strings"
"github.com/minio/madmin-go"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/auth"
"github.com/minio/pkg/env"
)
// Error config error type
type Error struct {
Err string
}
// Errorf - formats according to a format specifier and returns
// the string as a value that satisfies error of type config.Error
func Errorf(format string, a ...interface{}) error {
return Error{Err: fmt.Sprintf(format, a...)}
}
func (e Error) Error() string {
return e.Err
}
// Default keys
const (
Default = madmin.Default
Enable = madmin.EnableKey
Comment = madmin.CommentKey
// Enable values
EnableOn = madmin.EnableOn
EnableOff = madmin.EnableOff
RegionName = "name"
AccessKey = "access_key"
SecretKey = "secret_key"
)
// Top level config constants.
const (
CredentialsSubSys = "credentials"
PolicyOPASubSys = "policy_opa"
IdentityOpenIDSubSys = "identity_openid"
IdentityLDAPSubSys = "identity_ldap"
CacheSubSys = "cache"
RegionSubSys = "region"
EtcdSubSys = "etcd"
StorageClassSubSys = "storage_class"
APISubSys = "api"
CompressionSubSys = "compression"
KmsVaultSubSys = "kms_vault"
KmsKesSubSys = "kms_kes"
LoggerWebhookSubSys = "logger_webhook"
AuditWebhookSubSys = "audit_webhook"
HealSubSys = "heal"
ScannerSubSys = "scanner"
CrawlerSubSys = "crawler"
// Add new constants here if you add new fields to config.
)
// Notification config constants.
const (
NotifyKafkaSubSys = "notify_kafka"
NotifyMQTTSubSys = "notify_mqtt"
NotifyMySQLSubSys = "notify_mysql"
NotifyNATSSubSys = "notify_nats"
NotifyNSQSubSys = "notify_nsq"
NotifyESSubSys = "notify_elasticsearch"
NotifyAMQPSubSys = "notify_amqp"
NotifyPostgresSubSys = "notify_postgres"
NotifyRedisSubSys = "notify_redis"
NotifyWebhookSubSys = "notify_webhook"
// Add new constants here if you add new fields to config.
)
// SubSystems - all supported sub-systems
var SubSystems = set.CreateStringSet(
CredentialsSubSys,
RegionSubSys,
EtcdSubSys,
CacheSubSys,
APISubSys,
StorageClassSubSys,
CompressionSubSys,
KmsVaultSubSys,
KmsKesSubSys,
LoggerWebhookSubSys,
AuditWebhookSubSys,
PolicyOPASubSys,
IdentityLDAPSubSys,
IdentityOpenIDSubSys,
ScannerSubSys,
HealSubSys,
NotifyAMQPSubSys,
NotifyESSubSys,
NotifyKafkaSubSys,
NotifyMQTTSubSys,
NotifyMySQLSubSys,
NotifyNATSSubSys,
NotifyNSQSubSys,
NotifyPostgresSubSys,
NotifyRedisSubSys,
NotifyWebhookSubSys,
)
// SubSystemsDynamic - all sub-systems that have dynamic config.
var SubSystemsDynamic = set.CreateStringSet(
APISubSys,
CompressionSubSys,
ScannerSubSys,
HealSubSys,
)
// SubSystemsSingleTargets - subsystems which only support single target.
var SubSystemsSingleTargets = set.CreateStringSet([]string{
CredentialsSubSys,
RegionSubSys,
EtcdSubSys,
CacheSubSys,
APISubSys,
StorageClassSubSys,
CompressionSubSys,
KmsVaultSubSys,
KmsKesSubSys,
PolicyOPASubSys,
IdentityLDAPSubSys,
IdentityOpenIDSubSys,
HealSubSys,
ScannerSubSys,
}...)
// Constant separators
const (
SubSystemSeparator = madmin.SubSystemSeparator
KvSeparator = madmin.KvSeparator
KvSpaceSeparator = madmin.KvSpaceSeparator
KvComment = madmin.KvComment
KvNewline = madmin.KvNewline
KvDoubleQuote = madmin.KvDoubleQuote
KvSingleQuote = madmin.KvSingleQuote
// Env prefix used for all envs in MinIO
EnvPrefix = "MINIO_"
EnvWordDelimiter = `_`
)
// DefaultKVS - default kvs for all sub-systems
var DefaultKVS map[string]KVS
// RegisterDefaultKVS - this function saves input kvsMap
// globally, this should be called only once preferably
// during `init()`.
func RegisterDefaultKVS(kvsMap map[string]KVS) {
DefaultKVS = map[string]KVS{}
for subSys, kvs := range kvsMap {
DefaultKVS[subSys] = kvs
}
}
// HelpSubSysMap - help for all individual KVS for each sub-systems
// also carries a special empty sub-system which dumps
// help for each sub-system key.
var HelpSubSysMap map[string]HelpKVS
// RegisterHelpSubSys - this function saves
// input help KVS for each sub-system globally,
// this function should be called only once
// preferably in during `init()`.
func RegisterHelpSubSys(helpKVSMap map[string]HelpKVS) {
HelpSubSysMap = map[string]HelpKVS{}
for subSys, hkvs := range helpKVSMap {
HelpSubSysMap[subSys] = hkvs
}
}
// KV - is a shorthand of each key value.
type KV struct {
Key string `json:"key"`
Value string `json:"value"`
}
// KVS - is a shorthand for some wrapper functions
// to operate on list of key values.
type KVS []KV
// Empty - return if kv is empty
func (kvs KVS) Empty() bool {
return len(kvs) == 0
}
// Keys returns the list of keys for the current KVS
func (kvs KVS) Keys() []string {
var keys = make([]string, len(kvs))
var foundComment bool
for i := range kvs {
if kvs[i].Key == madmin.CommentKey {
foundComment = true
}
keys[i] = kvs[i].Key
}
// Comment KV not found, add it explicitly.
if !foundComment {
keys = append(keys, madmin.CommentKey)
}
return keys
}
func (kvs KVS) String() string {
var s strings.Builder
for _, kv := range kvs {
// Do not need to print if state is on
if kv.Key == Enable && kv.Value == EnableOn {
continue
}
s.WriteString(kv.Key)
s.WriteString(KvSeparator)
spc := madmin.HasSpace(kv.Value)
if spc {
s.WriteString(KvDoubleQuote)
}
s.WriteString(kv.Value)
if spc {
s.WriteString(KvDoubleQuote)
}
s.WriteString(KvSpaceSeparator)
}
return s.String()
}
// Set sets a value, if not sets a default value.
func (kvs *KVS) Set(key, value string) {
for i, kv := range *kvs {
if kv.Key == key {
(*kvs)[i] = KV{
Key: key,
Value: value,
}
return
}
}
*kvs = append(*kvs, KV{
Key: key,
Value: value,
})
}
// Get - returns the value of a key, if not found returns empty.
func (kvs KVS) Get(key string) string {
v, ok := kvs.Lookup(key)
if ok {
return v
}
return ""
}
// Delete - deletes the key if present from the KV list.
func (kvs *KVS) Delete(key string) {
for i, kv := range *kvs {
if kv.Key == key {
*kvs = append((*kvs)[:i], (*kvs)[i+1:]...)
return
}
}
}
// Lookup - lookup a key in a list of KVS
func (kvs KVS) Lookup(key string) (string, bool) {
for _, kv := range kvs {
if kv.Key == key {
return kv.Value, true
}
}
return "", false
}
// Config - MinIO server config structure.
type Config map[string]map[string]KVS
// DelFrom - deletes all keys in the input reader.
func (c Config) DelFrom(r io.Reader) error {
scanner := bufio.NewScanner(r)
for scanner.Scan() {
// Skip any empty lines, or comment like characters
text := scanner.Text()
if text == "" || strings.HasPrefix(text, KvComment) {
continue
}
if err := c.DelKVS(text); err != nil {
return err
}
}
return scanner.Err()
}
// ReadConfig - read content from input and write into c.
// Returns whether all parameters were dynamic.
func (c Config) ReadConfig(r io.Reader) (dynOnly bool, err error) {
var n int
scanner := bufio.NewScanner(r)
dynOnly = true
for scanner.Scan() {
// Skip any empty lines, or comment like characters
text := scanner.Text()
if text == "" || strings.HasPrefix(text, KvComment) {
continue
}
dynamic, err := c.SetKVS(text, DefaultKVS)
if err != nil {
return false, err
}
dynOnly = dynOnly && dynamic
n += len(text)
}
if err := scanner.Err(); err != nil {
return false, err
}
return dynOnly, nil
}
type configWriteTo struct {
Config
filterByKey string
}
// NewConfigWriteTo - returns a struct which
// allows for serializing the config/kv struct
// to a io.WriterTo
func NewConfigWriteTo(cfg Config, key string) io.WriterTo {
return &configWriteTo{Config: cfg, filterByKey: key}
}
// WriteTo - implements io.WriterTo interface implementation for config.
func (c *configWriteTo) WriteTo(w io.Writer) (int64, error) {
kvsTargets, err := c.GetKVS(c.filterByKey, DefaultKVS)
if err != nil {
return 0, err
}
var n int
for _, target := range kvsTargets {
m1, _ := w.Write([]byte(target.SubSystem))
m2, _ := w.Write([]byte(KvSpaceSeparator))
m3, _ := w.Write([]byte(target.KVS.String()))
if len(kvsTargets) > 1 {
m4, _ := w.Write([]byte(KvNewline))
n += m1 + m2 + m3 + m4
} else {
n += m1 + m2 + m3
}
}
return int64(n), nil
}
// Default KV configs for worm and region
var (
DefaultCredentialKVS = KVS{
KV{
Key: AccessKey,
Value: auth.DefaultAccessKey,
},
KV{
Key: SecretKey,
Value: auth.DefaultSecretKey,
},
}
DefaultRegionKVS = KVS{
KV{
Key: RegionName,
Value: "",
},
}
)
// LookupCreds - lookup credentials from config.
func LookupCreds(kv KVS) (auth.Credentials, error) {
if err := CheckValidKeys(CredentialsSubSys, kv, DefaultCredentialKVS); err != nil {
return auth.Credentials{}, err
}
accessKey := kv.Get(AccessKey)
secretKey := kv.Get(SecretKey)
if accessKey == "" || secretKey == "" {
accessKey = auth.DefaultAccessKey
secretKey = auth.DefaultSecretKey
}
return auth.CreateCredentials(accessKey, secretKey)
}
var validRegionRegex = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9-_-]+$")
// LookupRegion - get current region.
func LookupRegion(kv KVS) (string, error) {
if err := CheckValidKeys(RegionSubSys, kv, DefaultRegionKVS); err != nil {
return "", err
}
region := env.Get(EnvRegion, "")
if region == "" {
region = env.Get(EnvRegionName, kv.Get(RegionName))
}
if region != "" {
if validRegionRegex.MatchString(region) {
return region, nil
}
return "", Errorf(
"region '%s' is invalid, expected simple characters such as [us-east-1, myregion...]",
region)
}
return "", nil
}
// CheckValidKeys - checks if inputs KVS has the necessary keys,
// returns error if it find extra or superflous keys.
func CheckValidKeys(subSys string, kv KVS, validKVS KVS) error {
nkv := KVS{}
for _, kv := range kv {
// Comment is a valid key, its also fully optional
// ignore it since it is a valid key for all
// sub-systems.
if kv.Key == Comment {
continue
}
if _, ok := validKVS.Lookup(kv.Key); !ok {
nkv = append(nkv, kv)
}
}
if len(nkv) > 0 {
return Errorf(
"found invalid keys (%s) for '%s' sub-system, use 'mc admin config reset myminio %s' to fix invalid keys", nkv.String(), subSys, subSys)
}
return nil
}
// LookupWorm - check if worm is enabled
func LookupWorm() (bool, error) {
return ParseBool(env.Get(EnvWorm, EnableOff))
}
// Carries all the renamed sub-systems from their
// previously known names
var renamedSubsys = map[string]string{
CrawlerSubSys: ScannerSubSys,
// Add future sub-system renames
}
// Merge - merges a new config with all the
// missing values for default configs,
// returns a config.
func (c Config) Merge() Config {
cp := New()
for subSys, tgtKV := range c {
for tgt := range tgtKV {
ckvs := c[subSys][tgt]
for _, kv := range cp[subSys][Default] {
_, ok := c[subSys][tgt].Lookup(kv.Key)
if !ok {
ckvs.Set(kv.Key, kv.Value)
}
}
if _, ok := cp[subSys]; !ok {
rnSubSys, ok := renamedSubsys[subSys]
if !ok {
// A config subsystem was removed or server was downgraded.
Logger.Info("config: ignoring unknown subsystem config %q\n", subSys)
continue
}
// Copy over settings from previous sub-system
// to newly renamed sub-system
for _, kv := range cp[rnSubSys][Default] {
_, ok := c[subSys][tgt].Lookup(kv.Key)
if !ok {
ckvs.Set(kv.Key, kv.Value)
}
}
subSys = rnSubSys
}
cp[subSys][tgt] = ckvs
}
}
return cp
}
// New - initialize a new server config.
func New() Config {
srvCfg := make(Config)
for _, k := range SubSystems.ToSlice() {
srvCfg[k] = map[string]KVS{}
srvCfg[k][Default] = DefaultKVS[k]
}
return srvCfg
}
// Target signifies an individual target
type Target struct {
SubSystem string
KVS KVS
}
// Targets sub-system targets
type Targets []Target
// GetKVS - get kvs from specific subsystem.
func (c Config) GetKVS(s string, defaultKVS map[string]KVS) (Targets, error) {
if len(s) == 0 {
return nil, Errorf("input cannot be empty")
}
inputs := strings.Fields(s)
if len(inputs) > 1 {
return nil, Errorf("invalid number of arguments %s", s)
}
subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
if len(subSystemValue) == 0 {
return nil, Errorf("invalid number of arguments %s", s)
}
found := SubSystems.Contains(subSystemValue[0])
if !found {
// Check for sub-prefix only if the input value is only a
// single value, this rejects invalid inputs if any.
found = !SubSystems.FuncMatch(strings.HasPrefix, subSystemValue[0]).IsEmpty() && len(subSystemValue) == 1
}
if !found {
return nil, Errorf("unknown sub-system %s", s)
}
targets := Targets{}
subSysPrefix := subSystemValue[0]
if len(subSystemValue) == 2 {
if len(subSystemValue[1]) == 0 {
return nil, Errorf("sub-system target '%s' cannot be empty", s)
}
kvs, ok := c[subSysPrefix][subSystemValue[1]]
if !ok {
return nil, Errorf("sub-system target '%s' doesn't exist", s)
}
for _, kv := range defaultKVS[subSysPrefix] {
_, ok = kvs.Lookup(kv.Key)
if !ok {
kvs.Set(kv.Key, kv.Value)
}
}
targets = append(targets, Target{
SubSystem: inputs[0],
KVS: kvs,
})
} else {
hkvs := HelpSubSysMap[""]
// Use help for sub-system to preserve the order.
for _, hkv := range hkvs {
if !strings.HasPrefix(hkv.Key, subSysPrefix) {
continue
}
if c[hkv.Key][Default].Empty() {
targets = append(targets, Target{
SubSystem: hkv.Key,
KVS: defaultKVS[hkv.Key],
})
}
for k, kvs := range c[hkv.Key] {
for _, dkv := range defaultKVS[hkv.Key] {
_, ok := kvs.Lookup(dkv.Key)
if !ok {
kvs.Set(dkv.Key, dkv.Value)
}
}
if k != Default {
targets = append(targets, Target{
SubSystem: hkv.Key + SubSystemSeparator + k,
KVS: kvs,
})
} else {
targets = append(targets, Target{
SubSystem: hkv.Key,
KVS: kvs,
})
}
}
}
}
return targets, nil
}
// DelKVS - delete a specific key.
func (c Config) DelKVS(s string) error {
if len(s) == 0 {
return Errorf("input arguments cannot be empty")
}
inputs := strings.Fields(s)
if len(inputs) > 1 {
return Errorf("invalid number of arguments %s", s)
}
subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
if len(subSystemValue) == 0 {
return Errorf("invalid number of arguments %s", s)
}
if !SubSystems.Contains(subSystemValue[0]) {
// Unknown sub-system found try to remove it anyways.
delete(c, subSystemValue[0])
return nil
}
tgt := Default
subSys := subSystemValue[0]
if len(subSystemValue) == 2 {
if len(subSystemValue[1]) == 0 {
return Errorf("sub-system target '%s' cannot be empty", s)
}
tgt = subSystemValue[1]
}
_, ok := c[subSys][tgt]
if !ok {
return Errorf("sub-system %s already deleted", s)
}
delete(c[subSys], tgt)
return nil
}
// Clone - clones a config map entirely.
func (c Config) Clone() Config {
cp := New()
for subSys, tgtKV := range c {
cp[subSys] = make(map[string]KVS)
for tgt, kv := range tgtKV {
cp[subSys][tgt] = append(cp[subSys][tgt], kv...)
}
}
return cp
}
// SetKVS - set specific key values per sub-system.
func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err error) {
if len(s) == 0 {
return false, Errorf("input arguments cannot be empty")
}
inputs := strings.SplitN(s, KvSpaceSeparator, 2)
if len(inputs) <= 1 {
return false, Errorf("invalid number of arguments '%s'", s)
}
subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
if len(subSystemValue) == 0 {
return false, Errorf("invalid number of arguments %s", s)
}
if !SubSystems.Contains(subSystemValue[0]) {
return false, Errorf("unknown sub-system %s", s)
}
if SubSystemsSingleTargets.Contains(subSystemValue[0]) && len(subSystemValue) == 2 {
return false, Errorf("sub-system '%s' only supports single target", subSystemValue[0])
}
dynamic = SubSystemsDynamic.Contains(subSystemValue[0])
tgt := Default
subSys := subSystemValue[0]
if len(subSystemValue) == 2 {
tgt = subSystemValue[1]
}
fields := madmin.KvFields(inputs[1], defaultKVS[subSys].Keys())
if len(fields) == 0 {
return false, Errorf("sub-system '%s' cannot have empty keys", subSys)
}
var kvs = KVS{}
var prevK string
for _, v := range fields {
kv := strings.SplitN(v, KvSeparator, 2)
if len(kv) == 0 {
continue
}
if len(kv) == 1 && prevK != "" {
value := strings.Join([]string{
kvs.Get(prevK),
madmin.SanitizeValue(kv[0]),
}, KvSpaceSeparator)
kvs.Set(prevK, value)
continue
}
if len(kv) == 2 {
prevK = kv[0]
kvs.Set(prevK, madmin.SanitizeValue(kv[1]))
continue
}
return false, Errorf("key '%s', cannot have empty value", kv[0])
}
_, ok := kvs.Lookup(Enable)
// Check if state is required
_, enableRequired := defaultKVS[subSys].Lookup(Enable)
if !ok && enableRequired {
// implicit state "on" if not specified.
kvs.Set(Enable, EnableOn)
}
currKVS, ok := c[subSys][tgt]
if !ok {
currKVS = defaultKVS[subSys]
} else {
for _, kv := range defaultKVS[subSys] {
if _, ok = currKVS.Lookup(kv.Key); !ok {
currKVS.Set(kv.Key, kv.Value)
}
}
}
for _, kv := range kvs {
if kv.Key == Comment {
// Skip comment and add it later.
continue
}
currKVS.Set(kv.Key, kv.Value)
}
v, ok := kvs.Lookup(Comment)
if ok {
currKVS.Set(Comment, v)
}
hkvs := HelpSubSysMap[subSys]
for _, hkv := range hkvs {
var enabled bool
if enableRequired {
enabled = currKVS.Get(Enable) == EnableOn
} else {
// when enable arg is not required
// then it is implicit on for the sub-system.
enabled = true
}
v, _ := currKVS.Lookup(hkv.Key)
if v == "" && !hkv.Optional && enabled {
// Return error only if the
// key is enabled, for state=off
// let it be empty.
return false, Errorf(
"'%s' is not optional for '%s' sub-system, please check '%s' documentation",
hkv.Key, subSys, subSys)
}
}
c[subSys][tgt] = currKVS
return dynamic, nil
}

133
internal/config/config_test.go Normal file
View File

@@ -0,0 +1,133 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"testing"
"github.com/minio/madmin-go"
)
func TestKVFields(t *testing.T) {
tests := []struct {
input string
keys []string
expectedFields map[string]struct{}
}{
// No keys present
{
input: "",
keys: []string{"comment"},
expectedFields: map[string]struct{}{},
},
// No keys requested for tokenizing
{
input: `comment="Hi this is my comment ="`,
keys: []string{},
expectedFields: map[string]struct{}{},
},
// Single key requested and present
{
input: `comment="Hi this is my comment ="`,
keys: []string{"comment"},
expectedFields: map[string]struct{}{`comment="Hi this is my comment ="`: {}},
},
// Keys and input order of k=v is same.
{
input: `connection_string="host=localhost port=2832" comment="really long comment"`,
keys: []string{"connection_string", "comment"},
expectedFields: map[string]struct{}{
`connection_string="host=localhost port=2832"`: {},
`comment="really long comment"`: {},
},
},
// Keys with spaces in between
{
input: `enable=on format=namespace connection_string=" host=localhost port=5432 dbname = cesnietor sslmode=disable" table=holicrayoli`,
keys: []string{"enable", "connection_string", "comment", "format", "table"},
expectedFields: map[string]struct{}{
`enable=on`: {},
`format=namespace`: {},
`connection_string=" host=localhost port=5432 dbname = cesnietor sslmode=disable"`: {},
`table=holicrayoli`: {},
},
},
// One of the keys is not present and order of input has changed.
{
input: `comment="really long comment" connection_string="host=localhost port=2832"`,
keys: []string{"connection_string", "comment", "format"},
expectedFields: map[string]struct{}{
`connection_string="host=localhost port=2832"`: {},
`comment="really long comment"`: {},
},
},
// Incorrect delimiter, expected fields should be empty.
{
input: `comment:"really long comment" connection_string:"host=localhost port=2832"`,
keys: []string{"connection_string", "comment"},
expectedFields: map[string]struct{}{},
},
// Incorrect type of input v/s required keys.
{
input: `comme="really long comment" connection_str="host=localhost port=2832"`,
keys: []string{"connection_string", "comment"},
expectedFields: map[string]struct{}{},
},
}
for _, test := range tests {
test := test
t.Run("", func(t *testing.T) {
gotFields := madmin.KvFields(test.input, test.keys)
if len(gotFields) != len(test.expectedFields) {
t.Errorf("Expected keys %d, found %d", len(test.expectedFields), len(gotFields))
}
found := true
for _, field := range gotFields {
_, ok := test.expectedFields[field]
found = found && ok
}
if !found {
t.Errorf("Expected %s, got %s", test.expectedFields, gotFields)
}
})
}
}
func TestValidRegion(t *testing.T) {
tests := []struct {
name string
success bool
}{
{name: "us-east-1", success: true},
{name: "us_east", success: true},
{name: "helloWorld", success: true},
{name: "-fdslka", success: false},
{name: "^00[", success: false},
{name: "my region", success: false},
{name: "%%$#!", success: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ok := validRegionRegex.MatchString(test.name)
if test.success != ok {
t.Errorf("Expected %t, got %t", test.success, ok)
}
})
}
}

55
internal/config/constants.go Normal file
View File

@@ -0,0 +1,55 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
// Config value separator
const (
ValueSeparator = ","
)
// Top level common ENVs
const (
EnvAccessKey = "MINIO_ACCESS_KEY"
EnvSecretKey = "MINIO_SECRET_KEY"
EnvRootUser = "MINIO_ROOT_USER"
EnvRootPassword = "MINIO_ROOT_PASSWORD"
EnvBrowser = "MINIO_BROWSER"
EnvDomain = "MINIO_DOMAIN"
EnvRegionName = "MINIO_REGION_NAME"
EnvPublicIPs = "MINIO_PUBLIC_IPS"
EnvFSOSync = "MINIO_FS_OSYNC"
EnvArgs = "MINIO_ARGS"
EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
EnvRootDiskThresholdSize = "MINIO_ROOTDISK_THRESHOLD_SIZE"
EnvUpdate = "MINIO_UPDATE"
EnvKMSMasterKey = "MINIO_KMS_MASTER_KEY" // legacy
EnvKMSSecretKey = "MINIO_KMS_SECRET_KEY"
EnvKESEndpoint = "MINIO_KMS_KES_ENDPOINT"
EnvKESKeyName = "MINIO_KMS_KES_KEY_NAME"
EnvKESClientKey = "MINIO_KMS_KES_KEY_FILE"
EnvKESClientCert = "MINIO_KMS_KES_CERT_FILE"
EnvKESServerCA = "MINIO_KMS_KES_CAPATH"
EnvEndpoints = "MINIO_ENDPOINTS" // legacy
EnvWorm = "MINIO_WORM" // legacy
EnvRegion = "MINIO_REGION" // legacy
)

171
internal/config/crypto.go Normal file
View File

@@ -0,0 +1,171 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"bytes"
"crypto/rand"
"encoding/binary"
"errors"
"fmt"
"io"
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio/internal/fips"
"github.com/minio/minio/internal/kms"
"github.com/secure-io/sio-go"
"github.com/secure-io/sio-go/sioutil"
)
// EncryptBytes encrypts the plaintext with a key managed by KMS.
// The context is bound to the returned ciphertext.
//
// The same context must be provided when decrypting the
// ciphertext.
func EncryptBytes(KMS kms.KMS, plaintext []byte, context kms.Context) ([]byte, error) {
ciphertext, err := Encrypt(KMS, bytes.NewReader(plaintext), context)
if err != nil {
return nil, err
}
return io.ReadAll(ciphertext)
}
// DecryptBytes decrypts the ciphertext using a key managed by the KMS.
// The same context that have been used during encryption must be
// provided.
func DecryptBytes(KMS kms.KMS, ciphertext []byte, context kms.Context) ([]byte, error) {
plaintext, err := Decrypt(KMS, bytes.NewReader(ciphertext), context)
if err != nil {
return nil, err
}
return io.ReadAll(plaintext)
}
// Encrypt encrypts the plaintext with a key managed by KMS.
// The context is bound to the returned ciphertext.
//
// The same context must be provided when decrypting the
// ciphertext.
func Encrypt(KMS kms.KMS, plaintext io.Reader, context kms.Context) (io.Reader, error) {
var algorithm = sio.AES_256_GCM
if !fips.Enabled && !sioutil.NativeAES() {
algorithm = sio.ChaCha20Poly1305
}
key, err := KMS.GenerateKey("", context)
if err != nil {
return nil, err
}
stream, err := algorithm.Stream(key.Plaintext)
if err != nil {
return nil, err
}
nonce := make([]byte, stream.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return nil, err
}
const (
MaxMetadataSize = 1 << 20 // max. size of the metadata
Version = 1
)
var (
header [5]byte
buffer bytes.Buffer
)
var json = jsoniter.ConfigCompatibleWithStandardLibrary
metadata, err := json.Marshal(encryptedObject{
KeyID: key.KeyID,
KMSKey: key.Ciphertext,
Algorithm: algorithm,
Nonce: nonce,
})
if err != nil {
return nil, err
}
if len(metadata) > MaxMetadataSize {
return nil, errors.New("config: encryption metadata is too large")
}
header[0] = Version
binary.LittleEndian.PutUint32(header[1:], uint32(len(metadata)))
buffer.Write(header[:])
buffer.Write(metadata)
return io.MultiReader(
&buffer,
stream.EncryptReader(plaintext, nonce, nil),
), nil
}
// Decrypt decrypts the ciphertext using a key managed by the KMS.
// The same context that have been used during encryption must be
// provided.
func Decrypt(KMS kms.KMS, ciphertext io.Reader, context kms.Context) (io.Reader, error) {
const (
MaxMetadataSize = 1 << 20 // max. size of the metadata
Version = 1
)
var header [5]byte
if _, err := io.ReadFull(ciphertext, header[:]); err != nil {
return nil, err
}
if header[0] != Version {
return nil, fmt.Errorf("config: unknown ciphertext version %d", header[0])
}
size := binary.LittleEndian.Uint32(header[1:])
if size > MaxMetadataSize {
return nil, errors.New("config: encryption metadata is too large")
}
var (
metadataBuffer = make([]byte, size)
metadata encryptedObject
)
if _, err := io.ReadFull(ciphertext, metadataBuffer); err != nil {
return nil, err
}
var json = jsoniter.ConfigCompatibleWithStandardLibrary
if err := json.Unmarshal(metadataBuffer, &metadata); err != nil {
return nil, err
}
if fips.Enabled && metadata.Algorithm != sio.AES_256_GCM {
return nil, fmt.Errorf("config: unsupported encryption algorithm: %q is not supported in FIPS mode", metadata.Algorithm)
}
key, err := KMS.DecryptKey(metadata.KeyID, metadata.KMSKey, context)
if err != nil {
return nil, err
}
stream, err := metadata.Algorithm.Stream(key)
if err != nil {
return nil, err
}
if stream.NonceSize() != len(metadata.Nonce) {
return nil, sio.NotAuthentic
}
return stream.DecryptReader(ciphertext, metadata.Nonce, nil), nil
}
type encryptedObject struct {
KeyID string `json:"keyid"`
KMSKey []byte `json:"kmskey"`
Algorithm sio.Algorithm `json:"algorithm"`
Nonce []byte `json:"nonce"`
}

119
internal/config/crypto_test.go Normal file
View File

@@ -0,0 +1,119 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"bytes"
"encoding/hex"
"io"
"io/ioutil"
"testing"
"github.com/minio/minio/internal/kms"
)
var encryptDecryptTests = []struct {
Data []byte
Context kms.Context
}{
{
Data: nil,
Context: nil,
},
{
Data: []byte{1},
Context: nil,
},
{
Data: []byte{1},
Context: kms.Context{"key": "value"},
},
{
Data: make([]byte, 1<<20),
Context: kms.Context{"key": "value", "a": "b"},
},
}
func TestEncryptDecrypt(t *testing.T) {
key, err := hex.DecodeString("ddedadb867afa3f73bd33c25499a723ed7f9f51172ee7b1b679e08dc795debcc")
if err != nil {
t.Fatalf("Failed to decode master key: %v", err)
}
KMS, err := kms.New("my-key", key)
if err != nil {
t.Fatalf("Failed to create KMS: %v", err)
}
for i, test := range encryptDecryptTests {
ciphertext, err := Encrypt(KMS, bytes.NewReader(test.Data), test.Context)
if err != nil {
t.Fatalf("Test %d: failed to encrypt stream: %v", i, err)
}
data, err := ioutil.ReadAll(ciphertext)
if err != nil {
t.Fatalf("Test %d: failed to encrypt stream: %v", i, err)
}
plaintext, err := Decrypt(KMS, bytes.NewReader(data), test.Context)
if err != nil {
t.Fatalf("Test %d: failed to decrypt stream: %v", i, err)
}
data, err = ioutil.ReadAll(plaintext)
if err != nil {
t.Fatalf("Test %d: failed to decrypt stream: %v", i, err)
}
if !bytes.Equal(data, test.Data) {
t.Fatalf("Test %d: decrypted data does not match original data", i)
}
}
}
func BenchmarkEncrypt(b *testing.B) {
key, err := hex.DecodeString("ddedadb867afa3f73bd33c25499a723ed7f9f51172ee7b1b679e08dc795debcc")
if err != nil {
b.Fatalf("Failed to decode master key: %v", err)
}
KMS, err := kms.New("my-key", key)
if err != nil {
b.Fatalf("Failed to create KMS: %v", err)
}
benchmarkEncrypt := func(size int, b *testing.B) {
var (
data = make([]byte, size)
plaintext = bytes.NewReader(data)
context = kms.Context{"key": "value"}
)
b.SetBytes(int64(size))
for i := 0; i < b.N; i++ {
ciphertext, err := Encrypt(KMS, plaintext, context)
if err != nil {
b.Fatal(err)
}
if _, err = io.Copy(ioutil.Discard, ciphertext); err != nil {
b.Fatal(err)
}
plaintext.Reset(data)
}
}
b.Run("1KB", func(b *testing.B) { benchmarkEncrypt(1*1024, b) })
b.Run("512KB", func(b *testing.B) { benchmarkEncrypt(512*1024, b) })
b.Run("1MB", func(b *testing.B) { benchmarkEncrypt(1024*1024, b) })
b.Run("10MB", func(b *testing.B) { benchmarkEncrypt(10*1024*1024, b) })
}

302
internal/config/dns/etcd_dns.go Normal file
View File

@@ -0,0 +1,302 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"sort"
"strings"
"time"
"github.com/coredns/coredns/plugin/etcd/msg"
"github.com/minio/minio-go/v7/pkg/set"
clientv3 "go.etcd.io/etcd/client/v3"
)
// ErrNoEntriesFound - Indicates no entries were found for the given key (directory)
var ErrNoEntriesFound = errors.New("No entries found for this key")
// ErrDomainMissing - Indicates domain is missing
var ErrDomainMissing = errors.New("domain is missing")
const etcdPathSeparator = "/"
// create a new coredns service record for the bucket.
func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, error) {
return json.Marshal(&SrvRecord{
Host: ip,
Port: json.Number(port),
TTL: ttl,
CreationDate: t,
})
}
// Close closes the internal etcd client and cannot be used further
func (c *CoreDNS) Close() error {
c.etcdClient.Close()
return nil
}
// List - Retrieves list of DNS entries for the domain.
func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
var srvRecords = map[string][]SrvRecord{}
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
records, err := c.list(key+etcdPathSeparator, true)
if err != nil {
return srvRecords, err
}
for _, record := range records {
if record.Key == "" {
continue
}
srvRecords[record.Key] = append(srvRecords[record.Key], record)
}
}
return srvRecords, nil
}
// Get - Retrieves DNS records for a bucket.
func (c *CoreDNS) Get(bucket string) ([]SrvRecord, error) {
var srvRecords []SrvRecord
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
records, err := c.list(key, false)
if err != nil {
return nil, err
}
// Make sure we have record.Key is empty
// this can only happen when record.Key
// has bucket entry with exact prefix
// match any record.Key which do not
// match the prefixes we skip them.
for _, record := range records {
if record.Key != "" {
continue
}
srvRecords = append(srvRecords, record)
}
}
if len(srvRecords) == 0 {
return nil, ErrNoEntriesFound
}
return srvRecords, nil
}
// msgUnPath converts a etcd path to domainname.
func msgUnPath(s string) string {
ks := strings.Split(strings.Trim(s, etcdPathSeparator), etcdPathSeparator)
for i, j := 0, len(ks)-1; i < j; i, j = i+1, j-1 {
ks[i], ks[j] = ks[j], ks[i]
}
return strings.Join(ks, ".")
}
// Retrieves list of entries under the key passed.
// Note that this method fetches entries upto only two levels deep.
func (c *CoreDNS) list(key string, domain bool) ([]SrvRecord, error) {
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
r, err := c.etcdClient.Get(ctx, key, clientv3.WithPrefix())
defer cancel()
if err != nil {
return nil, err
}
if r.Count == 0 {
key = strings.TrimSuffix(key, etcdPathSeparator)
r, err = c.etcdClient.Get(ctx, key)
if err != nil {
return nil, err
}
// only if we are looking at `domain` as true
// we should return error here.
if domain && r.Count == 0 {
return nil, ErrDomainMissing
}
}
var srvRecords []SrvRecord
for _, n := range r.Kvs {
var srvRecord SrvRecord
if err = json.Unmarshal(n.Value, &srvRecord); err != nil {
return nil, err
}
srvRecord.Key = strings.TrimPrefix(string(n.Key), key)
srvRecord.Key = strings.TrimSuffix(srvRecord.Key, srvRecord.Host)
// Skip non-bucket entry like for a key
// /skydns/net/miniocloud/10.0.0.1 that may exist as
// dns entry for the server (rather than the bucket
// itself).
if srvRecord.Key == "" {
continue
}
srvRecord.Key = msgUnPath(srvRecord.Key)
srvRecords = append(srvRecords, srvRecord)
}
sort.Slice(srvRecords, func(i int, j int) bool {
return srvRecords[i].Key < srvRecords[j].Key
})
return srvRecords, nil
}
// Put - Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
func (c *CoreDNS) Put(bucket string) error {
c.Delete(bucket) // delete any existing entries.
t := time.Now().UTC()
for ip := range c.domainIPs {
bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL, t)
if err != nil {
return err
}
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.%s", bucket, domainName), c.prefixPath)
key = key + etcdPathSeparator + ip
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
_, err = c.etcdClient.Put(ctx, key, string(bucketMsg))
cancel()
if err != nil {
ctx, cancel = context.WithTimeout(context.Background(), defaultContextTimeout)
c.etcdClient.Delete(ctx, key)
cancel()
return err
}
}
}
return nil
}
// Delete - Removes DNS entries added in Put().
func (c *CoreDNS) Delete(bucket string) error {
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
_, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator, clientv3.WithPrefix())
cancel()
if err != nil {
return err
}
}
return nil
}
// DeleteRecord - Removes a specific DNS entry
func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
for _, domainName := range c.domainNames {
key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), c.prefixPath)
ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
_, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator+record.Host)
cancel()
if err != nil {
return err
}
}
return nil
}
// String stringer name for this implementation of dns.Store
func (c *CoreDNS) String() string {
return "etcdDNS"
}
// CoreDNS - represents dns config for coredns server.
type CoreDNS struct {
domainNames []string
domainIPs set.StringSet
domainPort string
prefixPath string
etcdClient *clientv3.Client
}
// EtcdOption - functional options pattern style
type EtcdOption func(*CoreDNS)
// DomainNames set a list of domain names used by this CoreDNS
// client setting, note this will fail if set to empty when
// constructor initializes.
func DomainNames(domainNames []string) EtcdOption {
return func(args *CoreDNS) {
args.domainNames = domainNames
}
}
// DomainIPs set a list of custom domain IPs, note this will
// fail if set to empty when constructor initializes.
func DomainIPs(domainIPs set.StringSet) EtcdOption {
return func(args *CoreDNS) {
args.domainIPs = domainIPs
}
}
// DomainPort - is a string version of server port
func DomainPort(domainPort string) EtcdOption {
return func(args *CoreDNS) {
args.domainPort = domainPort
}
}
// CoreDNSPath - custom prefix on etcd to populate DNS
// service records, optional and can be empty.
// if empty then c.prefixPath is used i.e "/skydns"
func CoreDNSPath(prefix string) EtcdOption {
return func(args *CoreDNS) {
args.prefixPath = prefix
}
}
// NewCoreDNS - initialize a new coreDNS set/unset values.
func NewCoreDNS(cfg clientv3.Config, setters ...EtcdOption) (Store, error) {
etcdClient, err := clientv3.New(cfg)
if err != nil {
return nil, err
}
args := &CoreDNS{
etcdClient: etcdClient,
}
for _, setter := range setters {
setter(args)
}
if len(args.domainNames) == 0 || args.domainIPs.IsEmpty() {
return nil, errors.New("invalid argument")
}
// strip ports off of domainIPs
domainIPsWithoutPorts := args.domainIPs.ApplyFunc(func(ip string) string {
host, _, err := net.SplitHostPort(ip)
if err != nil {
if strings.Contains(err.Error(), "missing port in address") {
host = ip
}
}
return host
})
args.domainIPs = domainIPsWithoutPorts
return args, nil
}

239
internal/config/dns/operator_dns.go Normal file
View File

@@ -0,0 +1,239 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/dgrijalva/jwt-go"
"github.com/minio/minio/internal/config"
xhttp "github.com/minio/minio/internal/http"
)
var (
defaultOperatorContextTimeout = 10 * time.Second
// ErrNotImplemented - Indicates the functionality which is not implemented
ErrNotImplemented = errors.New("The method is not implemented")
)
func (c *OperatorDNS) addAuthHeader(r *http.Request) error {
if c.username == "" || c.password == "" {
return nil
}
claims := &jwt.StandardClaims{
ExpiresAt: int64(15 * time.Minute),
Issuer: c.username,
Subject: config.EnvDNSWebhook,
}
token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
ss, err := token.SignedString([]byte(c.password))
if err != nil {
return err
}
r.Header.Set("Authorization", "Bearer "+ss)
return nil
}
func (c *OperatorDNS) endpoint(bucket string, delete bool) (string, error) {
u, err := url.Parse(c.Endpoint)
if err != nil {
return "", err
}
q := u.Query()
q.Add("bucket", bucket)
q.Add("delete", strconv.FormatBool(delete))
u.RawQuery = q.Encode()
return u.String(), nil
}
// Put - Adds DNS entries into operator webhook server
func (c *OperatorDNS) Put(bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
defer cancel()
e, err := c.endpoint(bucket, false)
if err != nil {
return newError(bucket, err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
if err != nil {
return newError(bucket, err)
}
if err = c.addAuthHeader(req); err != nil {
return newError(bucket, err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
if derr := c.Delete(bucket); derr != nil {
return newError(bucket, derr)
}
}
var errorStringBuilder strings.Builder
io.Copy(&errorStringBuilder, io.LimitReader(resp.Body, resp.ContentLength))
xhttp.DrainBody(resp.Body)
if resp.StatusCode != http.StatusOK {
errorString := errorStringBuilder.String()
switch resp.StatusCode {
case http.StatusConflict:
return ErrBucketConflict(Error{bucket, errors.New(errorString)})
}
return newError(bucket, fmt.Errorf("service create for bucket %s, failed with status %s, error %s", bucket, resp.Status, errorString))
}
return nil
}
func newError(bucket string, err error) error {
e := Error{bucket, err}
if strings.Contains(err.Error(), "invalid bucket name") {
return ErrInvalidBucketName(e)
}
return e
}
// Delete - Removes DNS entries added in Put().
func (c *OperatorDNS) Delete(bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
defer cancel()
e, err := c.endpoint(bucket, true)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
if err != nil {
return err
}
if err = c.addAuthHeader(req); err != nil {
return err
}
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
xhttp.DrainBody(resp.Body)
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("request to delete the service for bucket %s, failed with status %s", bucket, resp.Status)
}
return nil
}
// DeleteRecord - Removes a specific DNS entry
// No Op for Operator because operator deals on with bucket entries
func (c *OperatorDNS) DeleteRecord(record SrvRecord) error {
return ErrNotImplemented
}
// Close closes the internal http client
func (c *OperatorDNS) Close() error {
c.httpClient.CloseIdleConnections()
return nil
}
// List - Retrieves list of DNS entries for the domain.
// This is a No Op for Operator because, there is no intent to enforce global
// namespace at MinIO level with this DNS entry. The global namespace in
// enforced by the Kubernetes Operator
func (c *OperatorDNS) List() (srvRecords map[string][]SrvRecord, err error) {
return nil, ErrNotImplemented
}
// Get - Retrieves DNS records for a bucket.
// This is a No Op for Operator because, there is no intent to enforce global
// namespace at MinIO level with this DNS entry. The global namespace in
// enforced by the Kubernetes Operator
func (c *OperatorDNS) Get(bucket string) (srvRecords []SrvRecord, err error) {
return nil, ErrNotImplemented
}
// String stringer name for this implementation of dns.Store
func (c *OperatorDNS) String() string {
return "webhookDNS"
}
// OperatorDNS - represents dns config for MinIO k8s operator.
type OperatorDNS struct {
httpClient *http.Client
Endpoint string
rootCAs *x509.CertPool
username string
password string
}
// OperatorOption - functional options pattern style for OperatorDNS
type OperatorOption func(*OperatorDNS)
// Authentication - custom username and password for authenticating at the endpoint
func Authentication(username, password string) OperatorOption {
return func(args *OperatorDNS) {
args.username = username
args.password = password
}
}
// RootCAs - add custom trust certs pool
func RootCAs(CAs *x509.CertPool) OperatorOption {
return func(args *OperatorDNS) {
args.rootCAs = CAs
}
}
// NewOperatorDNS - initialize a new K8S Operator DNS set/unset values.
func NewOperatorDNS(endpoint string, setters ...OperatorOption) (Store, error) {
if endpoint == "" {
return nil, errors.New("invalid argument")
}
args := &OperatorDNS{
Endpoint: endpoint,
}
for _, setter := range setters {
setter(args)
}
args.httpClient = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 3 * time.Second,
KeepAlive: 5 * time.Second,
}).DialContext,
ResponseHeaderTimeout: 3 * time.Second,
TLSHandshakeTimeout: 3 * time.Second,
ExpectContinueTimeout: 3 * time.Second,
TLSClientConfig: &tls.Config{
RootCAs: args.rootCAs,
},
// Go net/http automatically unzip if content-type is
// gzip disable this feature, as we are always interested
// in raw stream.
DisableCompression: true,
},
}
return args, nil
}

53
internal/config/dns/store.go Normal file
View File

@@ -0,0 +1,53 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
// Error - DNS related errors error.
type Error struct {
Bucket string
Err error
}
// ErrInvalidBucketName for buckets with invalid name
type ErrInvalidBucketName Error
func (e ErrInvalidBucketName) Error() string {
return e.Bucket + " invalid bucket name error: " + e.Err.Error()
}
func (e Error) Error() string {
return "dns related error: " + e.Err.Error()
}
// ErrBucketConflict for buckets that already exist
type ErrBucketConflict Error
func (e ErrBucketConflict) Error() string {
return e.Bucket + " bucket conflict error: " + e.Err.Error()
}
// Store dns record store
type Store interface {
Put(bucket string) error
Get(bucket string) ([]SrvRecord, error)
Delete(bucket string) error
List() (map[string][]SrvRecord, error)
DeleteRecord(record SrvRecord) error
Close() error
String() string
}

57
internal/config/dns/types.go Normal file
View File

@@ -0,0 +1,57 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dns
import (
"encoding/json"
"time"
)
const (
defaultTTL = 30
defaultContextTimeout = 5 * time.Minute
)
// SrvRecord - represents a DNS service record
type SrvRecord struct {
Host string `json:"host,omitempty"`
Port json.Number `json:"port,omitempty"`
Priority int `json:"priority,omitempty"`
Weight int `json:"weight,omitempty"`
Text string `json:"text,omitempty"`
Mail bool `json:"mail,omitempty"` // Be an MX record. Priority becomes Preference.
TTL uint32 `json:"ttl,omitempty"`
// Holds info about when the entry was created first.
CreationDate time.Time `json:"creationDate"`
// When a SRV record with a "Host: IP-address" is added, we synthesize
// a srv.Target domain name. Normally we convert the full Key where
// the record lives to a DNS name and use this as the srv.Target. When
// TargetStrip > 0 we strip the left most TargetStrip labels from the
// DNS name.
TargetStrip int `json:"targetstrip,omitempty"`
// Group is used to group (or *not* to group) different services
// together. Services with an identical Group are returned in
// the same answer.
Group string `json:"group,omitempty"`
// Key carries the original key used during Put().
Key string `json:"-"`
}

152
internal/config/errors-utils.go Normal file
View File

@@ -0,0 +1,152 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"errors"
"fmt"
"net"
"syscall"
"github.com/minio/minio/internal/color"
)
// Err is a structure which contains all information
// to print a fatal error message in json or pretty mode
// Err implements error so we can use it anywhere
type Err struct {
msg string
detail string
action string
hint string
}
// Clone returns a new Err struct with the same information
func (u Err) Clone() Err {
return Err{
msg: u.msg,
detail: u.detail,
action: u.action,
hint: u.hint,
}
}
// Error returns the error message
func (u Err) Error() string {
if u.detail == "" {
if u.msg != "" {
return u.msg
}
return "<nil>"
}
return u.detail
}
// Msg - Replace the current error's message
func (u Err) Msg(m string, args ...interface{}) Err {
e := u.Clone()
e.msg = fmt.Sprintf(m, args...)
return e
}
// Hint - Replace the current error's message
func (u Err) Hint(m string, args ...interface{}) Err {
e := u.Clone()
e.hint = fmt.Sprintf(m, args...)
return e
}
// ErrFn function wrapper
type ErrFn func(err error) Err
// Create a UI error generator, this is needed to simplify
// the update of the detailed error message in several places
// in MinIO code
func newErrFn(msg, action, hint string) ErrFn {
return func(err error) Err {
u := Err{
msg: msg,
action: action,
hint: hint,
}
if err != nil {
u.detail = err.Error()
}
return u
}
}
// ErrorToErr inspects the passed error and transforms it
// to the appropriate UI error.
func ErrorToErr(err error) Err {
if err == nil {
return Err{}
}
// If this is already a Err, do nothing
if e, ok := err.(Err); ok {
return e
}
// Show a generic message for known golang errors
if errors.Is(err, syscall.EADDRINUSE) {
return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
} else if errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) {
switch err.(type) {
case *net.OpError:
return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
}
}
// Failed to identify what type of error this, return a simple UI error
return Err{msg: err.Error()}
}
// FmtError converts a fatal error message to a more clear error
// using some colors
func FmtError(introMsg string, err error, jsonFlag bool) string {
renderedTxt := ""
uiErr := ErrorToErr(err)
// JSON print
if jsonFlag {
// Message text in json should be simple
if uiErr.detail != "" {
return uiErr.msg + ": " + uiErr.detail
}
return uiErr.msg
}
// Pretty print error message
introMsg += ": "
if uiErr.msg != "" {
introMsg += color.Bold(uiErr.msg)
} else {
introMsg += color.Bold(err.Error())
}
renderedTxt += color.Red(introMsg) + "\n"
// Add action message
if uiErr.action != "" {
renderedTxt += "> " + color.BgYellow(color.Black(uiErr.action)) + "\n"
}
// Add hint
if uiErr.hint != "" {
renderedTxt += color.Bold("HINT:") + "\n"
renderedTxt += " " + uiErr.hint
}
return renderedTxt
}

274
internal/config/errors.go Normal file
View File

@@ -0,0 +1,274 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
// UI errors
var (
ErrInvalidBrowserValue = newErrFn(
"Invalid browser value",
"Please check the passed value",
"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
)
ErrInvalidFSOSyncValue = newErrFn(
"Invalid O_SYNC value",
"Please check the passed value",
"Can only accept `on` and `off` values. To enable O_SYNC for fs backend, set this value to `on`",
)
ErrOverlappingDomainValue = newErrFn(
"Overlapping domain values",
"Please check the passed value",
"MINIO_DOMAIN only accepts non-overlapping domain values",
)
ErrInvalidDomainValue = newErrFn(
"Invalid domain value",
"Please check the passed value",
"Domain can only accept DNS compatible values",
)
ErrInvalidErasureSetSize = newErrFn(
"Invalid erasure set size",
"Please check the passed value",
"Erasure set can only accept any of [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] values",
)
ErrInvalidWormValue = newErrFn(
"Invalid WORM value",
"Please check the passed value",
"WORM can only accept `on` and `off` values. To enable WORM, set this value to `on`",
)
ErrInvalidCacheDrivesValue = newErrFn(
"Invalid cache drive value",
"Please check the value in this ENV variable",
"MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `,`",
)
ErrInvalidCacheExcludesValue = newErrFn(
"Invalid cache excludes value",
"Please check the passed value",
"MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `,`",
)
ErrInvalidCacheExpiryValue = newErrFn(
"Invalid cache expiry value",
"Please check the passed value",
"MINIO_CACHE_EXPIRY: Valid cache expiry duration must be in days",
)
ErrInvalidCacheQuota = newErrFn(
"Invalid cache quota value",
"Please check the passed value",
"MINIO_CACHE_QUOTA: Valid cache quota value must be between 0-100",
)
ErrInvalidCacheAfter = newErrFn(
"Invalid cache after value",
"Please check the passed value",
"MINIO_CACHE_AFTER: Valid cache after value must be 0 or greater",
)
ErrInvalidCacheWatermarkLow = newErrFn(
"Invalid cache low watermark value",
"Please check the passed value",
"MINIO_CACHE_WATERMARK_LOW: Valid cache low watermark value must be between 0-100",
)
ErrInvalidCacheWatermarkHigh = newErrFn(
"Invalid cache high watermark value",
"Please check the passed value",
"MINIO_CACHE_WATERMARK_HIGH: Valid cache high watermark value must be between 0-100",
)
ErrInvalidCacheEncryptionKey = newErrFn(
"Invalid cache encryption master key value",
"Please check the passed value",
"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
)
ErrInvalidCacheRange = newErrFn(
"Invalid cache range value",
"Please check the passed value",
"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
)
ErrInvalidCacheCommitValue = newErrFn(
"Invalid cache commit value",
"Please check the passed value",
"MINIO_CACHE_COMMIT: Valid expected value is `writeback` or `writethrough`",
)
ErrInvalidCacheSetting = newErrFn(
"Incompatible cache setting",
"Please check the passed value",
"MINIO_CACHE_AFTER cannot be used with MINIO_CACHE_COMMIT setting",
)
ErrInvalidCredentialsBackendEncrypted = newErrFn(
"Invalid credentials",
"Please set correct credentials in the environment for decryption",
`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD to be able to decrypt the MinIO config, user IAM and policies`,
)
ErrInvalidCredentials = newErrFn(
"Invalid credentials",
"Please provide correct credentials",
`Access key length should be at least 3, and secret key length at least 8 characters`,
)
ErrEnvCredentialsMissingGateway = newErrFn(
"Credentials missing",
"Please set your credentials in the environment",
`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
)
ErrInvalidErasureEndpoints = newErrFn(
"Invalid endpoint(s) in erasure mode",
"Please provide correct combination of local/remote paths",
"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
)
ErrInvalidNumberOfErasureEndpoints = newErrFn(
"Invalid total number of endpoints for erasure mode",
"Please provide an even number of endpoints greater or equal to 4",
"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
)
ErrStorageClassValue = newErrFn(
"Invalid storage class value",
"Please check the value",
`MINIO_STORAGE_CLASS_STANDARD: Format "EC:<Default_Parity_Standard_Class>" (e.g. "EC:3"). This sets the number of parity disks for MinIO server in Standard mode. Objects are stored in Standard mode, if storage class is not defined in Put request
MINIO_STORAGE_CLASS_RRS: Format "EC:<Default_Parity_Reduced_Redundancy_Class>" (e.g. "EC:3"). This sets the number of parity disks for MinIO server in Reduced Redundancy mode. Objects are stored in Reduced Redundancy mode, if Put request specifies RRS storage class
Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storage-class for more information`,
)
ErrUnexpectedBackendVersion = newErrFn(
"Backend version seems to be too recent",
"Please update to the latest MinIO version",
"",
)
ErrInvalidAddressFlag = newErrFn(
"--address input is invalid",
"Please check --address parameter",
`--address binds to a specific ADDRESS:PORT, ADDRESS can be an IPv4/IPv6 address or hostname (default port is ':9000')
Examples: --address ':443'
--address '172.16.34.31:9000'
--address '[fe80::da00:a6c8:e3ae:ddd7]:9000'`,
)
ErrInvalidFSEndpoint = newErrFn(
"Invalid endpoint for standalone FS mode",
"Please check the FS endpoint",
`FS mode requires only one writable disk path
Example 1:
$ minio server /data/minio/`,
)
ErrUnsupportedBackend = newErrFn(
"Unable to write to the backend",
"Please ensure your disk supports O_DIRECT",
"",
)
ErrUnableToWriteInBackend = newErrFn(
"Unable to write to the backend",
"Please ensure MinIO binary has write permissions for the backend",
`Verify if MinIO binary is running as the same user who has write permissions for the backend`,
)
ErrPortAlreadyInUse = newErrFn(
"Port is already in use",
"Please ensure no other program uses the same address/port",
"",
)
ErrPortAccess = newErrFn(
"Unable to use specified port",
"Please ensure MinIO binary has 'cap_net_bind_service=+ep' permissions",
`Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`,
)
ErrSSLUnexpectedError = newErrFn(
"Invalid TLS certificate",
"Please check the content of your certificate data",
`Only PEM (x.509) format is accepted as valid public & private certificates`,
)
ErrSSLUnexpectedData = newErrFn(
"Invalid TLS certificate",
"Please check your certificate",
"",
)
ErrSSLNoPassword = newErrFn(
"Missing TLS password",
"Please set the password to environment variable `MINIO_CERT_PASSWD` so that the private key can be decrypted",
"",
)
ErrNoCertsAndHTTPSEndpoints = newErrFn(
"HTTPS specified in endpoints, but no TLS certificate is found on the local machine",
"Please add TLS certificate or use HTTP endpoints only",
"Refer to https://docs.min.io/docs/how-to-secure-access-to-minio-server-with-tls for information about how to load a TLS certificate in your server",
)
ErrCertsAndHTTPEndpoints = newErrFn(
"HTTP specified in endpoints, but the server in the local machine is configured with a TLS certificate",
"Please remove the certificate in the configuration directory or switch to HTTPS",
"",
)
ErrSSLWrongPassword = newErrFn(
"Unable to decrypt the private key using the provided password",
"Please set the correct password in environment variable `MINIO_CERT_PASSWD`",
"",
)
ErrUnexpectedError = newErrFn(
"Unexpected error",
"Please contact MinIO at https://slack.min.io",
"",
)
ErrInvalidCompressionIncludesValue = newErrFn(
"Invalid compression include value",
"Please check the passed value",
"Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_MIME_TYPES=\"A,B,C\"",
)
ErrInvalidGWSSEValue = newErrFn(
"Invalid gateway SSE value",
"Please check the passed value",
"MINIO_GATEWAY_SSE: Gateway SSE accepts only C and S3 as valid values. Delimit by `;` to set more than one value",
)
ErrInvalidGWSSEEnvValue = newErrFn(
"Invalid gateway SSE configuration",
"",
"Refer to https://docs.min.io/docs/minio-kms-quickstart-guide.html for setting up SSE",
)
ErrInvalidReplicationWorkersValue = newErrFn(
"Invalid value for replication workers",
"",
"MINIO_API_REPLICATION_WORKERS: should be > 0",
)
)

176
internal/config/etcd/etcd.go Normal file
View File

@@ -0,0 +1,176 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package etcd
import (
"crypto/tls"
"crypto/x509"
"strings"
"time"
"github.com/minio/minio/internal/config"
xnet "github.com/minio/minio/internal/net"
"github.com/minio/pkg/env"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/namespace"
"go.uber.org/zap"
)
const (
// Default values used while communicating with etcd.
defaultDialTimeout = 5 * time.Second
defaultDialKeepAlive = 30 * time.Second
)
// etcd environment values
const (
Endpoints = "endpoints"
PathPrefix = "path_prefix"
CoreDNSPath = "coredns_path"
ClientCert = "client_cert"
ClientCertKey = "client_cert_key"
EnvEtcdEndpoints = "MINIO_ETCD_ENDPOINTS"
EnvEtcdPathPrefix = "MINIO_ETCD_PATH_PREFIX"
EnvEtcdCoreDNSPath = "MINIO_ETCD_COREDNS_PATH"
EnvEtcdClientCert = "MINIO_ETCD_CLIENT_CERT"
EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
)
// DefaultKVS - default KV settings for etcd.
var (
DefaultKVS = config.KVS{
config.KV{
Key: Endpoints,
Value: "",
},
config.KV{
Key: PathPrefix,
Value: "",
},
config.KV{
Key: CoreDNSPath,
Value: "/skydns",
},
config.KV{
Key: ClientCert,
Value: "",
},
config.KV{
Key: ClientCertKey,
Value: "",
},
}
)
// Config - server etcd config.
type Config struct {
Enabled bool `json:"enabled"`
PathPrefix string `json:"pathPrefix"`
CoreDNSPath string `json:"coreDNSPath"`
clientv3.Config
}
// New - initialize new etcd client.
func New(cfg Config) (*clientv3.Client, error) {
if !cfg.Enabled {
return nil, nil
}
cli, err := clientv3.New(cfg.Config)
if err != nil {
return nil, err
}
cli.KV = namespace.NewKV(cli.KV, cfg.PathPrefix)
cli.Watcher = namespace.NewWatcher(cli.Watcher, cfg.PathPrefix)
cli.Lease = namespace.NewLease(cli.Lease, cfg.PathPrefix)
return cli, nil
}
func parseEndpoints(endpoints string) ([]string, bool, error) {
etcdEndpoints := strings.Split(endpoints, config.ValueSeparator)
var etcdSecure bool
for _, endpoint := range etcdEndpoints {
u, err := xnet.ParseHTTPURL(endpoint)
if err != nil {
return nil, false, err
}
if etcdSecure && u.Scheme == "http" {
return nil, false, config.Errorf("all endpoints should be https or http: %s", endpoint)
}
// If one of the endpoint is https, we will use https directly.
etcdSecure = etcdSecure || u.Scheme == "https"
}
return etcdEndpoints, etcdSecure, nil
}
// Enabled returns if etcd is enabled.
func Enabled(kvs config.KVS) bool {
endpoints := kvs.Get(Endpoints)
return endpoints != ""
}
// LookupConfig - Initialize new etcd config.
func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
cfg := Config{}
if err := config.CheckValidKeys(config.EtcdSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
endpoints := env.Get(EnvEtcdEndpoints, kvs.Get(Endpoints))
if endpoints == "" {
return cfg, nil
}
etcdEndpoints, etcdSecure, err := parseEndpoints(endpoints)
if err != nil {
return cfg, err
}
cfg.Enabled = true
cfg.DialTimeout = defaultDialTimeout
cfg.DialKeepAliveTime = defaultDialKeepAlive
// Disable etcd client SDK logging, etcd client
// incorrectly starts logging in unexpected data
// format.
cfg.LogConfig = &zap.Config{
Level: zap.NewAtomicLevelAt(zap.FatalLevel),
Encoding: "console",
}
cfg.Endpoints = etcdEndpoints
cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath))
// Default path prefix for all keys on etcd, other than CoreDNSPath.
cfg.PathPrefix = env.Get(EnvEtcdPathPrefix, kvs.Get(PathPrefix))
if etcdSecure {
cfg.TLS = &tls.Config{
RootCAs: rootCAs,
}
// This is only to support client side certificate authentication
// https://coreos.com/etcd/docs/latest/op-guide/security.html
etcdClientCertFile := env.Get(EnvEtcdClientCert, kvs.Get(ClientCert))
etcdClientCertKey := env.Get(EnvEtcdClientCertKey, kvs.Get(ClientCertKey))
if etcdClientCertFile != "" && etcdClientCertKey != "" {
cfg.TLS.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert, err := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
return &cert, err
}
}
}
return cfg, nil
}

67
internal/config/etcd/etcd_test.go Normal file
View File

@@ -0,0 +1,67 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package etcd
import (
"reflect"
"testing"
)
// TestParseEndpoints - tests parseEndpoints function with valid and invalid inputs.
func TestParseEndpoints(t *testing.T) {
testCases := []struct {
s string
endpoints []string
secure bool
success bool
}{
// Invalid inputs
{"https://localhost:2379,http://localhost:2380", nil, false, false},
{",,,", nil, false, false},
{"", nil, false, false},
{"ftp://localhost:2379", nil, false, false},
{"http://localhost:2379000", nil, false, false},
// Valid inputs
{"https://localhost:2379,https://localhost:2380", []string{
"https://localhost:2379", "https://localhost:2380"},
true, true},
{"http://localhost:2379", []string{"http://localhost:2379"}, false, true},
}
for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.s, func(t *testing.T) {
endpoints, secure, err := parseEndpoints(testCase.s)
if err != nil && testCase.success {
t.Errorf("expected to succeed but failed with %s", err)
}
if !testCase.success && err == nil {
t.Error("expected failure but succeeded instead")
}
if testCase.success {
if !reflect.DeepEqual(endpoints, testCase.endpoints) {
t.Errorf("expected %s, got %s", testCase.endpoints, endpoints)
}
if secure != testCase.secure {
t.Errorf("expected %t, got %t", testCase.secure, secure)
}
}
})
}
}

61
internal/config/etcd/help.go Normal file
View File

@@ -0,0 +1,61 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package etcd
import "github.com/minio/minio/internal/config"
// etcd config documented in default config
var (
Help = config.HelpKVS{
config.HelpKV{
Key: Endpoints,
Description: `comma separated list of etcd endpoints e.g. "http://localhost:2379"`,
Type: "csv",
},
config.HelpKV{
Key: PathPrefix,
Description: `namespace prefix to isolate tenants e.g. "customer1/"`,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: CoreDNSPath,
Description: `shared bucket DNS records, default is "/skydns"`,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: ClientCert,
Description: `client cert for mTLS authentication`,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: ClientCertKey,
Description: `client cert key for mTLS authentication`,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

107
internal/config/heal/heal.go Normal file
View File

@@ -0,0 +1,107 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package heal
import (
"fmt"
"strconv"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/env"
)
// Compression environment variables
const (
Bitrot = "bitrotscan"
Sleep = "max_sleep"
IOCount = "max_io"
EnvBitrot = "MINIO_HEAL_BITROTSCAN"
EnvSleep = "MINIO_HEAL_MAX_SLEEP"
EnvIOCount = "MINIO_HEAL_MAX_IO"
)
// Config represents the heal settings.
type Config struct {
// Bitrot will perform bitrot scan on local disk when checking objects.
Bitrot bool `json:"bitrotscan"`
// maximum sleep duration between objects to slow down heal operation.
Sleep time.Duration `json:"sleep"`
IOCount int `json:"iocount"`
}
var (
// DefaultKVS - default KV config for heal settings
DefaultKVS = config.KVS{
config.KV{
Key: Bitrot,
Value: config.EnableOff,
},
config.KV{
Key: Sleep,
Value: "1s",
},
config.KV{
Key: IOCount,
Value: "10",
},
}
// Help provides help for config values
Help = config.HelpKVS{
config.HelpKV{
Key: Bitrot,
Description: `perform bitrot scan on disks when checking objects during scanner`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: Sleep,
Description: `maximum sleep duration between objects to slow down heal operation. eg. 2s`,
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: IOCount,
Description: `maximum IO requests allowed between objects to slow down heal operation. eg. 3`,
Optional: true,
Type: "int",
},
}
)
// LookupConfig - lookup config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
if err = config.CheckValidKeys(config.HealSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
cfg.Bitrot, err = config.ParseBool(env.Get(EnvBitrot, kvs.Get(Bitrot)))
if err != nil {
return cfg, fmt.Errorf("'heal:bitrotscan' value invalid: %w", err)
}
cfg.Sleep, err = time.ParseDuration(env.Get(EnvSleep, kvs.Get(Sleep)))
if err != nil {
return cfg, fmt.Errorf("'heal:max_sleep' value invalid: %w", err)
}
cfg.IOCount, err = strconv.Atoi(env.Get(EnvIOCount, kvs.Get(IOCount)))
if err != nil {
return cfg, fmt.Errorf("'heal:max_io' value invalid: %w", err)
}
return cfg, nil
}

64
internal/config/help.go Normal file
View File

@@ -0,0 +1,64 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
// HelpKV - implements help messages for keys
// with value as description of the keys.
type HelpKV struct {
Key string `json:"key"`
Type string `json:"type"`
Description string `json:"description"`
Optional bool `json:"optional"`
// Indicates if sub-sys supports multiple targets.
MultipleTargets bool `json:"multipleTargets"`
}
// HelpKVS - implement order of keys help messages.
type HelpKVS []HelpKV
// Lookup - lookup a key from help kvs.
func (hkvs HelpKVS) Lookup(key string) (HelpKV, bool) {
for _, hkv := range hkvs {
if hkv.Key == key {
return hkv, true
}
}
return HelpKV{}, false
}
// DefaultComment used across all sub-systems.
const DefaultComment = "optionally add a comment to this setting"
// Region and Worm help is documented in default config
var (
RegionHelp = HelpKVS{
HelpKV{
Key: RegionName,
Type: "string",
Description: `name of the location of the server e.g. "us-west-rack2"`,
Optional: true,
},
HelpKV{
Key: Comment,
Type: "sentence",
Description: DefaultComment,
Optional: true,
},
}
)

570
internal/config/identity/ldap/config.go Normal file
View File

@@ -0,0 +1,570 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ldap
import (
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"math/rand"
"net"
"strings"
"time"
ldap "github.com/go-ldap/ldap/v3"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/env"
)
const (
defaultLDAPExpiry = time.Hour * 1
dnDelimiter = ";"
)
// Config contains AD/LDAP server connectivity information.
type Config struct {
Enabled bool `json:"enabled"`
// E.g. "ldap.minio.io:636"
ServerAddr string `json:"serverAddr"`
// STS credentials expiry duration
STSExpiryDuration string `json:"stsExpiryDuration"`
// Format string for usernames
UsernameFormat string `json:"usernameFormat"`
UsernameFormats []string `json:"-"`
// User DN search parameters
UserDNSearchBaseDN string `json:"userDNSearchBaseDN"`
UserDNSearchFilter string `json:"userDNSearchFilter"`
// Group search parameters
GroupSearchBaseDistName string `json:"groupSearchBaseDN"`
GroupSearchBaseDistNames []string `json:"-"`
GroupSearchFilter string `json:"groupSearchFilter"`
// Lookup bind LDAP service account
LookupBindDN string `json:"lookupBindDN"`
LookupBindPassword string `json:"lookupBindPassword"`
stsExpiryDuration time.Duration // contains converted value
tlsSkipVerify bool // allows skipping TLS verification
serverInsecure bool // allows plain text connection to LDAP server
serverStartTLS bool // allows using StartTLS connection to LDAP server
isUsingLookupBind bool
rootCAs *x509.CertPool
}
// LDAP keys and envs.
const (
ServerAddr = "server_addr"
STSExpiry = "sts_expiry"
LookupBindDN = "lookup_bind_dn"
LookupBindPassword = "lookup_bind_password"
UserDNSearchBaseDN = "user_dn_search_base_dn"
UserDNSearchFilter = "user_dn_search_filter"
UsernameFormat = "username_format"
GroupSearchFilter = "group_search_filter"
GroupSearchBaseDN = "group_search_base_dn"
TLSSkipVerify = "tls_skip_verify"
ServerInsecure = "server_insecure"
ServerStartTLS = "server_starttls"
EnvServerAddr = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
EnvSTSExpiry = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
EnvTLSSkipVerify = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
EnvServerInsecure = "MINIO_IDENTITY_LDAP_SERVER_INSECURE"
EnvServerStartTLS = "MINIO_IDENTITY_LDAP_SERVER_STARTTLS"
EnvUsernameFormat = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
EnvUserDNSearchBaseDN = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN"
EnvUserDNSearchFilter = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER"
EnvGroupSearchFilter = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
EnvGroupSearchBaseDN = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
EnvLookupBindDN = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN"
EnvLookupBindPassword = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD"
)
var removedKeys = []string{
"username_search_filter",
"username_search_base_dn",
"group_name_attribute",
}
// DefaultKVS - default config for LDAP config
var (
DefaultKVS = config.KVS{
config.KV{
Key: ServerAddr,
Value: "",
},
config.KV{
Key: UsernameFormat,
Value: "",
},
config.KV{
Key: UserDNSearchBaseDN,
Value: "",
},
config.KV{
Key: UserDNSearchFilter,
Value: "",
},
config.KV{
Key: GroupSearchFilter,
Value: "",
},
config.KV{
Key: GroupSearchBaseDN,
Value: "",
},
config.KV{
Key: STSExpiry,
Value: "1h",
},
config.KV{
Key: TLSSkipVerify,
Value: config.EnableOff,
},
config.KV{
Key: ServerInsecure,
Value: config.EnableOff,
},
config.KV{
Key: ServerStartTLS,
Value: config.EnableOff,
},
config.KV{
Key: LookupBindDN,
Value: "",
},
config.KV{
Key: LookupBindPassword,
Value: "",
},
}
)
func getGroups(conn *ldap.Conn, sreq *ldap.SearchRequest) ([]string, error) {
var groups []string
sres, err := conn.Search(sreq)
if err != nil {
// Check if there is no matching result and return empty slice.
// Ref: https://ldap.com/ldap-result-code-reference/
if ldap.IsErrorWithCode(err, 32) {
return nil, nil
}
return nil, err
}
for _, entry := range sres.Entries {
// We only queried one attribute,
// so we only look up the first one.
groups = append(groups, entry.DN)
}
return groups, nil
}
func (l *Config) lookupBind(conn *ldap.Conn) error {
var err error
if l.LookupBindPassword == "" {
err = conn.UnauthenticatedBind(l.LookupBindDN)
} else {
err = conn.Bind(l.LookupBindDN, l.LookupBindPassword)
}
if ldap.IsErrorWithCode(err, 49) {
return fmt.Errorf("LDAP Lookup Bind user invalid credentials error: %w", err)
}
return err
}
// usernameFormatsBind - Iterates over all given username formats and expects
// that only one will succeed if the credentials are valid. The succeeding
// bindDN is returned or an error.
//
// In the rare case that multiple username formats succeed, implying that two
// (or more) distinct users in the LDAP directory have the same username and
// password, we return an error as we cannot identify the account intended by
// the user.
func (l *Config) usernameFormatsBind(conn *ldap.Conn, username, password string) (string, error) {
var bindDistNames []string
var errs = make([]error, len(l.UsernameFormats))
var successCount = 0
for i, usernameFormat := range l.UsernameFormats {
bindDN := fmt.Sprintf(usernameFormat, username)
// Bind with user credentials to validate the password
errs[i] = conn.Bind(bindDN, password)
if errs[i] == nil {
bindDistNames = append(bindDistNames, bindDN)
successCount++
} else if !ldap.IsErrorWithCode(errs[i], 49) {
return "", fmt.Errorf("LDAP Bind request failed with unexpected error: %w", errs[i])
}
}
if successCount == 0 {
var errStrings []string
for _, err := range errs {
if err != nil {
errStrings = append(errStrings, err.Error())
}
}
outErr := fmt.Sprintf("All username formats failed due to invalid credentials: %s", strings.Join(errStrings, "; "))
return "", errors.New(outErr)
}
if successCount > 1 {
successDistNames := strings.Join(bindDistNames, ", ")
errMsg := fmt.Sprintf("Multiple username formats succeeded - ambiguous user login (succeeded for: %s)", successDistNames)
return "", errors.New(errMsg)
}
return bindDistNames[0], nil
}
// lookupUserDN searches for the DN of the user given their username. conn is
// assumed to be using the lookup bind service account. It is required that the
// search result in at most one result.
func (l *Config) lookupUserDN(conn *ldap.Conn, username string) (string, error) {
filter := strings.Replace(l.UserDNSearchFilter, "%s", ldap.EscapeFilter(username), -1)
searchRequest := ldap.NewSearchRequest(
l.UserDNSearchBaseDN,
ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
filter,
[]string{}, // only need DN, so no pass no attributes here
nil,
)
searchResult, err := conn.Search(searchRequest)
if err != nil {
return "", err
}
if len(searchResult.Entries) == 0 {
return "", fmt.Errorf("User DN for %s not found", username)
}
if len(searchResult.Entries) != 1 {
return "", fmt.Errorf("Multiple DNs for %s found - please fix the search filter", username)
}
return searchResult.Entries[0].DN, nil
}
func (l *Config) searchForUserGroups(conn *ldap.Conn, username, bindDN string) ([]string, error) {
// User groups lookup.
var groups []string
if l.GroupSearchFilter != "" {
for _, groupSearchBase := range l.GroupSearchBaseDistNames {
filter := strings.Replace(l.GroupSearchFilter, "%s", ldap.EscapeFilter(username), -1)
filter = strings.Replace(filter, "%d", ldap.EscapeFilter(bindDN), -1)
searchRequest := ldap.NewSearchRequest(
groupSearchBase,
ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
filter,
nil,
nil,
)
var newGroups []string
newGroups, err := getGroups(conn, searchRequest)
if err != nil {
errRet := fmt.Errorf("Error finding groups of %s: %w", bindDN, err)
return nil, errRet
}
groups = append(groups, newGroups...)
}
}
return groups, nil
}
// LookupUserDN searches for the full DN ang groups of a given username
func (l *Config) LookupUserDN(username string) (string, []string, error) {
if !l.isUsingLookupBind {
return "", nil, errors.New("current lookup mode does not support searching for User DN")
}
conn, err := l.Connect()
if err != nil {
return "", nil, err
}
defer conn.Close()
// Bind to the lookup user account
if err = l.lookupBind(conn); err != nil {
return "", nil, err
}
// Lookup user DN
bindDN, err := l.lookupUserDN(conn, username)
if err != nil {
errRet := fmt.Errorf("Unable to find user DN: %w", err)
return "", nil, errRet
}
groups, err := l.searchForUserGroups(conn, username, bindDN)
if err != nil {
return "", nil, err
}
return bindDN, groups, nil
}
// Bind - binds to ldap, searches LDAP and returns the distinguished name of the
// user and the list of groups.
func (l *Config) Bind(username, password string) (string, []string, error) {
conn, err := l.Connect()
if err != nil {
return "", nil, err
}
defer conn.Close()
var bindDN string
if l.isUsingLookupBind {
// Bind to the lookup user account
if err = l.lookupBind(conn); err != nil {
return "", nil, err
}
// Lookup user DN
bindDN, err = l.lookupUserDN(conn, username)
if err != nil {
errRet := fmt.Errorf("Unable to find user DN: %w", err)
return "", nil, errRet
}
// Authenticate the user credentials.
err = conn.Bind(bindDN, password)
if err != nil {
errRet := fmt.Errorf("LDAP auth failed for DN %s: %w", bindDN, err)
return "", nil, errRet
}
// Bind to the lookup user account again to perform group search.
if err = l.lookupBind(conn); err != nil {
return "", nil, err
}
} else {
// Verify login credentials by checking the username formats.
bindDN, err = l.usernameFormatsBind(conn, username, password)
if err != nil {
return "", nil, err
}
// Bind to the successful bindDN again.
err = conn.Bind(bindDN, password)
if err != nil {
errRet := fmt.Errorf("LDAP conn failed though auth for DN %s succeeded: %w", bindDN, err)
return "", nil, errRet
}
}
// User groups lookup.
groups, err := l.searchForUserGroups(conn, username, bindDN)
if err != nil {
return "", nil, err
}
return bindDN, groups, nil
}
// Connect connect to ldap server.
func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
if l == nil {
return nil, errors.New("LDAP is not configured")
}
serverHost, _, err := net.SplitHostPort(l.ServerAddr)
if err != nil {
serverHost = l.ServerAddr
// User default LDAP port if none specified "636"
l.ServerAddr = net.JoinHostPort(l.ServerAddr, "636")
}
if l.serverInsecure {
return ldap.Dial("tcp", l.ServerAddr)
}
tlsConfig := &tls.Config{
InsecureSkipVerify: l.tlsSkipVerify,
RootCAs: l.rootCAs,
ServerName: serverHost,
}
if l.serverStartTLS {
conn, err := ldap.Dial("tcp", l.ServerAddr)
if err != nil {
return nil, err
}
err = conn.StartTLS(tlsConfig)
return conn, err
}
return ldap.DialTLS("tcp", l.ServerAddr, tlsConfig)
}
// GetExpiryDuration - return parsed expiry duration.
func (l Config) GetExpiryDuration() time.Duration {
return l.stsExpiryDuration
}
func (l Config) testConnection() error {
conn, err := l.Connect()
if err != nil {
return fmt.Errorf("Error creating connection to LDAP server: %w", err)
}
defer conn.Close()
if l.isUsingLookupBind {
if err = l.lookupBind(conn); err != nil {
return fmt.Errorf("Error connecting as LDAP Lookup Bind user: %w", err)
}
return nil
}
// Generate some random user credentials for username formats mode test.
username := fmt.Sprintf("sometestuser%09d", rand.Int31n(1000000000))
charset := []byte("abcdefghijklmnopqrstuvwxyz" +
"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
rand.Shuffle(len(charset), func(i, j int) {
charset[i], charset[j] = charset[j], charset[i]
})
password := string(charset[:20])
_, err = l.usernameFormatsBind(conn, username, password)
if err == nil {
// We don't expect to successfully guess a credential in this
// way.
return fmt.Errorf("Unexpected random credentials success for user=%s password=%s", username, password)
} else if strings.HasPrefix(err.Error(), "All username formats failed due to invalid credentials: ") {
return nil
}
return fmt.Errorf("LDAP connection test error: %w", err)
}
// Enabled returns if jwks is enabled.
func Enabled(kvs config.KVS) bool {
return kvs.Get(ServerAddr) != ""
}
// Lookup - initializes LDAP config, overrides config, if any ENV values are set.
func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
l = Config{}
// Purge all removed keys first
for _, k := range removedKeys {
kvs.Delete(k)
}
if err = config.CheckValidKeys(config.IdentityLDAPSubSys, kvs, DefaultKVS); err != nil {
return l, err
}
ldapServer := env.Get(EnvServerAddr, kvs.Get(ServerAddr))
if ldapServer == "" {
return l, nil
}
l.Enabled = true
l.ServerAddr = ldapServer
l.stsExpiryDuration = defaultLDAPExpiry
if v := env.Get(EnvSTSExpiry, kvs.Get(STSExpiry)); v != "" {
expDur, err := time.ParseDuration(v)
if err != nil {
return l, errors.New("LDAP expiry time err:" + err.Error())
}
if expDur <= 0 {
return l, errors.New("LDAP expiry time has to be positive")
}
l.STSExpiryDuration = v
l.stsExpiryDuration = expDur
}
// LDAP connection configuration
if v := env.Get(EnvServerInsecure, kvs.Get(ServerInsecure)); v != "" {
l.serverInsecure, err = config.ParseBool(v)
if err != nil {
return l, err
}
}
if v := env.Get(EnvServerStartTLS, kvs.Get(ServerStartTLS)); v != "" {
l.serverStartTLS, err = config.ParseBool(v)
if err != nil {
return l, err
}
}
if v := env.Get(EnvTLSSkipVerify, kvs.Get(TLSSkipVerify)); v != "" {
l.tlsSkipVerify, err = config.ParseBool(v)
if err != nil {
return l, err
}
}
// Lookup bind user configuration
lookupBindDN := env.Get(EnvLookupBindDN, kvs.Get(LookupBindDN))
lookupBindPassword := env.Get(EnvLookupBindPassword, kvs.Get(LookupBindPassword))
if lookupBindDN != "" {
l.LookupBindDN = lookupBindDN
l.LookupBindPassword = lookupBindPassword
l.isUsingLookupBind = true
// User DN search configuration
userDNSearchBaseDN := env.Get(EnvUserDNSearchBaseDN, kvs.Get(UserDNSearchBaseDN))
userDNSearchFilter := env.Get(EnvUserDNSearchFilter, kvs.Get(UserDNSearchFilter))
if userDNSearchFilter == "" || userDNSearchBaseDN == "" {
return l, errors.New("In lookup bind mode, userDN search base DN and userDN search filter are both required")
}
l.UserDNSearchBaseDN = userDNSearchBaseDN
l.UserDNSearchFilter = userDNSearchFilter
}
// Username format configuration.
if v := env.Get(EnvUsernameFormat, kvs.Get(UsernameFormat)); v != "" {
if !strings.Contains(v, "%s") {
return l, errors.New("LDAP username format doesn't have '%s' substitution")
}
l.UsernameFormats = strings.Split(v, dnDelimiter)
}
// Either lookup bind mode or username format is supported, but not
// both.
if l.isUsingLookupBind && len(l.UsernameFormats) > 0 {
return l, errors.New("Lookup Bind mode and Username Format mode are not supported at the same time")
}
// At least one of bind mode or username format must be used.
if !l.isUsingLookupBind && len(l.UsernameFormats) == 0 {
return l, errors.New("Either Lookup Bind mode or Username Format mode is required.")
}
// Test connection to LDAP server.
if err := l.testConnection(); err != nil {
return l, fmt.Errorf("Connection test for LDAP server failed: %w", err)
}
// Group search params configuration
grpSearchFilter := env.Get(EnvGroupSearchFilter, kvs.Get(GroupSearchFilter))
grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, kvs.Get(GroupSearchBaseDN))
// Either all group params must be set or none must be set.
if (grpSearchFilter != "" && grpSearchBaseDN == "") || (grpSearchFilter == "" && grpSearchBaseDN != "") {
return l, errors.New("All group related parameters must be set")
}
if grpSearchFilter != "" {
l.GroupSearchFilter = grpSearchFilter
l.GroupSearchBaseDistName = grpSearchBaseDN
l.GroupSearchBaseDistNames = strings.Split(l.GroupSearchBaseDistName, dnDelimiter)
}
l.rootCAs = rootCAs
return l, nil
}

103
internal/config/identity/ldap/help.go Normal file
View File

@@ -0,0 +1,103 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ldap
import "github.com/minio/minio/internal/config"
// Help template for LDAP identity feature.
var (
Help = config.HelpKVS{
config.HelpKV{
Key: ServerAddr,
Description: `AD/LDAP server address e.g. "myldapserver.com:636"`,
Type: "address",
},
config.HelpKV{
Key: STSExpiry,
Description: `temporary credentials validity duration in s,m,h,d. Default is "1h"`,
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: LookupBindDN,
Description: `DN for LDAP read-only service account used to perform DN and group lookups`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: LookupBindPassword,
Description: `Password for LDAP read-only service account used to perform DN and group lookups`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: UserDNSearchBaseDN,
Description: `Base LDAP DN to search for user DN`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: UserDNSearchFilter,
Description: `Search filter to lookup user DN`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: UsernameFormat,
Description: `";" separated list of username bind DNs e.g. "uid=%s,cn=accounts,dc=myldapserver,dc=com"`,
Optional: true,
Type: "list",
},
config.HelpKV{
Key: GroupSearchFilter,
Description: `search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: GroupSearchBaseDN,
Description: `";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"`,
Optional: true,
Type: "list",
},
config.HelpKV{
Key: TLSSkipVerify,
Description: `trust server TLS without verification, defaults to "off" (verify)`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: ServerInsecure,
Description: `allow plain text connection to AD/LDAP server, defaults to "off"`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: ServerStartTLS,
Description: `use StartTLS connection to AD/LDAP server, defaults to "off"`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

50
internal/config/identity/ldap/legacy.go Normal file
View File

@@ -0,0 +1,50 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ldap
import "github.com/minio/minio/internal/config"
// SetIdentityLDAP - One time migration code needed, for migrating from older config to new for LDAPConfig.
func SetIdentityLDAP(s config.Config, ldapArgs Config) {
if !ldapArgs.Enabled {
// ldap not enabled no need to preserve it in new settings.
return
}
s[config.IdentityLDAPSubSys][config.Default] = config.KVS{
config.KV{
Key: ServerAddr,
Value: ldapArgs.ServerAddr,
},
config.KV{
Key: STSExpiry,
Value: ldapArgs.STSExpiryDuration,
},
config.KV{
Key: UsernameFormat,
Value: ldapArgs.UsernameFormat,
},
config.KV{
Key: GroupSearchFilter,
Value: ldapArgs.GroupSearchFilter,
},
config.KV{
Key: GroupSearchBaseDN,
Value: ldapArgs.GroupSearchBaseDistName,
},
}
}

52
internal/config/identity/openid/ecdsa-sha3_contrib.go Normal file
View File

@@ -0,0 +1,52 @@
// MinIO Object Storage (c) 2021 MinIO, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !fips
package openid
import (
"crypto"
"github.com/dgrijalva/jwt-go"
// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
_ "golang.org/x/crypto/sha3" // There is no SHA-3 FIPS-140 2 compliant implementation
)
// Specific instances for EC256 and company
var (
SigningMethodES3256 *jwt.SigningMethodECDSA
SigningMethodES3384 *jwt.SigningMethodECDSA
SigningMethodES3512 *jwt.SigningMethodECDSA
)
func init() {
// ES256
SigningMethodES3256 = &jwt.SigningMethodECDSA{Name: "ES3256", Hash: crypto.SHA3_256, KeySize: 32, CurveBits: 256}
jwt.RegisterSigningMethod(SigningMethodES3256.Alg(), func() jwt.SigningMethod {
return SigningMethodES3256
})
// ES384
SigningMethodES3384 = &jwt.SigningMethodECDSA{Name: "ES3384", Hash: crypto.SHA3_384, KeySize: 48, CurveBits: 384}
jwt.RegisterSigningMethod(SigningMethodES3384.Alg(), func() jwt.SigningMethod {
return SigningMethodES3384
})
// ES512
SigningMethodES3512 = &jwt.SigningMethodECDSA{Name: "ES3512", Hash: crypto.SHA3_512, KeySize: 66, CurveBits: 521}
jwt.RegisterSigningMethod(SigningMethodES3512.Alg(), func() jwt.SigningMethod {
return SigningMethodES3512
})
}

61
internal/config/identity/openid/help.go Normal file
View File

@@ -0,0 +1,61 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import "github.com/minio/minio/internal/config"
// Help template for OpenID identity feature.
var (
Help = config.HelpKVS{
config.HelpKV{
Key: ConfigURL,
Description: `openid discovery document e.g. "https://accounts.google.com/.well-known/openid-configuration"`,
Type: "url",
},
config.HelpKV{
Key: ClientID,
Description: `unique public identifier for apps e.g. "292085223830.apps.googleusercontent.com"`,
Type: "string",
Optional: true,
},
config.HelpKV{
Key: ClaimName,
Description: `JWT canned policy claim name, defaults to "policy"`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: ClaimPrefix,
Description: `JWT claim namespace prefix e.g. "customer1/"`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: Scopes,
Description: `Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin"`,
Optional: true,
Type: "csv",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

122
internal/config/identity/openid/jwks.go Normal file
View File

@@ -0,0 +1,122 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"crypto"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rsa"
"encoding/base64"
"errors"
"fmt"
"math/big"
)
// JWKS - https://tools.ietf.org/html/rfc7517
type JWKS struct {
Keys []*JWKS `json:"keys,omitempty"`
Kty string `json:"kty"`
Use string `json:"use,omitempty"`
Kid string `json:"kid,omitempty"`
Alg string `json:"alg,omitempty"`
Crv string `json:"crv,omitempty"`
X string `json:"x,omitempty"`
Y string `json:"y,omitempty"`
D string `json:"d,omitempty"`
N string `json:"n,omitempty"`
E string `json:"e,omitempty"`
K string `json:"k,omitempty"`
}
var (
errMalformedJWKRSAKey = errors.New("malformed JWK RSA key")
errMalformedJWKECKey = errors.New("malformed JWK EC key")
)
// DecodePublicKey - decodes JSON Web Key (JWK) as public key
func (key *JWKS) DecodePublicKey() (crypto.PublicKey, error) {
switch key.Kty {
case "RSA":
if key.N == "" || key.E == "" {
return nil, errMalformedJWKRSAKey
}
// decode exponent
ebuf, err := base64.RawURLEncoding.DecodeString(key.E)
if err != nil {
return nil, errMalformedJWKRSAKey
}
nbuf, err := base64.RawURLEncoding.DecodeString(key.N)
if err != nil {
return nil, errMalformedJWKRSAKey
}
var n, e big.Int
n.SetBytes(nbuf)
e.SetBytes(ebuf)
return &rsa.PublicKey{
E: int(e.Int64()),
N: &n,
}, nil
case "EC":
if key.Crv == "" || key.X == "" || key.Y == "" {
return nil, errMalformedJWKECKey
}
var curve elliptic.Curve
switch key.Crv {
case "P-224":
curve = elliptic.P224()
case "P-256":
curve = elliptic.P256()
case "P-384":
curve = elliptic.P384()
case "P-521":
curve = elliptic.P521()
default:
return nil, fmt.Errorf("Unknown curve type: %s", key.Crv)
}
xbuf, err := base64.RawURLEncoding.DecodeString(key.X)
if err != nil {
return nil, errMalformedJWKECKey
}
ybuf, err := base64.RawURLEncoding.DecodeString(key.Y)
if err != nil {
return nil, errMalformedJWKECKey
}
var x, y big.Int
x.SetBytes(xbuf)
y.SetBytes(ybuf)
return &ecdsa.PublicKey{
Curve: curve,
X: &x,
Y: &y,
}, nil
default:
return nil, fmt.Errorf("Unknown JWK key type %s", key.Kty)
}
}

127
internal/config/identity/openid/jwks_test.go Normal file
View File

@@ -0,0 +1,127 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"bytes"
"crypto"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rsa"
"encoding/json"
"testing"
)
func TestAzurePublicKey(t *testing.T) {
const jsonkey = `{"keys":[{"kty":"RSA","use":"sig","kid":"SsZsBNhZcF3Q9S4trpQBTByNRRI","x5t":"SsZsBNhZcF3Q9S4trpQBTByNRRI","n":"uHPewhg4WC3eLVPkEFlj7RDtaKYWXCI5G-LPVzsMKOuIu7qQQbeytIA6P6HT9_iIRt8zNQvuw4P9vbNjgUCpI6vfZGsjk3XuCVoB_bAIhvuBcQh9ePH2yEwS5reR-NrG1PsqzobnZZuigKCoDmuOb_UDx1DiVyNCbMBlEG7UzTQwLf5NP6HaRHx027URJeZvPAWY7zjHlSOuKoS_d1yUveaBFIgZqPWLCg44ck4gvik45HsNVWT9zYfT74dvUSSrMSR-SHFT7Hy1XjbVXpHJHNNAXpPoGoWXTuc0BxMsB4cqjfJqoftFGOG4x32vEzakArLPxAKwGvkvu0jToAyvSQ","e":"AQAB","x5c":"MIIDBTCCAe2gAwIBAgIQWHw7h/Ysh6hPcXpnrJ0N8DANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDQyNzAwMDAwMFoXDTI1MDQyNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALhz3sIYOFgt3i1T5BBZY+0Q7WimFlwiORviz1c7DCjriLu6kEG3srSAOj+h0/f4iEbfMzUL7sOD/b2zY4FAqSOr32RrI5N17glaAf2wCIb7gXEIfXjx9shMEua3kfjaxtT7Ks6G52WbooCgqA5rjm/1A8dQ4lcjQmzAZRBu1M00MC3+TT+h2kR8dNu1ESXmbzwFmO84x5UjriqEv3dclL3mgRSIGaj1iwoOOHJOIL4pOOR7DVVk/c2H0++Hb1EkqzEkfkhxU+x8tV421V6RyRzTQF6T6BqFl07nNAcTLAeHKo3yaqH7RRjhuMd9rxM2pAKyz8QCsBr5L7tI06AMr0kCAwEAAaMhMB8wHQYDVR0OBBYEFOI7M+DDFMlP7Ac3aomPnWo1QL1SMA0GCSqGSIb3DQEBCwUAA4IBAQBv+8rBiDY8sZDBoUDYwFQM74QjqCmgNQfv5B0Vjwg20HinERjQeH24uAWzyhWN9++FmeY4zcRXDY5UNmB0nJz7UGlprA9s7voQ0Lkyiud0DO072RPBg38LmmrqoBsLb3MB9MZ2CGBaHftUHfpdTvrgmXSP0IJn7mCUq27g+hFk7n/MLbN1k8JswEODIgdMRvGqN+mnrPKkviWmcVAZccsWfcmS1pKwXqICTKzd6WmVdz+cL7ZSd9I2X0pY4oRwauoE2bS95vrXljCYgLArI3XB2QcnglDDBRYu3Z3aIJb26PTIyhkVKT7xaXhXl4OgrbmQon9/O61G2dzpjzzBPqNP","issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"huN95IvPfehq34GzBDZ1GXGirnM","x5t":"huN95IvPfehq34GzBDZ1GXGirnM","n":"6lldKm5Rc_vMKa1RM_TtUv3tmtj52wLRrJqu13yGM3_h0dwru2ZP53y65wDfz6_tLCjoYuRCuVsjoW37-0zXUORJvZ0L90CAX-58lW7NcE4bAzA1pXv7oR9kQw0X8dp0atU4HnHeaTU8LZxcjJO79_H9cxgwa-clKfGxllcos8TsuurM8xi2dx5VqwzqNMB2s62l3MTN7AzctHUiQCiX2iJArGjAhs-mxS1wmyMIyOSipdodhjQWRAcseW-aFVyRTFVi8okl2cT1HJjPXdx0b1WqYSOzeRdrrLUcA0oR2Tzp7xzOYJZSGNnNLQqa9f6h6h52XbX0iAgxKgEDlRpbJw","e":"AQAB","x5c":["MIIDBTCCAe2gAwIBAgIQPCxFbySVSLZOggeWRzBWOjANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDYwNzAwMDAwMFoXDTI1MDYwNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOpZXSpuUXP7zCmtUTP07VL97ZrY+dsC0ayartd8hjN/4dHcK7tmT+d8uucA38+v7Swo6GLkQrlbI6Ft+/tM11DkSb2dC/dAgF/ufJVuzXBOGwMwNaV7+6EfZEMNF/HadGrVOB5x3mk1PC2cXIyTu/fx/XMYMGvnJSnxsZZXKLPE7LrqzPMYtnceVasM6jTAdrOtpdzEzewM3LR1IkAol9oiQKxowIbPpsUtcJsjCMjkoqXaHYY0FkQHLHlvmhVckUxVYvKJJdnE9RyYz13cdG9VqmEjs3kXa6y1HANKEdk86e8czmCWUhjZzS0KmvX+oeoedl219IgIMSoBA5UaWycCAwEAAaMhMB8wHQYDVR0OBBYEFFXP0ODFhjf3RS6oRijM5Tb+yB8CMA0GCSqGSIb3DQEBCwUAA4IBAQB9GtVikLTbJWIu5x9YCUTTKzNhi44XXogP/v8VylRSUHI5YTMdnWwvDIt/Y1sjNonmSy9PrioEjcIiI1U8nicveafMwIq5VLn+gEY2lg6KDJAzgAvA88CXqwfHHvtmYBovN7goolp8TY/kddMTf6TpNzN3lCTM2MK4Ye5xLLVGdp4bqWCOJ/qjwDxpTRSydYIkLUDwqNjv+sYfOElJpYAB4rTL/aw3ChJ1iaA4MtXEt6OjbUtbOa21lShfLzvNRbYK3+ukbrhmRl9lemJEeUls51vPuIe+jg+Ssp43aw7PQjxt4/MpfNMS2BfZ5F8GVSVG7qNb352cLLeJg5rc398Z"],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"M6pX7RHoraLsprfJeRCjSxuURhc","x5t":"M6pX7RHoraLsprfJeRCjSxuURhc","n":"xHScZMPo8FifoDcrgncWQ7mGJtiKhrsho0-uFPXg-OdnRKYudTD7-Bq1MDjcqWRf3IfDVjFJixQS61M7wm9wALDj--lLuJJ9jDUAWTA3xWvQLbiBM-gqU0sj4mc2lWm6nPfqlyYeWtQcSC0sYkLlayNgX4noKDaXivhVOp7bwGXq77MRzeL4-9qrRYKjuzHfZL7kNBCsqO185P0NI2Jtmw-EsqYsrCaHsfNRGRrTvUHUq3hWa859kK_5uNd7TeY2ZEwKVD8ezCmSfR59ZzyxTtuPpkCSHS9OtUvS3mqTYit73qcvprjl3R8hpjXLb8oftfpWr3hFRdpxrwuoQEO4QQ","e":"AQAB","x5c":["MIIC8TCCAdmgAwIBAgIQfEWlTVc1uINEc9RBi6qHMjANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwHhcNMTgxMDE0MDAwMDAwWhcNMjAxMDE0MDAwMDAwWjAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEdJxkw+jwWJ+gNyuCdxZDuYYm2IqGuyGjT64U9eD452dEpi51MPv4GrUwONypZF/ch8NWMUmLFBLrUzvCb3AAsOP76Uu4kn2MNQBZMDfFa9AtuIEz6CpTSyPiZzaVabqc9+qXJh5a1BxILSxiQuVrI2BfiegoNpeK+FU6ntvAZervsxHN4vj72qtFgqO7Md9kvuQ0EKyo7Xzk/Q0jYm2bD4SypiysJoex81EZGtO9QdSreFZrzn2Qr/m413tN5jZkTApUPx7MKZJ9Hn1nPLFO24+mQJIdL061S9LeapNiK3vepy+muOXdHyGmNctvyh+1+laveEVF2nGvC6hAQ7hBAgMBAAGjITAfMB0GA1UdDgQWBBQ5TKadw06O0cvXrQbXW0Nb3M3h/DANBgkqhkiG9w0BAQsFAAOCAQEAI48JaFtwOFcYS/3pfS5+7cINrafXAKTL+/+he4q+RMx4TCu/L1dl9zS5W1BeJNO2GUznfI+b5KndrxdlB6qJIDf6TRHh6EqfA18oJP5NOiKhU4pgkF2UMUw4kjxaZ5fQrSoD9omjfHAFNjradnHA7GOAoF4iotvXDWDBWx9K4XNZHWvD11Td66zTg5IaEQDIZ+f8WS6nn/98nAVMDtR9zW7Te5h9kGJGfe6WiHVaGRPpBvqC4iypGHjbRwANwofZvmp5wP08hY1CsnKY5tfP+E2k/iAQgKKa6QoxXToYvP7rsSkglak8N5g/+FJGnq4wP6cOzgZpjdPMwaVt5432GA=="],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"}]}`
var jk JWKS
if err := json.Unmarshal([]byte(jsonkey), &jk); err != nil {
t.Fatal("Unmarshal: ", err)
} else if len(jk.Keys) != 3 {
t.Fatalf("Expected 3 keys, got %d", len(jk.Keys))
}
var kids []string
for ii, jks := range jk.Keys {
_, err := jks.DecodePublicKey()
if err != nil {
t.Fatalf("Failed to decode key %d: %v", ii, err)
}
kids = append(kids, jks.Kid)
}
if len(kids) != 3 {
t.Fatalf("Failed to find the expected number of kids: 3, got %d", len(kids))
}
}
// A.1 - Example public keys
func TestPublicKey(t *testing.T) {
const jsonkey = `{"keys":
[
{"kty":"EC",
"crv":"P-256",
"x":"MKBCTNIcKUSDii11ySs3526iDZ8AiTo7Tu6KPAqv7D4",
"y":"4Etl6SRW2YiLUrN5vfvVHuhp7x8PxltmWWlbbM4IFyM",
"use":"enc",
"kid":"1"},
{"kty":"RSA",
"n": "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw",
"e":"AQAB",
"alg":"RS256",
"kid":"2011-04-29"}
]
}`
var jk JWKS
if err := json.Unmarshal([]byte(jsonkey), &jk); err != nil {
t.Fatal("Unmarshal: ", err)
} else if len(jk.Keys) != 2 {
t.Fatalf("Expected 2 keys, got %d", len(jk.Keys))
}
keys := make([]crypto.PublicKey, len(jk.Keys))
for ii, jks := range jk.Keys {
var err error
keys[ii], err = jks.DecodePublicKey()
if err != nil {
t.Fatalf("Failed to decode key %d: %v", ii, err)
}
}
if key0, ok := keys[0].(*ecdsa.PublicKey); !ok {
t.Fatalf("Expected ECDSA key[0], got %T", keys[0])
} else if key1, ok := keys[1].(*rsa.PublicKey); !ok {
t.Fatalf("Expected RSA key[1], got %T", keys[1])
} else if key0.Curve != elliptic.P256() {
t.Fatal("Key[0] is not using P-256 curve")
} else if !bytes.Equal(key0.X.Bytes(), []byte{0x30, 0xa0, 0x42, 0x4c, 0xd2,
0x1c, 0x29, 0x44, 0x83, 0x8a, 0x2d, 0x75, 0xc9, 0x2b, 0x37, 0xe7, 0x6e, 0xa2,
0xd, 0x9f, 0x0, 0x89, 0x3a, 0x3b, 0x4e, 0xee, 0x8a, 0x3c, 0xa, 0xaf, 0xec, 0x3e}) {
t.Fatalf("Bad key[0].X, got %v", key0.X.Bytes())
} else if !bytes.Equal(key0.Y.Bytes(), []byte{0xe0, 0x4b, 0x65, 0xe9, 0x24,
0x56, 0xd9, 0x88, 0x8b, 0x52, 0xb3, 0x79, 0xbd, 0xfb, 0xd5, 0x1e, 0xe8,
0x69, 0xef, 0x1f, 0xf, 0xc6, 0x5b, 0x66, 0x59, 0x69, 0x5b, 0x6c, 0xce,
0x8, 0x17, 0x23}) {
t.Fatalf("Bad key[0].Y, got %v", key0.Y.Bytes())
} else if key1.E != 0x10001 {
t.Fatalf("Bad key[1].E: %d", key1.E)
} else if !bytes.Equal(key1.N.Bytes(), []byte{0xd2, 0xfc, 0x7b, 0x6a, 0xa, 0x1e,
0x6c, 0x67, 0x10, 0x4a, 0xeb, 0x8f, 0x88, 0xb2, 0x57, 0x66, 0x9b, 0x4d, 0xf6,
0x79, 0xdd, 0xad, 0x9, 0x9b, 0x5c, 0x4a, 0x6c, 0xd9, 0xa8, 0x80, 0x15, 0xb5,
0xa1, 0x33, 0xbf, 0xb, 0x85, 0x6c, 0x78, 0x71, 0xb6, 0xdf, 0x0, 0xb, 0x55,
0x4f, 0xce, 0xb3, 0xc2, 0xed, 0x51, 0x2b, 0xb6, 0x8f, 0x14, 0x5c, 0x6e, 0x84,
0x34, 0x75, 0x2f, 0xab, 0x52, 0xa1, 0xcf, 0xc1, 0x24, 0x40, 0x8f, 0x79, 0xb5,
0x8a, 0x45, 0x78, 0xc1, 0x64, 0x28, 0x85, 0x57, 0x89, 0xf7, 0xa2, 0x49, 0xe3,
0x84, 0xcb, 0x2d, 0x9f, 0xae, 0x2d, 0x67, 0xfd, 0x96, 0xfb, 0x92, 0x6c, 0x19,
0x8e, 0x7, 0x73, 0x99, 0xfd, 0xc8, 0x15, 0xc0, 0xaf, 0x9, 0x7d, 0xde, 0x5a,
0xad, 0xef, 0xf4, 0x4d, 0xe7, 0xe, 0x82, 0x7f, 0x48, 0x78, 0x43, 0x24, 0x39,
0xbf, 0xee, 0xb9, 0x60, 0x68, 0xd0, 0x47, 0x4f, 0xc5, 0xd, 0x6d, 0x90, 0xbf,
0x3a, 0x98, 0xdf, 0xaf, 0x10, 0x40, 0xc8, 0x9c, 0x2, 0xd6, 0x92, 0xab, 0x3b,
0x3c, 0x28, 0x96, 0x60, 0x9d, 0x86, 0xfd, 0x73, 0xb7, 0x74, 0xce, 0x7, 0x40,
0x64, 0x7c, 0xee, 0xea, 0xa3, 0x10, 0xbd, 0x12, 0xf9, 0x85, 0xa8, 0xeb, 0x9f,
0x59, 0xfd, 0xd4, 0x26, 0xce, 0xa5, 0xb2, 0x12, 0xf, 0x4f, 0x2a, 0x34, 0xbc,
0xab, 0x76, 0x4b, 0x7e, 0x6c, 0x54, 0xd6, 0x84, 0x2, 0x38, 0xbc, 0xc4, 0x5, 0x87,
0xa5, 0x9e, 0x66, 0xed, 0x1f, 0x33, 0x89, 0x45, 0x77, 0x63, 0x5c, 0x47, 0xa,
0xf7, 0x5c, 0xf9, 0x2c, 0x20, 0xd1, 0xda, 0x43, 0xe1, 0xbf, 0xc4, 0x19, 0xe2,
0x22, 0xa6, 0xf0, 0xd0, 0xbb, 0x35, 0x8c, 0x5e, 0x38, 0xf9, 0xcb, 0x5, 0xa, 0xea,
0xfe, 0x90, 0x48, 0x14, 0xf1, 0xac, 0x1a, 0xa4, 0x9c, 0xca, 0x9e, 0xa0, 0xca, 0x83}) {
t.Fatalf("Bad key[1].N, got %v", key1.N.Bytes())
}
}

390
internal/config/identity/openid/jwt.go Normal file
View File

@@ -0,0 +1,390 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"crypto"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"sync"
"time"
jwtgo "github.com/dgrijalva/jwt-go"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/config"
xnet "github.com/minio/minio/internal/net"
"github.com/minio/pkg/env"
iampolicy "github.com/minio/pkg/iam/policy"
)
// Config - OpenID Config
// RSA authentication target arguments
type Config struct {
JWKS struct {
URL *xnet.URL `json:"url"`
} `json:"jwks"`
URL *xnet.URL `json:"url,omitempty"`
ClaimPrefix string `json:"claimPrefix,omitempty"`
ClaimName string `json:"claimName,omitempty"`
DiscoveryDoc DiscoveryDoc
ClientID string
publicKeys map[string]crypto.PublicKey
transport *http.Transport
closeRespFn func(io.ReadCloser)
mutex *sync.Mutex
}
// PopulatePublicKey - populates a new publickey from the JWKS URL.
func (r *Config) PopulatePublicKey() error {
r.mutex.Lock()
defer r.mutex.Unlock()
if r.JWKS.URL == nil || r.JWKS.URL.String() == "" {
return nil
}
transport := http.DefaultTransport
if r.transport != nil {
transport = r.transport
}
client := &http.Client{
Transport: transport,
}
resp, err := client.Get(r.JWKS.URL.String())
if err != nil {
return err
}
defer r.closeRespFn(resp.Body)
if resp.StatusCode != http.StatusOK {
return errors.New(resp.Status)
}
var jwk JWKS
if err = json.NewDecoder(resp.Body).Decode(&jwk); err != nil {
return err
}
for _, key := range jwk.Keys {
r.publicKeys[key.Kid], err = key.DecodePublicKey()
if err != nil {
return err
}
}
return nil
}
// UnmarshalJSON - decodes JSON data.
func (r *Config) UnmarshalJSON(data []byte) error {
// subtype to avoid recursive call to UnmarshalJSON()
type subConfig Config
var sr subConfig
if err := json.Unmarshal(data, &sr); err != nil {
return err
}
ar := Config(sr)
if ar.JWKS.URL == nil || ar.JWKS.URL.String() == "" {
*r = ar
return nil
}
*r = ar
return nil
}
// JWT - rs client grants provider details.
type JWT struct {
Config
}
// GetDefaultExpiration - returns the expiration seconds expected.
func GetDefaultExpiration(dsecs string) (time.Duration, error) {
defaultExpiryDuration := time.Duration(60) * time.Minute // Defaults to 1hr.
if dsecs != "" {
expirySecs, err := strconv.ParseInt(dsecs, 10, 64)
if err != nil {
return 0, auth.ErrInvalidDuration
}
// The duration, in seconds, of the role session.
// The value can range from 900 seconds (15 minutes)
// up to 7 days.
if expirySecs < 900 || expirySecs > 604800 {
return 0, auth.ErrInvalidDuration
}
defaultExpiryDuration = time.Duration(expirySecs) * time.Second
}
return defaultExpiryDuration, nil
}
func updateClaimsExpiry(dsecs string, claims map[string]interface{}) error {
expStr := claims["exp"]
if expStr == "" {
return ErrTokenExpired
}
// No custom duration requested, the claims can be used as is.
if dsecs == "" {
return nil
}
expAt, err := auth.ExpToInt64(expStr)
if err != nil {
return err
}
defaultExpiryDuration, err := GetDefaultExpiration(dsecs)
if err != nil {
return err
}
// Verify if JWT expiry is lesser than default expiry duration,
// if that is the case then set the default expiration to be
// from the JWT expiry claim.
if time.Unix(expAt, 0).UTC().Sub(time.Now().UTC()) < defaultExpiryDuration {
defaultExpiryDuration = time.Unix(expAt, 0).UTC().Sub(time.Now().UTC())
} // else honor the specified expiry duration.
expiry := time.Now().UTC().Add(defaultExpiryDuration).Unix()
claims["exp"] = strconv.FormatInt(expiry, 10) // update with new expiry.
return nil
}
// Validate - validates the access token.
func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
jp := new(jwtgo.Parser)
jp.ValidMethods = []string{
"RS256", "RS384", "RS512", "ES256", "ES384", "ES512",
"RS3256", "RS3384", "RS3512", "ES3256", "ES3384", "ES3512",
}
keyFuncCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
kid, ok := jwtToken.Header["kid"].(string)
if !ok {
return nil, fmt.Errorf("Invalid kid value %v", jwtToken.Header["kid"])
}
return p.publicKeys[kid], nil
}
var claims jwtgo.MapClaims
jwtToken, err := jp.ParseWithClaims(token, &claims, keyFuncCallback)
if err != nil {
// Re-populate the public key in-case the JWKS
// pubkeys are refreshed
if err = p.PopulatePublicKey(); err != nil {
return nil, err
}
jwtToken, err = jwtgo.ParseWithClaims(token, &claims, keyFuncCallback)
if err != nil {
return nil, err
}
}
if !jwtToken.Valid {
return nil, ErrTokenExpired
}
if err = updateClaimsExpiry(dsecs, claims); err != nil {
return nil, err
}
return claims, nil
}
// ID returns the provider name and authentication type.
func (p *JWT) ID() ID {
return "jwt"
}
// OpenID keys and envs.
const (
JwksURL = "jwks_url"
ConfigURL = "config_url"
ClaimName = "claim_name"
ClaimPrefix = "claim_prefix"
ClientID = "client_id"
Scopes = "scopes"
EnvIdentityOpenIDClientID = "MINIO_IDENTITY_OPENID_CLIENT_ID"
EnvIdentityOpenIDJWKSURL = "MINIO_IDENTITY_OPENID_JWKS_URL"
EnvIdentityOpenIDURL = "MINIO_IDENTITY_OPENID_CONFIG_URL"
EnvIdentityOpenIDClaimName = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
EnvIdentityOpenIDClaimPrefix = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
EnvIdentityOpenIDScopes = "MINIO_IDENTITY_OPENID_SCOPES"
)
// DiscoveryDoc - parses the output from openid-configuration
// for example https://accounts.google.com/.well-known/openid-configuration
type DiscoveryDoc struct {
Issuer string `json:"issuer,omitempty"`
AuthEndpoint string `json:"authorization_endpoint,omitempty"`
TokenEndpoint string `json:"token_endpoint,omitempty"`
UserInfoEndpoint string `json:"userinfo_endpoint,omitempty"`
RevocationEndpoint string `json:"revocation_endpoint,omitempty"`
JwksURI string `json:"jwks_uri,omitempty"`
ResponseTypesSupported []string `json:"response_types_supported,omitempty"`
SubjectTypesSupported []string `json:"subject_types_supported,omitempty"`
IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported,omitempty"`
ScopesSupported []string `json:"scopes_supported,omitempty"`
TokenEndpointAuthMethods []string `json:"token_endpoint_auth_methods_supported,omitempty"`
ClaimsSupported []string `json:"claims_supported,omitempty"`
CodeChallengeMethodsSupported []string `json:"code_challenge_methods_supported,omitempty"`
}
func parseDiscoveryDoc(u *xnet.URL, transport *http.Transport, closeRespFn func(io.ReadCloser)) (DiscoveryDoc, error) {
d := DiscoveryDoc{}
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return d, err
}
clnt := http.Client{
Transport: transport,
}
resp, err := clnt.Do(req)
if err != nil {
clnt.CloseIdleConnections()
return d, err
}
defer closeRespFn(resp.Body)
if resp.StatusCode != http.StatusOK {
return d, err
}
dec := json.NewDecoder(resp.Body)
if err = dec.Decode(&d); err != nil {
return d, err
}
return d, nil
}
// DefaultKVS - default config for OpenID config
var (
DefaultKVS = config.KVS{
config.KV{
Key: ConfigURL,
Value: "",
},
config.KV{
Key: ClientID,
Value: "",
},
config.KV{
Key: ClaimName,
Value: iampolicy.PolicyName,
},
config.KV{
Key: ClaimPrefix,
Value: "",
},
config.KV{
Key: Scopes,
Value: "",
},
config.KV{
Key: JwksURL,
Value: "",
},
}
)
// Enabled returns if jwks is enabled.
func Enabled(kvs config.KVS) bool {
return kvs.Get(JwksURL) != ""
}
// LookupConfig lookup jwks from config, override with any ENVs.
func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io.ReadCloser)) (c Config, err error) {
if err = config.CheckValidKeys(config.IdentityOpenIDSubSys, kvs, DefaultKVS); err != nil {
return c, err
}
jwksURL := env.Get(EnvIamJwksURL, "") // Legacy
if jwksURL == "" {
jwksURL = env.Get(EnvIdentityOpenIDJWKSURL, kvs.Get(JwksURL))
}
c = Config{
ClaimName: env.Get(EnvIdentityOpenIDClaimName, kvs.Get(ClaimName)),
ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kvs.Get(ClaimPrefix)),
publicKeys: make(map[string]crypto.PublicKey),
ClientID: env.Get(EnvIdentityOpenIDClientID, kvs.Get(ClientID)),
transport: transport,
closeRespFn: closeRespFn,
mutex: &sync.Mutex{}, // allocate for copying
}
configURL := env.Get(EnvIdentityOpenIDURL, kvs.Get(ConfigURL))
if configURL != "" {
c.URL, err = xnet.ParseHTTPURL(configURL)
if err != nil {
return c, err
}
c.DiscoveryDoc, err = parseDiscoveryDoc(c.URL, transport, closeRespFn)
if err != nil {
return c, err
}
}
if scopeList := env.Get(EnvIdentityOpenIDScopes, kvs.Get(Scopes)); scopeList != "" {
var scopes []string
for _, scope := range strings.Split(scopeList, ",") {
scope = strings.TrimSpace(scope)
if scope == "" {
return c, config.Errorf("empty scope value is not allowed '%s', please refer to our documentation", scopeList)
}
scopes = append(scopes, scope)
}
// Replace the discovery document scopes by client customized scopes.
c.DiscoveryDoc.ScopesSupported = scopes
}
if c.ClaimName == "" {
c.ClaimName = iampolicy.PolicyName
}
if jwksURL == "" {
// Fallback to discovery document jwksURL
jwksURL = c.DiscoveryDoc.JwksURI
}
if jwksURL == "" {
return c, nil
}
c.JWKS.URL, err = xnet.ParseHTTPURL(jwksURL)
if err != nil {
return c, err
}
if err = c.PopulatePublicKey(); err != nil {
return c, err
}
return c, nil
}
// NewJWT - initialize new jwt authenticator.
func NewJWT(c Config) *JWT {
return &JWT{c}
}

202
internal/config/identity/openid/jwt_test.go Normal file
View File

@@ -0,0 +1,202 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"crypto"
"encoding/json"
"net/url"
"sync"
"testing"
"time"
xnet "github.com/minio/minio/internal/net"
)
func TestUpdateClaimsExpiry(t *testing.T) {
testCases := []struct {
exp interface{}
dsecs string
expectedFailure bool
}{
{"", "", true},
{"-1", "0", true},
{"-1", "900", true},
{"1574812326", "900", false},
{1574812326, "900", false},
{int64(1574812326), "900", false},
{int(1574812326), "900", false},
{uint(1574812326), "900", false},
{uint64(1574812326), "900", false},
{json.Number("1574812326"), "900", false},
{1574812326.000, "900", false},
{time.Duration(3) * time.Minute, "900", false},
}
for _, testCase := range testCases {
testCase := testCase
t.Run("", func(t *testing.T) {
claims := map[string]interface{}{}
claims["exp"] = testCase.exp
err := updateClaimsExpiry(testCase.dsecs, claims)
if err != nil && !testCase.expectedFailure {
t.Errorf("Expected success, got failure %s", err)
}
if err == nil && testCase.expectedFailure {
t.Error("Expected failure, got success")
}
})
}
}
func TestJWTAzureFail(t *testing.T) {
const jsonkey = `{"keys":[{"kty":"RSA","use":"sig","kid":"SsZsBNhZcF3Q9S4trpQBTByNRRI","x5t":"SsZsBNhZcF3Q9S4trpQBTByNRRI","n":"uHPewhg4WC3eLVPkEFlj7RDtaKYWXCI5G-LPVzsMKOuIu7qQQbeytIA6P6HT9_iIRt8zNQvuw4P9vbNjgUCpI6vfZGsjk3XuCVoB_bAIhvuBcQh9ePH2yEwS5reR-NrG1PsqzobnZZuigKCoDmuOb_UDx1DiVyNCbMBlEG7UzTQwLf5NP6HaRHx027URJeZvPAWY7zjHlSOuKoS_d1yUveaBFIgZqPWLCg44ck4gvik45HsNVWT9zYfT74dvUSSrMSR-SHFT7Hy1XjbVXpHJHNNAXpPoGoWXTuc0BxMsB4cqjfJqoftFGOG4x32vEzakArLPxAKwGvkvu0jToAyvSQ","e":"AQAB","x5c":"MIIDBTCCAe2gAwIBAgIQWHw7h/Ysh6hPcXpnrJ0N8DANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDQyNzAwMDAwMFoXDTI1MDQyNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALhz3sIYOFgt3i1T5BBZY+0Q7WimFlwiORviz1c7DCjriLu6kEG3srSAOj+h0/f4iEbfMzUL7sOD/b2zY4FAqSOr32RrI5N17glaAf2wCIb7gXEIfXjx9shMEua3kfjaxtT7Ks6G52WbooCgqA5rjm/1A8dQ4lcjQmzAZRBu1M00MC3+TT+h2kR8dNu1ESXmbzwFmO84x5UjriqEv3dclL3mgRSIGaj1iwoOOHJOIL4pOOR7DVVk/c2H0++Hb1EkqzEkfkhxU+x8tV421V6RyRzTQF6T6BqFl07nNAcTLAeHKo3yaqH7RRjhuMd9rxM2pAKyz8QCsBr5L7tI06AMr0kCAwEAAaMhMB8wHQYDVR0OBBYEFOI7M+DDFMlP7Ac3aomPnWo1QL1SMA0GCSqGSIb3DQEBCwUAA4IBAQBv+8rBiDY8sZDBoUDYwFQM74QjqCmgNQfv5B0Vjwg20HinERjQeH24uAWzyhWN9++FmeY4zcRXDY5UNmB0nJz7UGlprA9s7voQ0Lkyiud0DO072RPBg38LmmrqoBsLb3MB9MZ2CGBaHftUHfpdTvrgmXSP0IJn7mCUq27g+hFk7n/MLbN1k8JswEODIgdMRvGqN+mnrPKkviWmcVAZccsWfcmS1pKwXqICTKzd6WmVdz+cL7ZSd9I2X0pY4oRwauoE2bS95vrXljCYgLArI3XB2QcnglDDBRYu3Z3aIJb26PTIyhkVKT7xaXhXl4OgrbmQon9/O61G2dzpjzzBPqNP","issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"huN95IvPfehq34GzBDZ1GXGirnM","x5t":"huN95IvPfehq34GzBDZ1GXGirnM","n":"6lldKm5Rc_vMKa1RM_TtUv3tmtj52wLRrJqu13yGM3_h0dwru2ZP53y65wDfz6_tLCjoYuRCuVsjoW37-0zXUORJvZ0L90CAX-58lW7NcE4bAzA1pXv7oR9kQw0X8dp0atU4HnHeaTU8LZxcjJO79_H9cxgwa-clKfGxllcos8TsuurM8xi2dx5VqwzqNMB2s62l3MTN7AzctHUiQCiX2iJArGjAhs-mxS1wmyMIyOSipdodhjQWRAcseW-aFVyRTFVi8okl2cT1HJjPXdx0b1WqYSOzeRdrrLUcA0oR2Tzp7xzOYJZSGNnNLQqa9f6h6h52XbX0iAgxKgEDlRpbJw","e":"AQAB","x5c":["MIIDBTCCAe2gAwIBAgIQPCxFbySVSLZOggeWRzBWOjANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDYwNzAwMDAwMFoXDTI1MDYwNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOpZXSpuUXP7zCmtUTP07VL97ZrY+dsC0ayartd8hjN/4dHcK7tmT+d8uucA38+v7Swo6GLkQrlbI6Ft+/tM11DkSb2dC/dAgF/ufJVuzXBOGwMwNaV7+6EfZEMNF/HadGrVOB5x3mk1PC2cXIyTu/fx/XMYMGvnJSnxsZZXKLPE7LrqzPMYtnceVasM6jTAdrOtpdzEzewM3LR1IkAol9oiQKxowIbPpsUtcJsjCMjkoqXaHYY0FkQHLHlvmhVckUxVYvKJJdnE9RyYz13cdG9VqmEjs3kXa6y1HANKEdk86e8czmCWUhjZzS0KmvX+oeoedl219IgIMSoBA5UaWycCAwEAAaMhMB8wHQYDVR0OBBYEFFXP0ODFhjf3RS6oRijM5Tb+yB8CMA0GCSqGSIb3DQEBCwUAA4IBAQB9GtVikLTbJWIu5x9YCUTTKzNhi44XXogP/v8VylRSUHI5YTMdnWwvDIt/Y1sjNonmSy9PrioEjcIiI1U8nicveafMwIq5VLn+gEY2lg6KDJAzgAvA88CXqwfHHvtmYBovN7goolp8TY/kddMTf6TpNzN3lCTM2MK4Ye5xLLVGdp4bqWCOJ/qjwDxpTRSydYIkLUDwqNjv+sYfOElJpYAB4rTL/aw3ChJ1iaA4MtXEt6OjbUtbOa21lShfLzvNRbYK3+ukbrhmRl9lemJEeUls51vPuIe+jg+Ssp43aw7PQjxt4/MpfNMS2BfZ5F8GVSVG7qNb352cLLeJg5rc398Z"],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"M6pX7RHoraLsprfJeRCjSxuURhc","x5t":"M6pX7RHoraLsprfJeRCjSxuURhc","n":"xHScZMPo8FifoDcrgncWQ7mGJtiKhrsho0-uFPXg-OdnRKYudTD7-Bq1MDjcqWRf3IfDVjFJixQS61M7wm9wALDj--lLuJJ9jDUAWTA3xWvQLbiBM-gqU0sj4mc2lWm6nPfqlyYeWtQcSC0sYkLlayNgX4noKDaXivhVOp7bwGXq77MRzeL4-9qrRYKjuzHfZL7kNBCsqO185P0NI2Jtmw-EsqYsrCaHsfNRGRrTvUHUq3hWa859kK_5uNd7TeY2ZEwKVD8ezCmSfR59ZzyxTtuPpkCSHS9OtUvS3mqTYit73qcvprjl3R8hpjXLb8oftfpWr3hFRdpxrwuoQEO4QQ","e":"AQAB","x5c":["MIIC8TCCAdmgAwIBAgIQfEWlTVc1uINEc9RBi6qHMjANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwHhcNMTgxMDE0MDAwMDAwWhcNMjAxMDE0MDAwMDAwWjAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEdJxkw+jwWJ+gNyuCdxZDuYYm2IqGuyGjT64U9eD452dEpi51MPv4GrUwONypZF/ch8NWMUmLFBLrUzvCb3AAsOP76Uu4kn2MNQBZMDfFa9AtuIEz6CpTSyPiZzaVabqc9+qXJh5a1BxILSxiQuVrI2BfiegoNpeK+FU6ntvAZervsxHN4vj72qtFgqO7Md9kvuQ0EKyo7Xzk/Q0jYm2bD4SypiysJoex81EZGtO9QdSreFZrzn2Qr/m413tN5jZkTApUPx7MKZJ9Hn1nPLFO24+mQJIdL061S9LeapNiK3vepy+muOXdHyGmNctvyh+1+laveEVF2nGvC6hAQ7hBAgMBAAGjITAfMB0GA1UdDgQWBBQ5TKadw06O0cvXrQbXW0Nb3M3h/DANBgkqhkiG9w0BAQsFAAOCAQEAI48JaFtwOFcYS/3pfS5+7cINrafXAKTL+/+he4q+RMx4TCu/L1dl9zS5W1BeJNO2GUznfI+b5KndrxdlB6qJIDf6TRHh6EqfA18oJP5NOiKhU4pgkF2UMUw4kjxaZ5fQrSoD9omjfHAFNjradnHA7GOAoF4iotvXDWDBWx9K4XNZHWvD11Td66zTg5IaEQDIZ+f8WS6nn/98nAVMDtR9zW7Te5h9kGJGfe6WiHVaGRPpBvqC4iypGHjbRwANwofZvmp5wP08hY1CsnKY5tfP+E2k/iAQgKKa6QoxXToYvP7rsSkglak8N5g/+FJGnq4wP6cOzgZpjdPMwaVt5432GA=="],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"}]}`
var jk JWKS
if err := json.Unmarshal([]byte(jsonkey), &jk); err != nil {
t.Fatal("Unmarshal: ", err)
} else if len(jk.Keys) != 3 {
t.Fatalf("Expected 3 keys, got %d", len(jk.Keys))
}
keys := make(map[string]crypto.PublicKey, len(jk.Keys))
for ii, jks := range jk.Keys {
var err error
keys[jks.Kid], err = jks.DecodePublicKey()
if err != nil {
t.Fatalf("Failed to decode key %d: %v", ii, err)
}
}
jwtToken := `eyJ0eXAiOiJKV1QiLCJub25jZSI6Il9KUlNlS0tjNmxIVVRJdk1tMmZNWktBTEtZOUpwenNPalc5cl96OEk2VFkiLCJhbGciOiJSUzI1NiIsIng1dCI6Imh1Tjk1SXZQZmVocTM0R3pCRFoxR1hHaXJuTSIsImtpZCI6Imh1Tjk1SXZQZmVocTM0R3pCRFoxR1hHaXJuTSJ9.eyJhdWQiOiIwMDAwMDAwMy0wMDAwLTAwMDAtYzAwMC0wMDAwMDAwMDAwMDAiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC85MDZhZWZlOS03NmE3LTRmNjUtYjgyZC01ZWMyMDc3NWQ1YWEvIiwiaWF0IjoxNTk0NjU3NTIwLCJuYmYiOjE1OTQ2NTc1MjAsImV4cCI6MTU5NDY2MTQyMCwiYWNjdCI6MCwiYWNyIjoiMSIsImFpbyI6IkUyQmdZTmliK3QydHh5SklRT1dEeXFsRDNVWUxwWGxVeXhmMGxFZmxMQ2t0VTU3TnpBVUEiLCJhbXIiOlsicHdkIl0sImFwcF9kaXNwbGF5bmFtZSI6ImR4YXp1cmUiLCJhcHBpZCI6ImY0ZDM0M2IyLTRmNDYtNGUyYy04M2RlLTVkN2QyN2Q2OTUyNSIsImFwcGlkYWNyIjoiMSIsImZhbWlseV9uYW1lIjoiS2FzYSIsImdpdmVuX25hbWUiOiJCYWxha3Jpc2huYSIsImluX2NvcnAiOiJ0cnVlIiwiaXBhZGRyIjoiMTk4LjE3OC4xMi42OCIsIm5hbWUiOiJLYXNhLCBCYWxha3Jpc2huYSIsIm9pZCI6IjZjNDJhMTYwLTIyZGMtNDJmNy05MDRlLTQwODZkNzg0MzQ0OCIsIm9ucHJlbV9zaWQiOiJTLTEtNS0yMS0yMDUyMTExMzAyLTQ0ODUzOTcyMy0xODAxNjc0NTMxLTQ2NDkzMDciLCJwbGF0ZiI6IjE0IiwicHVpZCI6IjEwMDNCRkZEOTZGRTM3MzkiLCJzY3AiOiJEaXJlY3RvcnkuUmVhZC5BbGwgb3BlbmlkIHByb2ZpbGUgVXNlci5SZWFkIGVtYWlsIiwic2lnbmluX3N0YXRlIjpbImlua25vd25udHdrIl0sInN1YiI6IkNkTEQ3X2tnbnRsdHQta2FqaUJOYWkyNkxvUUxsMF9xd3d6MXhCcDRzcHciLCJ0ZW5hbnRfcmVnaW9uX3Njb3BlIjoiTkEiLCJ0aWQiOiI5MDZhZWZlOS03NmE3LTRmNjUtYjgyZC01ZWMyMDc3NWQ1YWEiLCJ1bmlxdWVfbmFtZSI6ImJrYXNhNzI0QGNhYmxlLmNvbWNhc3QuY29tIiwidXBuIjoiYmthc2E3MjRAY2FibGUuY29tY2FzdC5jb20iLCJ1dGkiOiJ0UThJVEpjb0lVdUhaZXpBb2twZ0FBIiwidmVyIjoiMS4wIiwieG1zX3N0Ijp7InN1YiI6InJCQlZGX1NlOUZpcG16VUg5VVNWNXl1aVRwazFkb2s4ODNxb3R6UVN0bU0ifSwieG1zX3RjZHQiOjEzNzUxMjYzMzR9.TNzUp6b2ZJA6rBJzwpyC58UmH5CkEZFoB1d4sFnDGR_o3sdgtsRdR6ogeCZudaIPBCDCQz5_yMo59_hWUt0Q2iQI2sy1SUtdOAUtu4dcY-0LhqS0tIprc5mwBJytxJ9BVttmZ8r0_lqBSqn9dl8LajWpSCcVNBSFxT7V6N0zi8ONtWXbizkZOb52Tt2uVO4ak7bzi9gstEGiDTLxhDDJLpo3sZVy7LTI2gSMVsOoyeKBHk4GL5Fs0Ezz0yHad0MrJ8tULiqXocIC3vlA5u6-klOyfx04v-Lzs1L4F4XkAysJgGIAj7E9TBSw0XhMM5WKF25AzKGznLLt11r3cCIxCg`
u1, err := xnet.ParseHTTPURL("http://localhost:8443")
if err != nil {
t.Fatal(err)
}
cfg := Config{}
cfg.mutex = &sync.Mutex{}
cfg.JWKS.URL = u1
cfg.publicKeys = keys
jwt := NewJWT(cfg)
if jwt.ID() != "jwt" {
t.Fatalf("Uexpected id %s for the validator", jwt.ID())
}
if _, err := jwt.Validate(jwtToken, ""); err == nil {
// Azure should fail due to non OIDC compliant JWT
// generated by Azure AD
t.Fatal(err)
}
}
func TestJWT(t *testing.T) {
const jsonkey = `{"keys":
[
{"kty":"RSA",
"n": "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw",
"e":"AQAB",
"alg":"RS256",
"kid":"2011-04-29"}
]
}`
var jk JWKS
if err := json.Unmarshal([]byte(jsonkey), &jk); err != nil {
t.Fatal("Unmarshal: ", err)
} else if len(jk.Keys) != 1 {
t.Fatalf("Expected 1 keys, got %d", len(jk.Keys))
}
keys := make(map[string]crypto.PublicKey, len(jk.Keys))
for ii, jks := range jk.Keys {
var err error
keys[jks.Kid], err = jks.DecodePublicKey()
if err != nil {
t.Fatalf("Failed to decode key %d: %v", ii, err)
}
}
u1, err := xnet.ParseHTTPURL("http://localhost:8443")
if err != nil {
t.Fatal(err)
}
cfg := Config{}
cfg.mutex = &sync.Mutex{}
cfg.JWKS.URL = u1
cfg.publicKeys = keys
jwt := NewJWT(cfg)
if jwt.ID() != "jwt" {
t.Fatalf("Uexpected id %s for the validator", jwt.ID())
}
u, err := url.Parse("http://localhost:8443/?Token=invalid")
if err != nil {
t.Fatal(err)
}
if _, err := jwt.Validate(u.Query().Get("Token"), ""); err == nil {
t.Fatal(err)
}
}
func TestDefaultExpiryDuration(t *testing.T) {
testCases := []struct {
reqURL string
duration time.Duration
expectErr bool
}{
{
reqURL: "http://localhost:8443/?Token=xxxxx",
duration: time.Duration(60) * time.Minute,
},
{
reqURL: "http://localhost:8443/?DurationSeconds=9s",
expectErr: true,
},
{
reqURL: "http://localhost:8443/?DurationSeconds=604801",
expectErr: true,
},
{
reqURL: "http://localhost:8443/?DurationSeconds=800",
expectErr: true,
},
{
reqURL: "http://localhost:8443/?DurationSeconds=901",
duration: time.Duration(901) * time.Second,
},
}
for i, testCase := range testCases {
u, err := url.Parse(testCase.reqURL)
if err != nil {
t.Fatal(err)
}
d, err := GetDefaultExpiration(u.Query().Get("DurationSeconds"))
gotErr := (err != nil)
if testCase.expectErr != gotErr {
t.Errorf("Test %d: Expected %v, got %v with error %s", i+1, testCase.expectErr, gotErr, err)
}
if d != testCase.duration {
t.Errorf("Test %d: Expected duration %d, got %d", i+1, testCase.duration, d)
}
}
}

47
internal/config/identity/openid/legacy.go Normal file
View File

@@ -0,0 +1,47 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import "github.com/minio/minio/internal/config"
// Legacy envs
const (
EnvIamJwksURL = "MINIO_IAM_JWKS_URL"
)
// SetIdentityOpenID - One time migration code needed, for migrating from older config to new for OpenIDConfig.
func SetIdentityOpenID(s config.Config, cfg Config) {
if cfg.JWKS.URL == nil || cfg.JWKS.URL.String() == "" {
// No need to save not-enabled settings in new config.
return
}
s[config.IdentityOpenIDSubSys][config.Default] = config.KVS{
config.KV{
Key: JwksURL,
Value: cfg.JWKS.URL.String(),
},
config.KV{
Key: ConfigURL,
Value: "",
},
config.KV{
Key: ClaimPrefix,
Value: "",
},
}
}

53
internal/config/identity/openid/rsa-sha3_contrib.go Normal file
View File

@@ -0,0 +1,53 @@
// MinIO Object Storage (c) 2021 MinIO, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !fips
package openid
import (
"crypto"
"github.com/dgrijalva/jwt-go"
// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
_ "golang.org/x/crypto/sha3" // There is no SHA-3 FIPS-140 2 compliant implementation
)
// Specific instances for RS256 and company
var (
SigningMethodRS3256 *jwt.SigningMethodRSA
SigningMethodRS3384 *jwt.SigningMethodRSA
SigningMethodRS3512 *jwt.SigningMethodRSA
)
func init() {
// RS3256
SigningMethodRS3256 = &jwt.SigningMethodRSA{Name: "RS3256", Hash: crypto.SHA3_256}
jwt.RegisterSigningMethod(SigningMethodRS3256.Alg(), func() jwt.SigningMethod {
return SigningMethodRS3256
})
// RS3384
SigningMethodRS3384 = &jwt.SigningMethodRSA{Name: "RS3384", Hash: crypto.SHA3_384}
jwt.RegisterSigningMethod(SigningMethodRS3384.Alg(), func() jwt.SigningMethod {
return SigningMethodRS3384
})
// RS3512
SigningMethodRS3512 = &jwt.SigningMethodRSA{Name: "RS3512", Hash: crypto.SHA3_512}
jwt.RegisterSigningMethod(SigningMethodRS3512.Alg(), func() jwt.SigningMethod {
return SigningMethodRS3512
})
}

92
internal/config/identity/openid/validators.go Normal file
View File

@@ -0,0 +1,92 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"errors"
"fmt"
"sync"
)
// ID - holds identification name authentication validator target.
type ID string
// Validator interface describes basic implementation
// requirements of various authentication providers.
type Validator interface {
// Validate is a custom validator function for this provider,
// each validation is authenticationType or provider specific.
Validate(token string, duration string) (map[string]interface{}, error)
// ID returns provider name of this provider.
ID() ID
}
// ErrTokenExpired - error token expired
var (
ErrTokenExpired = errors.New("token expired")
)
// Validators - holds list of providers indexed by provider id.
type Validators struct {
sync.RWMutex
providers map[ID]Validator
}
// Add - adds unique provider to provider list.
func (list *Validators) Add(provider Validator) error {
list.Lock()
defer list.Unlock()
if _, ok := list.providers[provider.ID()]; ok {
return fmt.Errorf("provider %v already exists", provider.ID())
}
list.providers[provider.ID()] = provider
return nil
}
// List - returns available provider IDs.
func (list *Validators) List() []ID {
list.RLock()
defer list.RUnlock()
keys := []ID{}
for k := range list.providers {
keys = append(keys, k)
}
return keys
}
// Get - returns the provider for the given providerID, if not found
// returns an error.
func (list *Validators) Get(id ID) (p Validator, err error) {
list.RLock()
defer list.RUnlock()
var ok bool
if p, ok = list.providers[id]; !ok {
return nil, fmt.Errorf("provider %v doesn't exist", id)
}
return p, nil
}
// NewValidators - creates Validators.
func NewValidators() *Validators {
return &Validators{providers: make(map[ID]Validator)}
}

105
internal/config/identity/openid/validators_test.go Normal file
View File

@@ -0,0 +1,105 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package openid
import (
"net/http"
"net/http/httptest"
"testing"
xnet "github.com/minio/minio/internal/net"
)
type errorValidator struct{}
func (e errorValidator) Validate(token, dsecs string) (map[string]interface{}, error) {
return nil, ErrTokenExpired
}
func (e errorValidator) ID() ID {
return "err"
}
func TestValidators(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("content-type", "application/json")
w.Write([]byte(`{
"keys" : [ {
"kty" : "RSA",
"kid" : "1438289820780",
"use" : "sig",
"alg" : "RS256",
"n" : "idWPro_QiAFOdMsJD163lcDIPogOwXogRo3Pct2MMyeE2GAGqV20Sc8QUbuLDfPl-7Hi9IfFOz--JY6QL5l92eV-GJXkTmidUEooZxIZSp3ghRxLCqlyHeF5LuuM5LPRFDeF4YWFQT_D2eNo_w95g6qYSeOwOwGIfaHa2RMPcQAiM6LX4ot-Z7Po9z0_3ztFa02m3xejEFr2rLRqhFl3FZJaNnwTUk6an6XYsunxMk3Ya3lRaKJReeXeFtfTpShgtPiAl7lIfLJH9h26h2OAlww531DpxHSm1gKXn6bjB0NTC55vJKft4wXoc_0xKZhnWmjQE8d9xE8e1Z3Ll1LYbw",
"e" : "AQAB"
}, {
"kty" : "RSA",
"kid" : "1438289856256",
"use" : "sig",
"alg" : "RS256",
"n" : "zo5cKcbFECeiH8eGx2D-DsFSpjSKbTVlXD6uL5JAy9rYIv7eYEP6vrKeX-x1z70yEdvgk9xbf9alc8siDfAz3rLCknqlqL7XGVAQL0ZP63UceDmD60LHOzMrx4eR6p49B3rxFfjvX2SWSV3-1H6XNyLk_ALbG6bGCFGuWBQzPJB4LMKCrOFq-6jtRKOKWBXYgkYkaYs5dG-3e2ULbq-y2RdgxYh464y_-MuxDQfvUgP787XKfcXP_XjJZvyuOEANjVyJYZSOyhHUlSGJapQ8ztHdF-swsnf7YkePJ2eR9fynWV2ZoMaXOdidgZtGTa4R1Z4BgH2C0hKJiqRy9fB7Gw",
"e" : "AQAB"
} ]
}
`))
w.(http.Flusher).Flush()
}))
defer ts.Close()
vrs := NewValidators()
if err := vrs.Add(&errorValidator{}); err != nil {
t.Fatal(err)
}
if err := vrs.Add(&errorValidator{}); err == nil {
t.Fatal("Unexpected should return error for double inserts")
}
if _, err := vrs.Get("unknown"); err == nil {
t.Fatal("Unexpected should return error for unknown validators")
}
v, err := vrs.Get("err")
if err != nil {
t.Fatal(err)
}
if _, err = v.Validate("", ""); err != ErrTokenExpired {
t.Fatalf("Expected error %s, got %s", ErrTokenExpired, err)
}
vids := vrs.List()
if len(vids) == 0 || len(vids) > 1 {
t.Fatalf("Unexpected number of vids %v", vids)
}
u, err := xnet.ParseHTTPURL(ts.URL)
if err != nil {
t.Fatal(err)
}
cfg := Config{}
cfg.JWKS.URL = u
if err = vrs.Add(NewJWT(cfg)); err != nil {
t.Fatal(err)
}
if _, err = vrs.Get("jwt"); err != nil {
t.Fatal(err)
}
}

56
internal/config/legacy.go Normal file
View File

@@ -0,0 +1,56 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import "github.com/minio/minio/internal/auth"
//// One time migration code section
// SetCredentials - One time migration code needed, for migrating from older config to new for server credentials.
func SetCredentials(c Config, cred auth.Credentials) {
creds, err := auth.CreateCredentials(cred.AccessKey, cred.SecretKey)
if err != nil {
return
}
if !creds.IsValid() {
return
}
c[CredentialsSubSys][Default] = KVS{
KV{
Key: AccessKey,
Value: cred.AccessKey,
},
KV{
Key: SecretKey,
Value: cred.SecretKey,
},
}
}
// SetRegion - One time migration code needed, for migrating from older config to new for server Region.
func SetRegion(c Config, name string) {
if name == "" {
return
}
c[RegionSubSys][Default] = KVS{
KV{
Key: RegionName,
Value: name,
},
}
}

28
internal/config/logger.go Normal file
View File

@@ -0,0 +1,28 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import "context"
// Logger contains injected logger methods.
var Logger = struct {
Info func(msg string, data ...interface{})
LogIf func(ctx context.Context, err error, errKind ...interface{})
}{
// Initialized via injection.
}

69
internal/config/notify/config.go Normal file
View File

@@ -0,0 +1,69 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package notify
import (
"github.com/minio/minio/internal/event/target"
)
// Config - notification target configuration structure, holds
// information about various notification targets.
type Config struct {
AMQP map[string]target.AMQPArgs `json:"amqp"`
Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
Kafka map[string]target.KafkaArgs `json:"kafka"`
MQTT map[string]target.MQTTArgs `json:"mqtt"`
MySQL map[string]target.MySQLArgs `json:"mysql"`
NATS map[string]target.NATSArgs `json:"nats"`
NSQ map[string]target.NSQArgs `json:"nsq"`
PostgreSQL map[string]target.PostgreSQLArgs `json:"postgresql"`
Redis map[string]target.RedisArgs `json:"redis"`
Webhook map[string]target.WebhookArgs `json:"webhook"`
}
const (
defaultTarget = "1"
)
// NewConfig - initialize notification config.
func NewConfig() Config {
// Make sure to initialize notification targets
cfg := Config{
NSQ: make(map[string]target.NSQArgs),
AMQP: make(map[string]target.AMQPArgs),
MQTT: make(map[string]target.MQTTArgs),
NATS: make(map[string]target.NATSArgs),
Redis: make(map[string]target.RedisArgs),
MySQL: make(map[string]target.MySQLArgs),
Kafka: make(map[string]target.KafkaArgs),
Webhook: make(map[string]target.WebhookArgs),
PostgreSQL: make(map[string]target.PostgreSQLArgs),
Elasticsearch: make(map[string]target.ElasticsearchArgs),
}
cfg.NSQ[defaultTarget] = target.NSQArgs{}
cfg.AMQP[defaultTarget] = target.AMQPArgs{}
cfg.MQTT[defaultTarget] = target.MQTTArgs{}
cfg.NATS[defaultTarget] = target.NATSArgs{}
cfg.Redis[defaultTarget] = target.RedisArgs{}
cfg.MySQL[defaultTarget] = target.MySQLArgs{}
cfg.Kafka[defaultTarget] = target.KafkaArgs{}
cfg.Webhook[defaultTarget] = target.WebhookArgs{}
cfg.PostgreSQL[defaultTarget] = target.PostgreSQLArgs{}
cfg.Elasticsearch[defaultTarget] = target.ElasticsearchArgs{}
return cfg
}

636
internal/config/notify/help.go Normal file
View File

@@ -0,0 +1,636 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package notify
import (
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/event/target"
)
const (
formatComment = `'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'`
queueDirComment = `staging dir for undelivered messages e.g. '/home/events'`
queueLimitComment = `maximum limit for undelivered messages, defaults to '100000'`
)
// Help template inputs for all notification targets
var (
HelpWebhook = config.HelpKVS{
config.HelpKV{
Key: target.WebhookEndpoint,
Description: "webhook server endpoint e.g. http://localhost:8080/minio/events",
Type: "url",
},
config.HelpKV{
Key: target.WebhookAuthToken,
Description: "opaque string or JWT authorization token",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.WebhookQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.WebhookQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
config.HelpKV{
Key: target.WebhookClientCert,
Description: "client cert for Webhook mTLS auth",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.WebhookClientKey,
Description: "client cert key for Webhook mTLS auth",
Optional: true,
Type: "string",
},
}
HelpAMQP = config.HelpKVS{
config.HelpKV{
Key: target.AmqpURL,
Description: "AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672`",
Type: "url",
},
config.HelpKV{
Key: target.AmqpExchange,
Description: "name of the AMQP exchange",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.AmqpExchangeType,
Description: "AMQP exchange type",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.AmqpRoutingKey,
Description: "routing key for publishing",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.AmqpMandatory,
Description: "quietly ignore undelivered messages when set to 'off', default is 'on'",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpDurable,
Description: "persist queue across broker restarts when set to 'on', default is 'off'",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpNoWait,
Description: "non-blocking message delivery when set to 'on', default is 'off'",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpInternal,
Description: "set to 'on' for exchange to be not used directly by publishers, but only when bound to other exchanges",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpAutoDeleted,
Description: "auto delete queue when set to 'on', when there are no consumers",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.AmqpDeliveryMode,
Description: "set to '1' for non-persistent or '2' for persistent queue",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.AmqpQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.AmqpQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpKafka = config.HelpKVS{
config.HelpKV{
Key: target.KafkaBrokers,
Description: "comma separated list of Kafka broker addresses",
Type: "csv",
},
config.HelpKV{
Key: target.KafkaTopic,
Description: "Kafka topic used for bucket notifications",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.KafkaSASLUsername,
Description: "username for SASL/PLAIN or SASL/SCRAM authentication",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.KafkaSASLPassword,
Description: "password for SASL/PLAIN or SASL/SCRAM authentication",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.KafkaSASLMechanism,
Description: "sasl authentication mechanism, default 'plain'",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.KafkaTLSClientAuth,
Description: "clientAuth determines the Kafka server's policy for TLS client auth",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.KafkaSASL,
Description: "set to 'on' to enable SASL authentication",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.KafkaTLS,
Description: "set to 'on' to enable TLS",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.KafkaTLSSkipVerify,
Description: `trust server TLS without verification, defaults to "on" (verify)`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.KafkaClientTLSCert,
Description: "path to client certificate for mTLS auth",
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.KafkaClientTLSKey,
Description: "path to client key for mTLS auth",
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.KafkaQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.KafkaQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.KafkaVersion,
Description: "specify the version of the Kafka cluster",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpMQTT = config.HelpKVS{
config.HelpKV{
Key: target.MqttBroker,
Description: "MQTT server endpoint e.g. `tcp://localhost:1883`",
Type: "uri",
},
config.HelpKV{
Key: target.MqttTopic,
Description: "name of the MQTT topic to publish",
Type: "string",
},
config.HelpKV{
Key: target.MqttUsername,
Description: "MQTT username",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.MqttPassword,
Description: "MQTT password",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.MqttQoS,
Description: "set the quality of service priority, defaults to '0'",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.MqttKeepAliveInterval,
Description: "keep-alive interval for MQTT connections in s,m,h,d",
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: target.MqttReconnectInterval,
Description: "reconnect interval for MQTT connections in s,m,h,d",
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: target.MqttQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.MqttQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpPostgres = config.HelpKVS{
config.HelpKV{
Key: target.PostgresConnectionString,
Description: `Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable"`,
Type: "string",
},
config.HelpKV{
Key: target.PostgresTable,
Description: "DB table name to store/update events, table is auto-created",
Type: "string",
},
config.HelpKV{
Key: target.PostgresFormat,
Description: formatComment,
Type: "namespace*|access",
},
config.HelpKV{
Key: target.PostgresQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.PostgresQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
config.HelpKV{
Key: target.PostgresMaxOpenConnections,
Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.",
Optional: true,
Type: "number",
},
}
HelpMySQL = config.HelpKVS{
config.HelpKV{
Key: target.MySQLDSNString,
Description: `MySQL data-source-name connection string e.g. "<user>:<password>@tcp(<host>:<port>)/<database>"`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.MySQLTable,
Description: "DB table name to store/update events, table is auto-created",
Type: "string",
},
config.HelpKV{
Key: target.MySQLFormat,
Description: formatComment,
Type: "namespace*|access",
},
config.HelpKV{
Key: target.MySQLQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.MySQLQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
config.HelpKV{
Key: target.MySQLMaxOpenConnections,
Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.",
Optional: true,
Type: "number",
},
}
HelpNATS = config.HelpKVS{
config.HelpKV{
Key: target.NATSAddress,
Description: "NATS server address e.g. '0.0.0.0:4222'",
Type: "address",
},
config.HelpKV{
Key: target.NATSSubject,
Description: "NATS subscription subject",
Type: "string",
},
config.HelpKV{
Key: target.NATSUsername,
Description: "NATS username",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.NATSPassword,
Description: "NATS password",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.NATSToken,
Description: "NATS token",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.NATSTLS,
Description: "set to 'on' to enable TLS",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NATSTLSSkipVerify,
Description: `trust server TLS without verification, defaults to "on" (verify)`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NATSPingInterval,
Description: "client ping commands interval in s,m,h,d. Disabled by default",
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: target.NATSStreaming,
Description: "set to 'on', to use streaming NATS server",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NATSStreamingAsync,
Description: "set to 'on', to enable asynchronous publish",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NATSStreamingMaxPubAcksInFlight,
Description: "number of messages to publish without waiting for ACKs",
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.NATSStreamingClusterID,
Description: "unique ID for NATS streaming cluster",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.NATSCertAuthority,
Description: "path to certificate chain of the target NATS server",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.NATSClientCert,
Description: "client cert for NATS mTLS auth",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.NATSClientKey,
Description: "client cert key for NATS mTLS auth",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.NATSQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.NATSQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpNSQ = config.HelpKVS{
config.HelpKV{
Key: target.NSQAddress,
Description: "NSQ server address e.g. '127.0.0.1:4150'",
Type: "address",
},
config.HelpKV{
Key: target.NSQTopic,
Description: "NSQ topic",
Type: "string",
},
config.HelpKV{
Key: target.NSQTLS,
Description: "set to 'on' to enable TLS",
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NSQTLSSkipVerify,
Description: `trust server TLS without verification, defaults to "on" (verify)`,
Optional: true,
Type: "on|off",
},
config.HelpKV{
Key: target.NSQQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.NSQQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpES = config.HelpKVS{
config.HelpKV{
Key: target.ElasticURL,
Description: "Elasticsearch server's address, with optional authentication info",
Type: "url",
},
config.HelpKV{
Key: target.ElasticIndex,
Description: `Elasticsearch index to store/update events, index is auto-created`,
Type: "string",
},
config.HelpKV{
Key: target.ElasticFormat,
Description: formatComment,
Type: "namespace*|access",
},
config.HelpKV{
Key: target.ElasticQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.ElasticQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: target.ElasticUsername,
Description: "username for Elasticsearch basic-auth",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.ElasticPassword,
Description: "password for Elasticsearch basic-auth",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
HelpRedis = config.HelpKVS{
config.HelpKV{
Key: target.RedisAddress,
Description: "Redis server's address. For example: `localhost:6379`",
Type: "address",
},
config.HelpKV{
Key: target.RedisKey,
Description: "Redis key to store/update events, key is auto-created",
Type: "string",
},
config.HelpKV{
Key: target.RedisFormat,
Description: formatComment,
Type: "namespace*|access",
},
config.HelpKV{
Key: target.RedisPassword,
Description: "Redis server password",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: target.RedisQueueDir,
Description: queueDirComment,
Optional: true,
Type: "path",
},
config.HelpKV{
Key: target.RedisQueueLimit,
Description: queueLimitComment,
Optional: true,
Type: "number",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

625
internal/config/notify/legacy.go Normal file
View File

@@ -0,0 +1,625 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package notify
import (
"fmt"
"strconv"
"strings"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/event/target"
)
// SetNotifyKafka - helper for config migration from older config.
func SetNotifyKafka(s config.Config, name string, cfg target.KafkaArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyKafkaSubSys][name] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.KafkaBrokers,
Value: func() string {
var brokers []string
for _, broker := range cfg.Brokers {
brokers = append(brokers, broker.String())
}
return strings.Join(brokers, config.ValueSeparator)
}(),
},
config.KV{
Key: target.KafkaTopic,
Value: cfg.Topic,
},
config.KV{
Key: target.KafkaQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.KafkaClientTLSCert,
Value: cfg.TLS.ClientTLSCert,
},
config.KV{
Key: target.KafkaClientTLSKey,
Value: cfg.TLS.ClientTLSKey,
},
config.KV{
Key: target.KafkaQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.KafkaTLS,
Value: config.FormatBool(cfg.TLS.Enable),
},
config.KV{
Key: target.KafkaTLSSkipVerify,
Value: config.FormatBool(cfg.TLS.SkipVerify),
},
config.KV{
Key: target.KafkaTLSClientAuth,
Value: strconv.Itoa(int(cfg.TLS.ClientAuth)),
},
config.KV{
Key: target.KafkaSASL,
Value: config.FormatBool(cfg.SASL.Enable),
},
config.KV{
Key: target.KafkaSASLUsername,
Value: cfg.SASL.User,
},
config.KV{
Key: target.KafkaSASLPassword,
Value: cfg.SASL.Password,
},
}
return nil
}
// SetNotifyAMQP - helper for config migration from older config.
func SetNotifyAMQP(s config.Config, amqpName string, cfg target.AMQPArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyAMQPSubSys][amqpName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.AmqpURL,
Value: cfg.URL.String(),
},
config.KV{
Key: target.AmqpExchange,
Value: cfg.Exchange,
},
config.KV{
Key: target.AmqpRoutingKey,
Value: cfg.RoutingKey,
},
config.KV{
Key: target.AmqpExchangeType,
Value: cfg.ExchangeType,
},
config.KV{
Key: target.AmqpDeliveryMode,
Value: strconv.Itoa(int(cfg.DeliveryMode)),
},
config.KV{
Key: target.AmqpMandatory,
Value: config.FormatBool(cfg.Mandatory),
},
config.KV{
Key: target.AmqpInternal,
Value: config.FormatBool(cfg.Immediate),
},
config.KV{
Key: target.AmqpDurable,
Value: config.FormatBool(cfg.Durable),
},
config.KV{
Key: target.AmqpNoWait,
Value: config.FormatBool(cfg.NoWait),
},
config.KV{
Key: target.AmqpAutoDeleted,
Value: config.FormatBool(cfg.AutoDeleted),
},
config.KV{
Key: target.AmqpQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.AmqpQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
}
return nil
}
// SetNotifyES - helper for config migration from older config.
func SetNotifyES(s config.Config, esName string, cfg target.ElasticsearchArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyESSubSys][esName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.ElasticFormat,
Value: cfg.Format,
},
config.KV{
Key: target.ElasticURL,
Value: cfg.URL.String(),
},
config.KV{
Key: target.ElasticIndex,
Value: cfg.Index,
},
config.KV{
Key: target.ElasticQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.ElasticQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.ElasticUsername,
Value: cfg.Username,
},
config.KV{
Key: target.ElasticPassword,
Value: cfg.Password,
},
}
return nil
}
// SetNotifyRedis - helper for config migration from older config.
func SetNotifyRedis(s config.Config, redisName string, cfg target.RedisArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyRedisSubSys][redisName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.RedisFormat,
Value: cfg.Format,
},
config.KV{
Key: target.RedisAddress,
Value: cfg.Addr.String(),
},
config.KV{
Key: target.RedisPassword,
Value: cfg.Password,
},
config.KV{
Key: target.RedisKey,
Value: cfg.Key,
},
config.KV{
Key: target.RedisQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.RedisQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
}
return nil
}
// SetNotifyWebhook - helper for config migration from older config.
func SetNotifyWebhook(s config.Config, whName string, cfg target.WebhookArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyWebhookSubSys][whName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.WebhookEndpoint,
Value: cfg.Endpoint.String(),
},
config.KV{
Key: target.WebhookAuthToken,
Value: cfg.AuthToken,
},
config.KV{
Key: target.WebhookQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.WebhookQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.WebhookClientCert,
Value: cfg.ClientCert,
},
config.KV{
Key: target.WebhookClientKey,
Value: cfg.ClientKey,
},
}
return nil
}
// SetNotifyPostgres - helper for config migration from older config.
func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyPostgresSubSys][psqName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.PostgresFormat,
Value: cfg.Format,
},
config.KV{
Key: target.PostgresConnectionString,
Value: cfg.ConnectionString,
},
config.KV{
Key: target.PostgresTable,
Value: cfg.Table,
},
config.KV{
Key: target.PostgresHost,
Value: cfg.Host.String(),
},
config.KV{
Key: target.PostgresPort,
Value: cfg.Port,
},
config.KV{
Key: target.PostgresUsername,
Value: cfg.Username,
},
config.KV{
Key: target.PostgresPassword,
Value: cfg.Password,
},
config.KV{
Key: target.PostgresDatabase,
Value: cfg.Database,
},
config.KV{
Key: target.PostgresQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.PostgresQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.PostgresMaxOpenConnections,
Value: strconv.Itoa(cfg.MaxOpenConnections),
},
}
return nil
}
// SetNotifyNSQ - helper for config migration from older config.
func SetNotifyNSQ(s config.Config, nsqName string, cfg target.NSQArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyNSQSubSys][nsqName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.NSQAddress,
Value: cfg.NSQDAddress.String(),
},
config.KV{
Key: target.NSQTopic,
Value: cfg.Topic,
},
config.KV{
Key: target.NSQTLS,
Value: config.FormatBool(cfg.TLS.Enable),
},
config.KV{
Key: target.NSQTLSSkipVerify,
Value: config.FormatBool(cfg.TLS.SkipVerify),
},
config.KV{
Key: target.NSQQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.NSQQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
}
return nil
}
// SetNotifyNATS - helper for config migration from older config.
func SetNotifyNATS(s config.Config, natsName string, cfg target.NATSArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyNATSSubSys][natsName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.NATSAddress,
Value: cfg.Address.String(),
},
config.KV{
Key: target.NATSSubject,
Value: cfg.Subject,
},
config.KV{
Key: target.NATSUsername,
Value: cfg.Username,
},
config.KV{
Key: target.NATSPassword,
Value: cfg.Password,
},
config.KV{
Key: target.NATSToken,
Value: cfg.Token,
},
config.KV{
Key: target.NATSCertAuthority,
Value: cfg.CertAuthority,
},
config.KV{
Key: target.NATSClientCert,
Value: cfg.ClientCert,
},
config.KV{
Key: target.NATSClientKey,
Value: cfg.ClientKey,
},
config.KV{
Key: target.NATSTLS,
Value: config.FormatBool(cfg.Secure),
},
config.KV{
Key: target.NATSTLSSkipVerify,
Value: config.FormatBool(cfg.Secure),
},
config.KV{
Key: target.NATSPingInterval,
Value: strconv.FormatInt(cfg.PingInterval, 10),
},
config.KV{
Key: target.NATSQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.NATSQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.NATSStreaming,
Value: func() string {
if cfg.Streaming.Enable {
return config.EnableOn
}
return config.EnableOff
}(),
},
config.KV{
Key: target.NATSStreamingClusterID,
Value: cfg.Streaming.ClusterID,
},
config.KV{
Key: target.NATSStreamingAsync,
Value: config.FormatBool(cfg.Streaming.Async),
},
config.KV{
Key: target.NATSStreamingMaxPubAcksInFlight,
Value: strconv.Itoa(cfg.Streaming.MaxPubAcksInflight),
},
}
return nil
}
// SetNotifyMySQL - helper for config migration from older config.
func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyMySQLSubSys][sqlName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.MySQLFormat,
Value: cfg.Format,
},
config.KV{
Key: target.MySQLDSNString,
Value: cfg.DSN,
},
config.KV{
Key: target.MySQLTable,
Value: cfg.Table,
},
config.KV{
Key: target.MySQLHost,
Value: cfg.Host.String(),
},
config.KV{
Key: target.MySQLPort,
Value: cfg.Port,
},
config.KV{
Key: target.MySQLUsername,
Value: cfg.User,
},
config.KV{
Key: target.MySQLPassword,
Value: cfg.Password,
},
config.KV{
Key: target.MySQLDatabase,
Value: cfg.Database,
},
config.KV{
Key: target.MySQLQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.MySQLQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
config.KV{
Key: target.MySQLMaxOpenConnections,
Value: strconv.Itoa(cfg.MaxOpenConnections),
},
}
return nil
}
// SetNotifyMQTT - helper for config migration from older config.
func SetNotifyMQTT(s config.Config, mqttName string, cfg target.MQTTArgs) error {
if !cfg.Enable {
return nil
}
if err := cfg.Validate(); err != nil {
return err
}
s[config.NotifyMQTTSubSys][mqttName] = config.KVS{
config.KV{
Key: config.Enable,
Value: config.EnableOn,
},
config.KV{
Key: target.MqttBroker,
Value: cfg.Broker.String(),
},
config.KV{
Key: target.MqttTopic,
Value: cfg.Topic,
},
config.KV{
Key: target.MqttQoS,
Value: fmt.Sprintf("%d", cfg.QoS),
},
config.KV{
Key: target.MqttUsername,
Value: cfg.User,
},
config.KV{
Key: target.MqttPassword,
Value: cfg.Password,
},
config.KV{
Key: target.MqttReconnectInterval,
Value: cfg.MaxReconnectInterval.String(),
},
config.KV{
Key: target.MqttKeepAliveInterval,
Value: cfg.KeepAlive.String(),
},
config.KV{
Key: target.MqttQueueDir,
Value: cfg.QueueDir,
},
config.KV{
Key: target.MqttQueueLimit,
Value: strconv.Itoa(int(cfg.QueueLimit)),
},
}
return nil
}

1816
internal/config/notify/parse.go Normal file
View File

File diff suppressed because it is too large Load Diff

229
internal/config/policy/opa/config.go Normal file
View File

@@ -0,0 +1,229 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package opa
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"github.com/minio/minio/internal/config"
xnet "github.com/minio/minio/internal/net"
"github.com/minio/pkg/env"
iampolicy "github.com/minio/pkg/iam/policy"
)
// Env IAM OPA URL
const (
URL = "url"
AuthToken = "auth_token"
EnvPolicyOpaURL = "MINIO_POLICY_OPA_URL"
EnvPolicyOpaAuthToken = "MINIO_POLICY_OPA_AUTH_TOKEN"
)
// DefaultKVS - default config for OPA config
var (
DefaultKVS = config.KVS{
config.KV{
Key: URL,
Value: "",
},
config.KV{
Key: AuthToken,
Value: "",
},
}
)
// Args opa general purpose policy engine configuration.
type Args struct {
URL *xnet.URL `json:"url"`
AuthToken string `json:"authToken"`
Transport http.RoundTripper `json:"-"`
CloseRespFn func(r io.ReadCloser) `json:"-"`
}
// Validate - validate opa configuration params.
func (a *Args) Validate() error {
req, err := http.NewRequest(http.MethodPost, a.URL.String(), bytes.NewReader([]byte("")))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if a.AuthToken != "" {
req.Header.Set("Authorization", a.AuthToken)
}
client := &http.Client{Transport: a.Transport}
resp, err := client.Do(req)
if err != nil {
return err
}
defer a.CloseRespFn(resp.Body)
return nil
}
// UnmarshalJSON - decodes JSON data.
func (a *Args) UnmarshalJSON(data []byte) error {
// subtype to avoid recursive call to UnmarshalJSON()
type subArgs Args
var so subArgs
if err := json.Unmarshal(data, &so); err != nil {
return err
}
oa := Args(so)
if oa.URL == nil || oa.URL.String() == "" {
*a = oa
return nil
}
*a = oa
return nil
}
// Opa - implements opa policy agent calls.
type Opa struct {
args Args
client *http.Client
}
// Enabled returns if opa is enabled.
func Enabled(kvs config.KVS) bool {
return kvs.Get(URL) != ""
}
// LookupConfig lookup Opa from config, override with any ENVs.
func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.ReadCloser)) (Args, error) {
args := Args{}
if err := config.CheckValidKeys(config.PolicyOPASubSys, kv, DefaultKVS); err != nil {
return args, err
}
opaURL := env.Get(EnvIamOpaURL, "")
if opaURL == "" {
opaURL = env.Get(EnvPolicyOpaURL, kv.Get(URL))
if opaURL == "" {
return args, nil
}
}
authToken := env.Get(EnvIamOpaAuthToken, "")
if authToken == "" {
authToken = env.Get(EnvPolicyOpaAuthToken, kv.Get(AuthToken))
}
u, err := xnet.ParseHTTPURL(opaURL)
if err != nil {
return args, err
}
args = Args{
URL: u,
AuthToken: authToken,
Transport: transport,
CloseRespFn: closeRespFn,
}
if err = args.Validate(); err != nil {
return args, err
}
return args, nil
}
// New - initializes opa policy engine connector.
func New(args Args) *Opa {
// No opa args.
if args.URL == nil || args.URL.Scheme == "" && args.AuthToken == "" {
return nil
}
return &Opa{
args: args,
client: &http.Client{Transport: args.Transport},
}
}
// IsAllowed - checks given policy args is allowed to continue the REST API.
func (o *Opa) IsAllowed(args iampolicy.Args) (bool, error) {
if o == nil {
return false, nil
}
// OPA input
body := make(map[string]interface{})
body["input"] = args
inputBytes, err := json.Marshal(body)
if err != nil {
return false, err
}
req, err := http.NewRequest(http.MethodPost, o.args.URL.String(), bytes.NewReader(inputBytes))
if err != nil {
return false, err
}
req.Header.Set("Content-Type", "application/json")
if o.args.AuthToken != "" {
req.Header.Set("Authorization", o.args.AuthToken)
}
resp, err := o.client.Do(req)
if err != nil {
return false, err
}
defer o.args.CloseRespFn(resp.Body)
// Read the body to be saved later.
opaRespBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, err
}
// Handle large OPA responses when OPA URL is of
// form http://localhost:8181/v1/data/httpapi/authz
type opaResultAllow struct {
Result struct {
Allow bool `json:"allow"`
} `json:"result"`
}
// Handle simpler OPA responses when OPA URL is of
// form http://localhost:8181/v1/data/httpapi/authz/allow
type opaResult struct {
Result bool `json:"result"`
}
respBody := bytes.NewReader(opaRespBytes)
var result opaResult
if err = json.NewDecoder(respBody).Decode(&result); err != nil {
respBody.Seek(0, 0)
var resultAllow opaResultAllow
if err = json.NewDecoder(respBody).Decode(&resultAllow); err != nil {
return false, err
}
return resultAllow.Result.Allow, nil
}
return result.Result, nil
}

43
internal/config/policy/opa/help.go Normal file
View File

@@ -0,0 +1,43 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package opa
import "github.com/minio/minio/internal/config"
// Help template for OPA policy feature.
var (
Help = config.HelpKVS{
config.HelpKV{
Key: URL,
Description: `[DEPRECATED] OPA HTTP(s) endpoint e.g. "http://localhost:8181/v1/data/httpapi/authz/allow"`,
Type: "url",
},
config.HelpKV{
Key: AuthToken,
Description: "[DEPRECATED] authorization token for OPA endpoint",
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

46
internal/config/policy/opa/legacy.go Normal file
View File

@@ -0,0 +1,46 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package opa
import (
"github.com/minio/minio/internal/config"
)
// Legacy OPA envs
const (
EnvIamOpaURL = "MINIO_IAM_OPA_URL"
EnvIamOpaAuthToken = "MINIO_IAM_OPA_AUTHTOKEN"
)
// SetPolicyOPAConfig - One time migration code needed, for migrating from older config to new for PolicyOPAConfig.
func SetPolicyOPAConfig(s config.Config, opaArgs Args) {
if opaArgs.URL == nil || opaArgs.URL.String() == "" {
// Do not enable if opaArgs was empty.
return
}
s[config.PolicyOPASubSys][config.Default] = config.KVS{
config.KV{
Key: URL,
Value: opaArgs.URL.String(),
},
config.KV{
Key: AuthToken,
Value: opaArgs.AuthToken,
},
}
}

118
internal/config/scanner/scanner.go Normal file
View File

@@ -0,0 +1,118 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package scanner
import (
"strconv"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/env"
)
// Compression environment variables
const (
Delay = "delay"
MaxWait = "max_wait"
Cycle = "cycle"
EnvDelay = "MINIO_SCANNER_DELAY"
EnvCycle = "MINIO_SCANNER_CYCLE"
EnvDelayLegacy = "MINIO_CRAWLER_DELAY"
EnvMaxWait = "MINIO_SCANNER_MAX_WAIT"
EnvMaxWaitLegacy = "MINIO_CRAWLER_MAX_WAIT"
)
// Config represents the heal settings.
type Config struct {
// Delay is the sleep multiplier.
Delay float64 `json:"delay"`
// MaxWait is maximum wait time between operations
MaxWait time.Duration
// Cycle is the time.Duration between each scanner cycles
Cycle time.Duration
}
var (
// DefaultKVS - default KV config for heal settings
DefaultKVS = config.KVS{
config.KV{
Key: Delay,
Value: "10",
},
config.KV{
Key: MaxWait,
Value: "15s",
},
config.KV{
Key: Cycle,
Value: "1m",
},
}
// Help provides help for config values
Help = config.HelpKVS{
config.HelpKV{
Key: Delay,
Description: `scanner delay multiplier, defaults to '10.0'`,
Optional: true,
Type: "float",
},
config.HelpKV{
Key: MaxWait,
Description: `maximum wait time between operations, defaults to '15s'`,
Optional: true,
Type: "duration",
},
config.HelpKV{
Key: Cycle,
Description: `time duration between scanner cycles, defaults to '1m'`,
Optional: true,
Type: "duration",
},
}
)
// LookupConfig - lookup config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS) (cfg Config, err error) {
if err = config.CheckValidKeys(config.ScannerSubSys, kvs, DefaultKVS); err != nil {
return cfg, err
}
delay := env.Get(EnvDelayLegacy, "")
if delay == "" {
delay = env.Get(EnvDelay, kvs.Get(Delay))
}
cfg.Delay, err = strconv.ParseFloat(delay, 64)
if err != nil {
return cfg, err
}
maxWait := env.Get(EnvMaxWaitLegacy, "")
if maxWait == "" {
maxWait = env.Get(EnvMaxWait, kvs.Get(MaxWait))
}
cfg.MaxWait, err = time.ParseDuration(maxWait)
if err != nil {
return cfg, err
}
cfg.Cycle, err = time.ParseDuration(env.Get(EnvCycle, kvs.Get(Cycle)))
if err != nil {
return cfg, err
}
return cfg, nil
}

50
internal/config/storageclass/help.go Normal file
View File

@@ -0,0 +1,50 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storageclass
import "github.com/minio/minio/internal/config"
// Help template for storageclass feature.
var (
Help = config.HelpKVS{
config.HelpKV{
Key: ClassStandard,
Description: `set the parity count for default standard storage class e.g. "EC:4"`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: ClassRRS,
Description: `set the parity count for reduced redundancy storage class e.g. "EC:2"`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: ClassDMA,
Description: `enable O_DIRECT for both read and write, defaults to "write" e.g. "read+write"`,
Optional: true,
Type: "string",
},
config.HelpKV{
Key: config.Comment,
Description: config.DefaultComment,
Optional: true,
Type: "sentence",
},
}
)

40
internal/config/storageclass/legacy.go Normal file
View File

@@ -0,0 +1,40 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storageclass
import (
"github.com/minio/minio/internal/config"
)
// SetStorageClass - One time migration code needed, for migrating from older config to new for StorageClass.
func SetStorageClass(s config.Config, cfg Config) {
if len(cfg.Standard.String()) == 0 && len(cfg.RRS.String()) == 0 {
// Do not enable storage-class if no settings found.
return
}
s[config.StorageClassSubSys][config.Default] = config.KVS{
config.KV{
Key: ClassStandard,
Value: cfg.Standard.String(),
},
config.KV{
Key: ClassRRS,
Value: cfg.RRS.String(),
},
}
}

321
internal/config/storageclass/storage-class.go Normal file
View File

@@ -0,0 +1,321 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storageclass
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"sync"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/env"
)
// Standard constants for all storage class
const (
// Reduced redundancy storage class
RRS = "REDUCED_REDUNDANCY"
// Standard storage class
STANDARD = "STANDARD"
// DMA storage class
DMA = "DMA"
// Valid values are "write" and "read+write"
DMAWrite = "write"
DMAReadWrite = "read+write"
)
// Standard constats for config info storage class
const (
ClassStandard = "standard"
ClassRRS = "rrs"
ClassDMA = "dma"
// Reduced redundancy storage class environment variable
RRSEnv = "MINIO_STORAGE_CLASS_RRS"
// Standard storage class environment variable
StandardEnv = "MINIO_STORAGE_CLASS_STANDARD"
// DMA storage class environment variable
DMAEnv = "MINIO_STORAGE_CLASS_DMA"
// Supported storage class scheme is EC
schemePrefix = "EC"
// Min parity disks
minParityDisks = 2
// Default RRS parity is always minimum parity.
defaultRRSParity = minParityDisks
// Default DMA value
defaultDMA = DMAReadWrite
)
// DefaultKVS - default storage class config
var (
DefaultKVS = config.KVS{
config.KV{
Key: ClassStandard,
Value: "",
},
config.KV{
Key: ClassRRS,
Value: "EC:2",
},
config.KV{
Key: ClassDMA,
Value: defaultDMA,
},
}
)
// StorageClass - holds storage class information
type StorageClass struct {
Parity int
}
// ConfigLock is a global lock for storage-class config
var ConfigLock = sync.RWMutex{}
// Config storage class configuration
type Config struct {
Standard StorageClass `json:"standard"`
RRS StorageClass `json:"rrs"`
DMA string `json:"dma"`
}
// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
func (sCfg *Config) UnmarshalJSON(data []byte) error {
type Alias Config
aux := &struct {
*Alias
}{
Alias: (*Alias)(sCfg),
}
return json.Unmarshal(data, &aux)
}
// IsValid - returns true if input string is a valid
// storage class kind supported.
func IsValid(sc string) bool {
return sc == RRS || sc == STANDARD
}
// UnmarshalText unmarshals storage class from its textual form into
// storageClass structure.
func (sc *StorageClass) UnmarshalText(b []byte) error {
scStr := string(b)
if scStr == "" {
return nil
}
s, err := parseStorageClass(scStr)
if err != nil {
return err
}
sc.Parity = s.Parity
return nil
}
// MarshalText - marshals storage class string.
func (sc *StorageClass) MarshalText() ([]byte, error) {
if sc.Parity != 0 {
return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
}
return []byte{}, nil
}
func (sc *StorageClass) String() string {
if sc.Parity != 0 {
return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
}
return ""
}
// Parses given storageClassEnv and returns a storageClass structure.
// Supported Storage Class format is "Scheme:Number of parity disks".
// Currently only supported scheme is "EC".
func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) {
s := strings.Split(storageClassEnv, ":")
// only two elements allowed in the string - "scheme" and "number of parity disks"
if len(s) > 2 {
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too many sections in " + storageClassEnv)
} else if len(s) < 2 {
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too few sections in " + storageClassEnv)
}
// only allowed scheme is "EC"
if s[0] != schemePrefix {
return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Unsupported scheme " + s[0] + ". Supported scheme is EC")
}
// Number of parity disks should be integer
parityDisks, err := strconv.Atoi(s[1])
if err != nil {
return StorageClass{}, config.ErrStorageClassValue(err)
}
return StorageClass{
Parity: parityDisks,
}, nil
}
// ValidateParity validate standard storage class parity.
func ValidateParity(ssParity, setDriveCount int) error {
// SS parity disks should be greater than or equal to minParityDisks.
// Parity below minParityDisks is not supported.
if ssParity > 0 && ssParity < minParityDisks {
return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
ssParity, minParityDisks)
}
if ssParity > setDriveCount/2 {
return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, setDriveCount/2)
}
return nil
}
// Validates the parity disks.
func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
// SS parity disks should be greater than or equal to minParityDisks.
// Parity below minParityDisks is not supported.
if ssParity > 0 && ssParity < minParityDisks {
return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
ssParity, minParityDisks)
}
// RRS parity disks should be greater than or equal to minParityDisks.
// Parity below minParityDisks is not supported.
if rrsParity > 0 && rrsParity < minParityDisks {
return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDisks)
}
if ssParity > setDriveCount/2 {
return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, setDriveCount/2)
}
if rrsParity > setDriveCount/2 {
return fmt.Errorf("Reduced redundancy storage class parity %d should be less than or equal to %d", rrsParity, setDriveCount/2)
}
if ssParity > 0 && rrsParity > 0 {
if ssParity > 0 && ssParity < rrsParity {
return fmt.Errorf("Standard storage class parity disks %d should be greater than or equal to Reduced redundancy storage class parity disks %d", ssParity, rrsParity)
}
}
return nil
}
// GetParityForSC - Returns the data and parity drive count based on storage class
// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and
// MINIO_STORAGE_CLASS_STANDARD or server config fields corresponding values are
// returned.
//
// -- if input storage class is empty then standard is assumed
// -- if input is RRS but RRS is not configured default '2' parity
// for RRS is assumed
// -- if input is STANDARD but STANDARD is not configured '0' parity
// is returned, the caller is expected to choose the right parity
// at that point.
func (sCfg Config) GetParityForSC(sc string) (parity int) {
ConfigLock.RLock()
defer ConfigLock.RUnlock()
switch strings.TrimSpace(sc) {
case RRS:
// set the rrs parity if available
if sCfg.RRS.Parity == 0 {
return defaultRRSParity
}
return sCfg.RRS.Parity
default:
return sCfg.Standard.Parity
}
}
// Update update storage-class with new config
func (sCfg Config) Update(newCfg Config) {
ConfigLock.Lock()
defer ConfigLock.Unlock()
sCfg.RRS = newCfg.RRS
sCfg.DMA = newCfg.DMA
sCfg.Standard = newCfg.Standard
}
// GetDMA - returns DMA configuration.
func (sCfg Config) GetDMA() string {
ConfigLock.RLock()
defer ConfigLock.RUnlock()
return sCfg.DMA
}
// Enabled returns if etcd is enabled.
func Enabled(kvs config.KVS) bool {
ssc := kvs.Get(ClassStandard)
rrsc := kvs.Get(ClassRRS)
return ssc != "" || rrsc != ""
}
// LookupConfig - lookup storage class config and override with valid environment settings if any.
func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
cfg = Config{}
if err = config.CheckValidKeys(config.StorageClassSubSys, kvs, DefaultKVS); err != nil {
return Config{}, err
}
ssc := env.Get(StandardEnv, kvs.Get(ClassStandard))
rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS))
dma := env.Get(DMAEnv, kvs.Get(ClassDMA))
// Check for environment variables and parse into storageClass struct
if ssc != "" {
cfg.Standard, err = parseStorageClass(ssc)
if err != nil {
return Config{}, err
}
}
if rrsc != "" {
cfg.RRS, err = parseStorageClass(rrsc)
if err != nil {
return Config{}, err
}
}
if cfg.RRS.Parity == 0 {
cfg.RRS.Parity = defaultRRSParity
}
if dma == "" {
dma = defaultDMA
}
if dma != DMAReadWrite && dma != DMAWrite {
return Config{}, errors.New(`valid dma values are "read-write" and "write"`)
}
cfg.DMA = dma
// Validation is done after parsing both the storage classes. This is needed because we need one
// storage class value to deduce the correct value of the other storage class.
if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil {
return Config{}, err
}
return cfg, nil
}

164
internal/config/storageclass/storage-class_test.go Normal file
View File

@@ -0,0 +1,164 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package storageclass
import (
"errors"
"reflect"
"testing"
)
func TestParseStorageClass(t *testing.T) {
tests := []struct {
storageClassEnv string
wantSc StorageClass
expectedError error
}{
{"EC:3", StorageClass{
Parity: 3},
nil},
{"EC:4", StorageClass{
Parity: 4},
nil},
{"AB:4", StorageClass{
Parity: 4},
errors.New("Unsupported scheme AB. Supported scheme is EC")},
{"EC:4:5", StorageClass{
Parity: 4},
errors.New("Too many sections in EC:4:5")},
{"EC:A", StorageClass{
Parity: 4},
errors.New(`strconv.Atoi: parsing "A": invalid syntax`)},
{"AB", StorageClass{
Parity: 4},
errors.New("Too few sections in AB")},
}
for i, tt := range tests {
gotSc, err := parseStorageClass(tt.storageClassEnv)
if err != nil && tt.expectedError == nil {
t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
return
}
if err == nil && tt.expectedError != nil {
t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
return
}
if tt.expectedError == nil && !reflect.DeepEqual(gotSc, tt.wantSc) {
t.Errorf("Test %d, Expected %v, got %v", i+1, tt.wantSc, gotSc)
return
}
if tt.expectedError != nil && err.Error() != tt.expectedError.Error() {
t.Errorf("Test %d, Expected `%v`, got `%v`", i+1, tt.expectedError, err)
}
}
}
func TestValidateParity(t *testing.T) {
tests := []struct {
rrsParity int
ssParity int
success bool
setDriveCount int
}{
{2, 4, true, 16},
{3, 3, true, 16},
{0, 0, true, 16},
{1, 4, false, 16},
{7, 6, false, 16},
{9, 0, false, 16},
{9, 9, false, 16},
{2, 9, false, 16},
{9, 2, false, 16},
}
for i, tt := range tests {
err := validateParity(tt.ssParity, tt.rrsParity, tt.setDriveCount)
if err != nil && tt.success {
t.Errorf("Test %d, Expected success, got %s", i+1, err)
}
if err == nil && !tt.success {
t.Errorf("Test %d, Expected failure, got success", i+1)
}
}
}
func TestParityCount(t *testing.T) {
tests := []struct {
sc string
disksCount int
expectedData int
expectedParity int
}{
{RRS, 16, 14, 2},
{STANDARD, 16, 8, 8},
{"", 16, 8, 8},
{RRS, 16, 9, 7},
{STANDARD, 16, 10, 6},
{"", 16, 9, 7},
}
for i, tt := range tests {
scfg := Config{
Standard: StorageClass{
Parity: 8,
},
RRS: StorageClass{
Parity: 0,
},
}
// Set env var for test case 4
if i+1 == 4 {
scfg.RRS.Parity = 7
}
// Set env var for test case 5
if i+1 == 5 {
scfg.Standard.Parity = 6
}
// Set env var for test case 6
if i+1 == 6 {
scfg.Standard.Parity = 7
}
parity := scfg.GetParityForSC(tt.sc)
if (tt.disksCount - parity) != tt.expectedData {
t.Errorf("Test %d, Expected data disks %d, got %d", i+1, tt.expectedData, tt.disksCount-parity)
continue
}
if parity != tt.expectedParity {
t.Errorf("Test %d, Expected parity disks %d, got %d", i+1, tt.expectedParity, parity)
}
}
}
// Test IsValid method with valid and invalid inputs
func TestIsValidStorageClassKind(t *testing.T) {
tests := []struct {
sc string
want bool
}{
{"STANDARD", true},
{"REDUCED_REDUNDANCY", true},
{"", false},
{"INVALID", false},
{"123", false},
{"MINIO_STORAGE_CLASS_RRS", false},
{"MINIO_STORAGE_CLASS_STANDARD", false},
}
for i, tt := range tests {
if got := IsValid(tt.sc); got != tt.want {
t.Errorf("Test %d, Expected Storage Class to be %t, got %t", i+1, tt.want, got)
}
}
}