MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Migrate config.json from config-dir to backend

Browse Source 
This PR is the first set of changes to move the config
to the backend, the changes use the existing `config.json`
allows it to be migrated such that we can save it in on
backend disks.

In future releases, we will slowly migrate out of the
current architecture.

Fixes #6182
This commit is contained in:
Harshavardhana 2018-07-23 15:36:10 -07:00
parent 380524ae27
commit 1fb2e9ef95
46 changed files with 818 additions and 845 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -22,3 +22,4 @@ parts/
prime/
stage/
.sia_temp/
config.json

4
buildscripts/verify-build.sh
View File

@ -71,6 +71,8 @@ function start_minio_erasure_sets()
function start_minio_dist_erasure_sets()
{
declare -a minio_pids
export MINIO_ACCESS_KEY=$ACCESS_KEY
export MINIO_SECRET_KEY=$SECRET_KEY
"${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
minio_pids[0]=$!
"${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
@ -99,6 +101,8 @@ function start_minio_dist_erasure_sets()
function start_minio_dist_erasure()
{
declare -a minio_pids
export MINIO_ACCESS_KEY=$ACCESS_KEY
export MINIO_SECRET_KEY=$SECRET_KEY
"${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
minio_pids[0]=$!
"${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &

45
cmd/admin-handlers.go
View File

@ -25,6 +25,7 @@ import (
"io"
"net/http"
"net/url"
"path"
"sync"
"time"
@ -462,14 +463,22 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
// Get config.json - in distributed mode, the configuration
// occurring on a quorum of the servers is returned.
configBytes, err := getPeerConfig(globalAdminPeers)
configData, err := getPeerConfig(globalAdminPeers)
if err != nil {
logger.LogIf(context.Background(), err)
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
return
}
writeSuccessResponseJSON(w, configBytes)
password := globalServerConfig.GetCredential().SecretKey
econfigData, err := madmin.EncryptServerConfigData(password, configData)
if err != nil {
logger.LogIf(context.Background(), err)
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
return
}
writeSuccessResponseJSON(w, econfigData)
}
// toAdminAPIErrCode - converts errXLWriteQuorum error to admin API
@ -572,7 +581,13 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
return
}
configBytes := configBuf[:n]
password := globalServerConfig.GetCredential().SecretKey
configBytes, err := madmin.DecryptServerConfigData(password, bytes.NewReader(configBuf[:n]))
if err != nil {
logger.LogIf(ctx, err)
writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
return
}
// Validate JSON provided in the request body: check the
// client has not sent JSON objects with duplicate keys.
@ -669,8 +684,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
// Decode request body
var req madmin.SetCredsReq
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
logger.LogIf(context.Background(), err)
writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
return
@ -697,13 +711,28 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
// Update local credentials in memory.
globalServerConfig.SetCredential(creds)
if err = globalServerConfig.Save(getConfigFile()); err != nil {
writeErrorResponseJSON(w, ErrInternalError, r.URL)
// Construct path to config.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, minioConfigFile)
transactionConfigFile := configFile + ".transaction"
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
// and configFile, take a transaction lock to avoid race.
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
if err = objLock.GetLock(globalOperationTimeout); err != nil {
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
return
}
if err = saveServerConfig(newObjectLayerFn(), globalServerConfig); err != nil {
objLock.Unlock()
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
return
}
objLock.Unlock()
// Notify all other Minio peers to update credentials
for host, err := range globalNotificationSys.SetCredentials(creds) {
for host, err := range globalNotificationSys.LoadCredentials() {
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", host.String())
ctx := logger.SetReqInfo(context.Background(), reqInfo)
logger.LogIf(ctx, err)

30
cmd/admin-handlers_test.go
View File

@ -193,17 +193,17 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
// reset global variables to start afresh.
resetTestGlobals()
// Initialize minio server config.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
return nil, err
}
// Initializing objectLayer for HealFormatHandler.
objLayer, xlDirs, xlErr := initTestXLObjLayer()
if xlErr != nil {
return nil, xlErr
}
// Initialize minio server config.
if err := newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
return nil, err
}
// Initialize boot time
globalBootTime = UTCNow()
@ -230,7 +230,6 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
registerAdminRouter(adminRouter)
return &adminXLTestBed{
configPath: rootPath,
xlDirs: xlDirs,
objLayer: objLayer,
router: adminRouter,
@ -240,7 +239,6 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
// TearDown - method that resets the test bed for subsequent unit
// tests to start afresh.
func (atb *adminXLTestBed) TearDown() {
os.RemoveAll(atb.configPath)
removeRoots(atb.xlDirs)
resetTestGlobals()
}
@ -680,8 +678,14 @@ func TestSetConfigHandler(t *testing.T) {
queryVal := url.Values{}
queryVal.Set("config", "")
password := globalServerConfig.GetCredential().SecretKey
econfigJSON, err := madmin.EncryptServerConfigData(password, configJSON)
if err != nil {
t.Fatal(err)
}
req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
int64(len(configJSON)), bytes.NewReader(configJSON))
int64(len(econfigJSON)), bytes.NewReader(econfigJSON))
if err != nil {
t.Fatalf("Failed to construct set-config object request - %v", err)
}
@ -724,7 +728,7 @@ func TestSetConfigHandler(t *testing.T) {
// Check that a config with duplicate keys in an object return
// error.
{
invalidCfg := append(configJSON[:len(configJSON)-1], []byte(`, "version": "15"}`)...)
invalidCfg := append(econfigJSON[:len(econfigJSON)-1], []byte(`, "version": "15"}`)...)
req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
if err != nil {
@ -823,11 +827,15 @@ func TestToAdminAPIErr(t *testing.T) {
}
func TestWriteSetConfigResponse(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
testCases := []struct {
status bool
errs []error

9
cmd/api-router.go
View File

@ -20,7 +20,6 @@ import (
"net/http"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/logger"
)
// objectAPIHandler implements and provides http handlers for S3 API.
@ -31,14 +30,6 @@ type objectAPIHandlers struct {
// registerAPIRouter - registers S3 compatible APIs.
func registerAPIRouter(router *mux.Router) {
var err error
var cacheConfig = globalServerConfig.GetCacheConfig()
if len(cacheConfig.Drives) > 0 {
// initialize the new disk cache objects.
globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
logger.FatalIf(err, "Unable to initialize disk caching")
}
// Initialize API.
api := objectAPIHandlers{
ObjectAPI: newObjectLayerFn,

15
cmd/auth-handler_test.go
View File

@ -344,11 +344,14 @@ func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int6
// Tests is requested authenticated function, tests replies for s3 errors.
func TestIsReqAuthenticated(t *testing.T) {
path, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
defer os.RemoveAll(path)
creds, err := auth.CreateCredentials("myuser", "mypassword")
if err != nil {
@ -384,11 +387,15 @@ func TestIsReqAuthenticated(t *testing.T) {
}
}
func TestCheckAdminRequestAuthType(t *testing.T) {
path, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
defer os.RemoveAll(path)
creds, err := auth.CreateCredentials("myuser", "mypassword")
if err != nil {

60
cmd/benchmark-utils_test.go
View File

@ -22,7 +22,6 @@ import (
"io/ioutil"
"math"
"math/rand"
"os"
"strconv"
"testing"
@ -138,12 +137,6 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -151,18 +144,13 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
}
// cleaning up the backend by removing all the directories and files created on function return.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runPutObjectPartBenchmark(b, objLayer, objSize)
}
// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -170,18 +158,13 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
}
// cleaning up the backend by removing all the directories and files created on function return.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runPutObjectBenchmark(b, objLayer, objSize)
}
// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -189,6 +172,7 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
}
// cleaning up the backend by removing all the directories and files created on function return.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runPutObjectBenchmarkParallel(b, objLayer, objSize)
}
@ -196,16 +180,10 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
// Benchmark utility functions for ObjectLayer.GetObject().
// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
if err != nil {
b.Fatal(err)
}
@ -269,12 +247,6 @@ func generateBytesData(size int) []byte {
// creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -282,18 +254,13 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
}
// cleaning up the backend by removing all the directories and files created.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runGetObjectBenchmark(b, objLayer, objSize)
}
// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// create a temp XL/FS backend.
objLayer, disks, err := prepareBenchmarkBackend(instanceType)
if err != nil {
@ -301,6 +268,7 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
}
// cleaning up the backend by removing all the directories and files created.
defer removeRoots(disks)
// uses *testing.B and the object Layer to run the benchmark.
runGetObjectBenchmarkParallel(b, objLayer, objSize)
}
@ -308,16 +276,10 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
// Parallel benchmark utility functions for ObjectLayer.PutObject().
// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
if err != nil {
b.Fatal(err)
}
@ -359,16 +321,10 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
// Parallel benchmark utility functions for ObjectLayer.GetObject().
// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
if err != nil {
b.Fatal(err)
}

39
cmd/common-main.go
View File

@ -17,7 +17,6 @@
package cmd
import (
"context"
"errors"
"net"
"os"
@ -48,39 +47,6 @@ func checkUpdate(mode string) {
}
}
// Initialize and load config from remote etcd or local config directory
func initConfig() {
if globalEtcdClient != nil {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
resp, err := globalEtcdClient.Get(ctx, getConfigFile())
cancel()
// This means there are no entries in etcd with config file
// So create a new config
if err == nil && resp.Count == 0 {
logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
logger.Info("Created minio configuration file successfully at %v", globalEtcdClient.Endpoints())
} else {
// This means there is an entry in etcd, update it if required and proceed
if err == nil && resp.Count > 0 {
logger.FatalIf(migrateConfig(), "Config migration failed.")
logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
} else {
logger.FatalIf(err, "Unable to load config version: '%s'.", serverConfigVersion)
}
}
return
}
if isFile(getConfigFile()) {
logger.FatalIf(migrateConfig(), "Config migration failed")
logger.FatalIf(loadConfig(), "Unable to load the configuration file")
} else {
// Config file does not exist, we create it fresh and return upon success.
logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
logger.Info("Created minio configuration file successfully at " + getConfigDir())
}
}
// Load logger targets based on user's configuration
func loadLoggers() {
if globalServerConfig.Logger.Console.Enabled {
@ -148,6 +114,11 @@ func handleCommonEnvVars() {
globalActiveCred = cred
}
// In distributed setup users need to set ENVs always.
if !globalIsEnvCreds && globalIsDistXL {
logger.Fatal(uiErrEnvCredentialsMissingServer(nil), "Unable to start distributed server mode")
}
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
browserFlag, err := ParseBoolFlag(browser)
if err != nil {

229
cmd/config-current.go
View File

@ -29,7 +29,6 @@ import (
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/event/target"
"github.com/minio/minio/pkg/quick"
)
// Steps to move from version N to version N+1
@ -69,6 +68,10 @@ func (s *serverConfig) GetRegion() string {
// SetCredential sets new credential and returns the previous credential.
func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
if creds.IsValid() && globalActiveCred.IsValid() {
globalActiveCred = creds
}
// Save previous credential.
prevCred = s.Credential
@ -81,6 +84,9 @@ func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Cred
// GetCredentials get current credentials.
func (s *serverConfig) GetCredential() auth.Credentials {
if globalActiveCred.IsValid() {
return globalActiveCred
}
return s.Credential
}
@ -208,19 +214,35 @@ func (s *serverConfig) Validate() error {
return nil
}
// Save config file to corresponding backend
func Save(configFile string, data interface{}) error {
return quick.SaveConfig(data, configFile, globalEtcdClient)
func (s *serverConfig) loadFromEnvs() {
// If env is set override the credentials from config file.
if globalIsEnvCreds {
s.SetCredential(globalActiveCred)
}
// Load config from backend
func Load(configFile string, data interface{}) (quick.Config, error) {
return quick.LoadConfig(configFile, globalEtcdClient, data)
if globalIsEnvBrowser {
s.SetBrowser(globalIsBrowserEnabled)
}
// GetVersion gets config version from backend
func GetVersion(configFile string) (string, error) {
return quick.GetVersion(configFile, globalEtcdClient)
if globalIsEnvWORM {
s.SetWorm(globalWORMEnabled)
}
if globalIsEnvRegion {
s.SetRegion(globalServerRegion)
}
if globalIsEnvDomainName {
s.Domain = globalDomainName
}
if globalIsStorageClass {
s.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
}
if globalIsDiskCacheEnabled {
s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
}
}
// Returns the string describing a difference with the given
@ -327,153 +349,82 @@ func newServerConfig() *serverConfig {
return srvCfg
}
// newConfig - initialize a new server config, saves env parameters if
// found, otherwise use default parameters
func newConfig() error {
// Initialize server config.
srvCfg, err := newQuickConfig(newServerConfig())
if err != nil {
return err
}
// If env is set override the credentials from config file.
if globalIsEnvCreds {
srvCfg.SetCredential(globalActiveCred)
}
if globalIsEnvBrowser {
srvCfg.SetBrowser(globalIsBrowserEnabled)
}
if globalIsEnvWORM {
srvCfg.SetWorm(globalWORMEnabled)
}
if globalIsEnvRegion {
srvCfg.SetRegion(globalServerRegion)
}
if globalIsEnvDomainName {
srvCfg.Domain = globalDomainName
}
if globalIsStorageClass {
srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
}
if globalIsDiskCacheEnabled {
srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
}
// hold the mutex lock before a new config is assigned.
// Save the new config globally.
// unlock the mutex.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
globalServerConfigMu.Unlock()
// Save config into file.
return Save(getConfigFile(), globalServerConfig)
}
// newQuickConfig - initialize a new server config, with an allocated
// quick.Config interface.
func newQuickConfig(srvCfg *serverConfig) (*serverConfig, error) {
qcfg, err := quick.NewConfig(srvCfg, globalEtcdClient)
if err != nil {
return nil, err
}
srvCfg.Config = qcfg
return srvCfg, nil
}
// getValidConfig - returns valid server configuration
func getValidConfig() (*serverConfig, error) {
srvCfg := &serverConfig{
Region: globalMinioDefaultRegion,
Browser: true,
}
var err error
srvCfg, err = newQuickConfig(srvCfg)
if err != nil {
return nil, err
}
configFile := getConfigFile()
if err = srvCfg.Load(configFile); err != nil {
return nil, err
}
if err = srvCfg.Validate(); err != nil {
return nil, err
}
return srvCfg, nil
}
// loadConfig - loads a new config from disk, overrides params from env
// if found and valid
func loadConfig() error {
srvCfg, err := getValidConfig()
if err != nil {
return uiErrInvalidConfig(nil).Msg(err.Error())
}
// If env is set override the credentials from config file.
if globalIsEnvCreds {
srvCfg.SetCredential(globalActiveCred)
}
if globalIsEnvBrowser {
srvCfg.SetBrowser(globalIsBrowserEnabled)
}
if globalIsEnvRegion {
srvCfg.SetRegion(globalServerRegion)
}
if globalIsEnvDomainName {
srvCfg.Domain = globalDomainName
}
if globalIsStorageClass {
srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
}
if globalIsDiskCacheEnabled {
srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
}
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
func (s *serverConfig) loadToCachedConfigs() {
if !globalIsEnvCreds {
globalActiveCred = globalServerConfig.GetCredential()
globalActiveCred = s.GetCredential()
}
if !globalIsEnvBrowser {
globalIsBrowserEnabled = globalServerConfig.GetBrowser()
globalIsBrowserEnabled = s.GetBrowser()
}
if !globalIsEnvWORM {
globalWORMEnabled = globalServerConfig.GetWorm()
globalWORMEnabled = s.GetWorm()
}
if !globalIsEnvRegion {
globalServerRegion = globalServerConfig.GetRegion()
globalServerRegion = s.GetRegion()
}
if !globalIsEnvDomainName {
globalDomainName = globalServerConfig.Domain
globalDomainName = s.Domain
}
if !globalIsStorageClass {
globalStandardStorageClass, globalRRStorageClass = globalServerConfig.GetStorageClass()
globalStandardStorageClass, globalRRStorageClass = s.GetStorageClass()
}
if !globalIsDiskCacheEnabled {
cacheConf := globalServerConfig.GetCacheConfig()
cacheConf := s.GetCacheConfig()
globalCacheDrives = cacheConf.Drives
globalCacheExcludes = cacheConf.Exclude
globalCacheExpiry = cacheConf.Expiry
globalCacheMaxUse = cacheConf.MaxUse
}
}
// newConfig - initialize a new server config, saves env parameters if
// found, otherwise use default parameters
func newConfig(objAPI ObjectLayer) error {
// Initialize server config.
srvCfg := newServerConfig()
// Override any values from ENVs.
srvCfg.loadFromEnvs()
// Load values to cached global values.
srvCfg.loadToCachedConfigs()
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
globalServerConfigMu.Unlock()
// Save config into file.
return saveServerConfig(objAPI, globalServerConfig)
}
// getValidConfig - returns valid server configuration
func getValidConfig(objAPI ObjectLayer) (*serverConfig, error) {
srvCfg, err := readServerConfig(context.Background(), objAPI)
if err != nil {
return nil, err
}
return srvCfg, srvCfg.Validate()
}
// loadConfig - loads a new config from disk, overrides params from env
// if found and valid
func loadConfig(objAPI ObjectLayer) error {
srvCfg, err := getValidConfig(objAPI)
if err != nil {
return uiErrInvalidConfig(nil).Msg(err.Error())
}
// Override any values from ENVs.
srvCfg.loadFromEnvs()
// Load values to cached global values.
srvCfg.loadToCachedConfigs()
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
globalServerConfigMu.Unlock()
return nil

65
cmd/config-current_test.go
View File

@ -17,9 +17,8 @@
package cmd
import (
"io/ioutil"
"os"
"path/filepath"
"path"
"testing"
"github.com/minio/minio/pkg/auth"
@ -27,12 +26,15 @@ import (
)
func TestServerConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("Init Test config failed")
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
if globalServerConfig.GetRegion() != globalMinioDefaultRegion {
t.Errorf("Expecting region `us-east-1` found %s", globalServerConfig.GetRegion())
@ -49,16 +51,12 @@ func TestServerConfig(t *testing.T) {
t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
}
// Attempt to save.
if err := globalServerConfig.Save(getConfigFile()); err != nil {
if err := saveServerConfig(objLayer, globalServerConfig); err != nil {
t.Fatalf("Unable to save updated config file %s", err)
}
// Do this only once here.
setConfigDir(rootPath)
// Initialize server config.
if err := loadConfig(); err != nil {
if err := loadConfig(objLayer); err != nil {
t.Fatalf("Unable to initialize from updated config file %s", err)
}
}
@ -82,23 +80,25 @@ func TestServerConfigWithEnvs(t *testing.T) {
defer resetGlobalIsEnvs()
// Get test root.
rootPath, err := getTestRoot()
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Error(err)
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("Init Test config failed")
}
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
serverHandleEnvVars()
// Do this only once here.
setConfigDir(rootPath)
// Init config
initConfig()
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
// Check if serverConfig has
if globalServerConfig.GetBrowser() {
t.Errorf("Expecting browser is set to false found %v", globalServerConfig.GetBrowser())
@ -127,16 +127,17 @@ func TestServerConfigWithEnvs(t *testing.T) {
// Tests config validator..
func TestValidateConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("Init Test config failed")
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
configPath := filepath.Join(rootPath, minioConfigFile)
configPath := path.Join(minioConfigPrefix, minioConfigFile)
v := serverConfigVersion
testCases := []struct {
@ -226,14 +227,14 @@ func TestValidateConfig(t *testing.T) {
}
for i, testCase := range testCases {
if werr := ioutil.WriteFile(configPath, []byte(testCase.configData), 0700); werr != nil {
t.Fatal(werr)
if err = saveConfig(objLayer, configPath, []byte(testCase.configData)); err != nil {
t.Fatal(err)
}
_, verr := getValidConfig()
if testCase.shouldPass && verr != nil {
t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
_, err = getValidConfig(objLayer)
if testCase.shouldPass && err != nil {
t.Errorf("Test %d, should pass but it failed with err = %v", i+1, err)
}
if !testCase.shouldPass && verr == nil {
if !testCase.shouldPass && err == nil {
t.Errorf("Test %d, should fail but it succeeded.", i+1)
}
}

3
cmd/config-dir.go
View File

@ -28,9 +28,6 @@ const (
// Default minio configuration directory where below configuration files/directories are stored.
defaultMinioConfigDir = ".minio"
// Minio configuration file.
minioConfigFile = "config.json"
// Directory contains below files/directories for HTTPS configuration.
certsDir = "certs"

18
cmd/config-migrate.go
View File

@ -33,6 +33,21 @@ import (
// DO NOT EDIT following message template, please open a github issue to discuss instead.
var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully."
// Save config file to corresponding backend
func Save(configFile string, data interface{}) error {
return quick.SaveConfig(data, configFile, globalEtcdClient)
}
// Load config from backend
func Load(configFile string, data interface{}) (quick.Config, error) {
return quick.LoadConfig(configFile, globalEtcdClient, data)
}
// GetVersion gets config version from backend
func GetVersion(configFile string) (string, error) {
return quick.GetVersion(configFile, globalEtcdClient)
}
// Migrates all config versions from "1" to "18".
func migrateConfig() error {
// Purge all configs with version '1',
@ -45,6 +60,9 @@ func migrateConfig() error {
// Load only config version information.
version, err := GetVersion(getConfigFile())
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}

66
cmd/config-migrate_test.go
View File

@ -25,15 +25,26 @@ import (
// Test if config v1 is purged
func TestServerConfigMigrateV1(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
err = newTestConfig(globalMinioDefaultRegion, objLayer)
if err != nil {
t.Fatalf("Init Test config failed")
}
// remove the root directory after the test ends.
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
// Create a V1 config json file and store it
configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
configPath := rootPath + "/fsUsers.json"
@ -45,13 +56,14 @@ func TestServerConfigMigrateV1(t *testing.T) {
if err := migrateConfig(); err != nil {
t.Fatal("Unexpected error: ", err)
}
// Check if config v1 is removed from filesystem
if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
t.Fatal("Config V1 file is not purged")
}
// Initialize server config and check again if everything is fine
if err := loadConfig(); err != nil {
if err := loadConfig(objLayer); err != nil {
t.Fatalf("Unable to initialize from updated config file %s", err)
}
}
@ -59,20 +71,13 @@ func TestServerConfigMigrateV1(t *testing.T) {
// Test if all migrate code returns nil when config file does not
// exist
func TestServerConfigMigrateInexistentConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
configPath := rootPath + "/" + minioConfigFile
// Remove config file
if err := os.Remove(configPath); err != nil {
t.Fatal("Unexpected error: ", err)
}
if err := migrateV2ToV3(); err != nil {
t.Fatal("migrate v2 to v3 should succeed when no config file is found")
@ -153,14 +158,23 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
// Test if a config migration from v2 to v27 is successfully done
func TestServerConfigMigrateV2toV27(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
globalObjLayerMutex.Unlock()
configPath := rootPath + "/" + minioConfigFile
// Create a corrupted config file
@ -186,8 +200,12 @@ func TestServerConfigMigrateV2toV27(t *testing.T) {
t.Fatal("Unexpected error: ", err)
}
if err := migrateConfigToMinioSys(); err != nil {
t.Fatal("Unexpected error: ", err)
}
// Initialize server config and check again if everything is fine
if err := loadConfig(); err != nil {
if err := loadConfig(newObjectLayerFn()); err != nil {
t.Fatalf("Unable to initialize from updated config file %s", err)
}
@ -208,13 +226,11 @@ func TestServerConfigMigrateV2toV27(t *testing.T) {
// Test if all migrate code returns error with corrupted config files
func TestServerConfigMigrateFaultyConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
configPath := rootPath + "/" + minioConfigFile
@ -303,13 +319,11 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
// Test if all migrate code returns error with corrupted config files
func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
setConfigDir(rootPath)
configPath := rootPath + "/" + minioConfigFile

261
cmd/config.go Normal file
View File

@ -0,0 +1,261 @@
/*
* Minio Cloud Storage, (C) 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"io/ioutil"
"path"
"runtime"
"time"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/quick"
)
const (
minioConfigPrefix = "config"
// Minio configuration file.
minioConfigFile = "config.json"
)
func saveServerConfig(objAPI ObjectLayer, config *serverConfig) error {
if err := quick.CheckData(config); err != nil {
return err
}
data, err := json.Marshal(config)
if err != nil {
return err
}
configFile := path.Join(minioConfigPrefix, minioConfigFile)
if globalEtcdClient != nil {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
_, err := globalEtcdClient.Put(ctx, configFile, string(data))
defer cancel()
return err
}
return saveConfig(objAPI, configFile, data)
}
func readConfigEtcd(configFile string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
resp, err := globalEtcdClient.Get(ctx, configFile)
defer cancel()
if err != nil {
return nil, err
}
if resp.Count == 0 {
return nil, errConfigNotFound
}
for _, ev := range resp.Kvs {
if string(ev.Key) == configFile {
return ev.Value, nil
}
}
return nil, errConfigNotFound
}
func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
var configData []byte
var err error
configFile := path.Join(minioConfigPrefix, minioConfigFile)
if globalEtcdClient != nil {
configData, err = readConfigEtcd(configFile)
} else {
var reader io.Reader
reader, err = readConfig(ctx, objAPI, configFile)
if err != nil {
return nil, err
}
configData, err = ioutil.ReadAll(reader)
}
if err != nil {
return nil, err
}
if runtime.GOOS == "windows" {
configData = bytes.Replace(configData, []byte("\r\n"), []byte("\n"), -1)
}
if err = quick.CheckDuplicateKeys(string(configData)); err != nil {
return nil, err
}
var config = &serverConfig{}
if err := json.Unmarshal(configData, config); err != nil {
return nil, err
}
if err := quick.CheckData(config); err != nil {
return nil, err
}
return config, nil
}
func checkServerConfigEtcd(configFile string) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
resp, err := globalEtcdClient.Get(ctx, configFile)
defer cancel()
if err != nil {
return err
}
if resp.Count == 0 {
return errConfigNotFound
}
return nil
}
func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
configFile := path.Join(minioConfigPrefix, minioConfigFile)
if globalEtcdClient != nil {
return checkServerConfigEtcd(configFile)
}
if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile); err != nil {
if isErrObjectNotFound(err) {
return errConfigNotFound
}
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
logger.LogIf(ctx, err)
return err
}
return nil
}
func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data))
if err != nil {
return err
}
_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil)
return err
}
var errConfigNotFound = errors.New("config file not found")
func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
var buffer bytes.Buffer
// Read entire content by setting size to -1
if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, ""); err != nil {
// Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification
if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
return nil, errConfigNotFound
}
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
logger.LogIf(ctx, err)
return nil, err
}
// Return config not found on empty content.
if buffer.Len() == 0 {
return nil, errConfigNotFound
}
return &buffer, nil
}
// ConfigSys - config system.
type ConfigSys struct{}
// Load - load config.json.
func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
return sys.Init(objAPI)
}
// Init - initializes config system from config.json.
func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
if objAPI == nil {
return errInvalidArgument
}
return loadConfig(objAPI)
}
// NewConfigSys - creates new config system object.
func NewConfigSys() *ConfigSys {
return &ConfigSys{}
}
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/minio.json'
func migrateConfigToMinioSys() error {
// Construct path to config.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, minioConfigFile)
transactionConfigFile := configFile + ".transaction"
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
// and configFile, take a transaction lock to avoid race.
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
if err := objLock.GetLock(globalOperationTimeout); err != nil {
return err
}
defer objLock.Unlock()
// Verify if backend already has the file.
if err := checkServerConfig(context.Background(), newObjectLayerFn()); err != errConfigNotFound {
return err
} // if errConfigNotFound proceed to migrate..
var config = &serverConfig{}
if _, err := Load(getConfigFile(), config); err != nil {
return err
}
return saveServerConfig(newObjectLayerFn(), config)
}
// Initialize and load config from remote etcd or local config directory
func initConfig() {
if globalEtcdClient != nil {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
resp, err := globalEtcdClient.Get(ctx, getConfigFile())
cancel()
if err == nil && resp.Count > 0 {
logger.FatalIf(migrateConfig(), "Config migration failed")
}
} else {
if isFile(getConfigFile()) {
logger.FatalIf(migrateConfig(), "Config migration failed")
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
if err := migrateConfigToMinioSys(); err != nil {
logger.Fatal(err, "Unable to migrate 'config.json' to '.minio.sys/config/config.json'")
}
}
}
if err := checkServerConfig(context.Background(), newObjectLayerFn()); err != nil {
if err == errConfigNotFound {
// Config file does not exist, we create it fresh and return upon success.
logger.FatalIf(newConfig(newObjectLayerFn()), "Unable to initialize minio config for the first time")
logger.Info("Created minio configuration file successfully at " + getConfigDir())
} else {
logger.FatalIf(err, "Unable to load the configuration file")
}
}
logger.FatalIf(loadConfig(newObjectLayerFn()), "Unable to load the configuration file")
}

18
cmd/disk-cache_test.go
View File

@ -19,7 +19,6 @@ package cmd
import (
"bytes"
"context"
"os"
"reflect"
"testing"
"time"
@ -28,21 +27,15 @@ import (
)
// Initialize cache FS objects.
func initCacheFSObjects(disk string, cacheMaxUse int, t *testing.T) (*cacheFSObjects, error) {
newTestConfig(globalMinioDefaultRegion)
var err error
obj, err := newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
if err != nil {
t.Fatal(err)
}
return obj, nil
func initCacheFSObjects(disk string, cacheMaxUse int) (*cacheFSObjects, error) {
return newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
}
// inits diskCache struct for nDisks
func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
var cfs []*cacheFSObjects
for _, d := range drives {
obj, err := initCacheFSObjects(d, cacheMaxUse, t)
obj, err := initCacheFSObjects(d, cacheMaxUse)
if err != nil {
return nil, err
}
@ -131,11 +124,6 @@ func TestGetCacheFSMaxUse(t *testing.T) {
// test wildcard patterns for excluding entries from cache
func TestCacheExclusion(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
fsDirs, err := getRandomDisks(1)
if err != nil {
t.Fatal(err)

9
cmd/fs-v1-multipart_test.go
View File

@ -90,23 +90,18 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
// TestPutObjectPartFaultyDisk - test PutObjectPart with faulty disks
func TestPutObjectPartFaultyDisk(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
// Prepare for tests
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
defer os.RemoveAll(disk)
obj := initFSObjects(disk, t)
fs := obj.(*FSObjects)
bucketName := "bucket"
objectName := "object"
data := []byte("12345")
dataLen := int64(len(data))
if err = obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
t.Fatal("Cannot create bucket, err: ", err)
}

13
cmd/fs-v1_test.go
View File

@ -26,12 +26,6 @@ import (
// Tests for if parent directory is object
func TestFSParentDirIsObject(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
obj, disk, err := prepareFS()
if err != nil {
t.Fatal(err)
@ -120,12 +114,6 @@ func TestNewFS(t *testing.T) {
// TestFSShutdown - initialize a new FS object layer then calls
// Shutdown to check returned results
func TestFSShutdown(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
bucketName := "testbucket"
objectName := "object"
// Create and return an fsObject with its path in the disk
@ -133,6 +121,7 @@ func TestFSShutdown(t *testing.T) {
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
obj := initFSObjects(disk, t)
fs := obj.(*FSObjects)
objectContent := "12345"
obj.MakeBucketWithLocation(context.Background(), bucketName, "")
obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)

18
cmd/gateway-main.go
View File

@ -150,17 +150,25 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
// Validate if we have access, secret set through environment.
if !globalIsEnvCreds {
logger.Fatal(uiErrEnvCredentialsMissing(nil), "Unable to start gateway")
logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
}
// Create certs path.
logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
// Initialize gateway config.
initConfig()
// Initialize server config.
srvCfg := newServerConfig()
// Load logger subsystem
loadLoggers()
// Override any values from ENVs.
srvCfg.loadFromEnvs()
// Load values to cached global values.
srvCfg.loadToCachedConfigs()
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
globalServerConfigMu.Unlock()
// Check and load SSL certificates.
var err error

14
cmd/gateway-startup-msg_test.go
View File

@ -23,11 +23,14 @@ import (
// Test printing Gateway common message.
func TestPrintGatewayCommonMessage(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printGatewayCommonMsg(apiEndpoints)
@ -35,11 +38,14 @@ func TestPrintGatewayCommonMessage(t *testing.T) {
// Test print gateway startup message.
func TestPrintGatewayStartupMessage(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printGatewayStartupMessage(apiEndpoints, "azure")

3
cmd/globals.go
View File

@ -126,6 +126,9 @@ var (
// Holds the host that was passed using --address
globalMinioHost = ""
// globalConfigSys server config system.
globalConfigSys *ConfigSys
globalNotificationSys *NotificationSys
globalPolicySys *PolicySys

9
cmd/handler-utils_test.go
View File

@ -30,11 +30,14 @@ import (
// Tests validate bucket LocationConstraint.
func TestIsValidLocationContraint(t *testing.T) {
path, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("unable initialize config file, %s", err)
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
defer os.RemoveAll(path)
// Test with corrupted XML
malformedReq := &http.Request{

37
cmd/jwt_test.go
View File

@ -25,11 +25,15 @@ import (
)
func testAuthenticate(authType string, t *testing.T) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("unable initialize config file, %s", err)
t.Fatal(err)
}
defer os.RemoveAll(testPath)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
cred, err := auth.GetNewCredentials()
if err != nil {
t.Fatalf("Error getting new credentials: %s", err)
@ -92,11 +96,14 @@ func TestAuthenticateURL(t *testing.T) {
// Tests web request authenticator.
func TestWebRequestAuthenticate(t *testing.T) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("unable initialize config file, %s", err)
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
defer os.RemoveAll(testPath)
creds := globalServerConfig.GetCredential()
token, err := getTokenString(creds.AccessKey, creds.SecretKey)
@ -143,11 +150,14 @@ func TestWebRequestAuthenticate(t *testing.T) {
}
func BenchmarkAuthenticateNode(b *testing.B) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
b.Fatalf("unable initialize config file, %s", err)
b.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
b.Fatal(err)
}
defer os.RemoveAll(testPath)
creds := globalServerConfig.GetCredential()
b.ResetTimer()
@ -158,11 +168,14 @@ func BenchmarkAuthenticateNode(b *testing.B) {
}
func BenchmarkAuthenticateWeb(b *testing.B) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
b.Fatalf("unable initialize config file, %s", err)
b.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
b.Fatal(err)
}
defer os.RemoveAll(testPath)
creds := globalServerConfig.GetCredential()
b.ResetTimer()

16
cmd/lock-rpc-server_test.go
View File

@ -44,8 +44,11 @@ func testLockEquality(lriLeft, lriRight []lockRequesterInfo) bool {
// Helper function to create a lock server for testing
func createLockTestServer(t *testing.T) (string, *lockRPCReceiver, string) {
testPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
@ -61,7 +64,7 @@ func createLockTestServer(t *testing.T) (string, *lockRPCReceiver, string) {
if err != nil {
t.Fatal(err)
}
return testPath, locker, token
return fsDir, locker, token
}
// Test Lock functionality
@ -470,11 +473,14 @@ func TestLockServerInit(t *testing.T) {
return
}
rootPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("Init Test config failed")
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatalf("unable initialize config file, %s", err)
}
defer os.RemoveAll(rootPath)
currentIsDistXL := globalIsDistXL
currentLockServer := globalLockServer

45
cmd/notification.go
View File

@ -17,11 +17,9 @@
package cmd
import (
"bytes"
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"net/url"
"path"
@ -29,9 +27,7 @@ import (
"time"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event"
"github.com/minio/minio/pkg/hash"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/policy"
)
@ -85,8 +81,8 @@ func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string)
}()
}
// SetCredentials - calls SetCredentials RPC call on all peers.
func (sys *NotificationSys) SetCredentials(credentials auth.Credentials) map[xnet.Host]error {
// LoadCredentials - calls LoadCredentials RPC call on all peers.
func (sys *NotificationSys) LoadCredentials() map[xnet.Host]error {
errors := make(map[xnet.Host]error)
var wg sync.WaitGroup
for addr, client := range sys.peerRPCClientMap {
@ -95,7 +91,7 @@ func (sys *NotificationSys) SetCredentials(credentials auth.Credentials) map[xne
defer wg.Done()
// Try to set credentials in three attempts.
for i := 0; i < 3; i++ {
err := client.SetCredentials(credentials)
err := client.LoadCredentials()
if err == nil {
break
}
@ -529,41 +525,6 @@ func sendEvent(args eventArgs) {
}()
}
func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data))
if err != nil {
return err
}
_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil)
return err
}
var errConfigNotFound = errors.New("config file not found")
func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
var buffer bytes.Buffer
// Read entire content by setting size to -1
err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "")
if err != nil {
// Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification
if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
return nil, errConfigNotFound
}
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
logger.LogIf(ctx, err)
return nil, err
}
// Return NoSuchNotifications on empty content.
if buffer.Len() == 0 {
return nil, errNoSuchNotifications
}
return &buffer, nil
}
func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) (*event.Config, error) {
// Construct path to notification.xml for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)

6
cmd/object-api-listobjects_test.go
View File

@ -592,13 +592,7 @@ func BenchmarkListObjects(b *testing.B) {
b.Fatal(err)
}
defer os.RemoveAll(directory)
// initialize the root directory.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
b.Fatalf("Unable to initialize config. %s", err)
}
defer os.RemoveAll(rootPath)
// Create the obj.
obj := initFSObjectsB(directory, b)

7
cmd/peer-rpc-client.go
View File

@ -21,7 +21,6 @@ import (
"crypto/tls"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/policy"
@ -116,9 +115,9 @@ func (rpcClient *PeerRPCClient) SendEvent(bucketName string, targetID, remoteTar
return err
}
// SetCredentials - calls set credentials RPC.
func (rpcClient *PeerRPCClient) SetCredentials(credentials auth.Credentials) error {
args := SetCredentialsArgs{Credentials: credentials}
// LoadCredentials - calls load credentials RPC.
func (rpcClient *PeerRPCClient) LoadCredentials() error {
args := AuthArgs{}
reply := VoidReply{}
return rpcClient.Call(peerServiceName+".SetCredentials", &args, &reply)

37
cmd/peer-rpc-server.go
View File

@ -24,7 +24,6 @@ import (
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/logger"
xrpc "github.com/minio/minio/cmd/rpc"
"github.com/minio/minio/pkg/auth"
"github.com/minio/minio/pkg/event"
xnet "github.com/minio/minio/pkg/net"
"github.com/minio/minio/pkg/policy"
@ -158,35 +157,21 @@ func (receiver *peerRPCReceiver) SendEvent(args *SendEventArgs, reply *bool) err
return nil
}
// SetCredentialsArgs - set credentials RPC arguments.
type SetCredentialsArgs struct {
AuthArgs
Credentials auth.Credentials
}
// LoadCredentials - handles load credentials RPC call.
func (receiver *peerRPCReceiver) LoadCredentials(args *AuthArgs, reply *VoidReply) error {
// Construct path to config.json for the given bucket.
configFile := path.Join(bucketConfigPrefix, minioConfigFile)
transactionConfigFile := configFile + ".transaction"
// SetCredentials - handles set credentials RPC call.
func (receiver *peerRPCReceiver) SetCredentials(args *SetCredentialsArgs, reply *VoidReply) error {
if !args.Credentials.IsValid() {
return fmt.Errorf("invalid credentials passed")
}
// Acquire lock before updating global configuration.
globalServerConfigMu.Lock()
defer globalServerConfigMu.Unlock()
// Update credentials in memory
prevCred := globalServerConfig.SetCredential(args.Credentials)
// Save credentials to config file
if err := globalServerConfig.Save(getConfigFile()); err != nil {
// As saving configurstion failed, restore previous credential in memory.
globalServerConfig.SetCredential(prevCred)
logger.LogIf(context.Background(), err)
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
// and configFile, take a transaction lock to avoid race.
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
if err := objLock.GetRLock(globalOperationTimeout); err != nil {
return err
}
objLock.RUnlock()
return nil
return globalConfigSys.Load(newObjectLayerFn())
}
// NewPeerRPCServer - returns new peer RPC server.

11
cmd/post-policy_test.go
View File

@ -26,7 +26,6 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"os"
"testing"
"time"
@ -118,11 +117,9 @@ func TestPostPolicyBucketHandler(t *testing.T) {
// testPostPolicyBucketHandler - Tests validate post policy handler uploading objects.
func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatalf("Initializing config.json failed")
}
defer os.RemoveAll(root)
// get random bucket name.
bucketName := getRandomBucketName()
@ -139,7 +136,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
// objectNames[0].
// uploadIds [0].
// Create bucket before initiating NewMultipartUpload.
err = obj.MakeBucketWithLocation(context.Background(), bucketName, "")
err := obj.MakeBucketWithLocation(context.Background(), bucketName, "")
if err != nil {
// Failed to create newbucket, abort.
t.Fatalf("%s : %s", instanceType, err.Error())
@ -420,11 +417,9 @@ func TestPostPolicyBucketHandlerRedirect(t *testing.T) {
// testPostPolicyBucketHandlerRedirect tests POST Object when success_action_redirect is specified
func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t TestErrHandler) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatalf("Initializing config.json failed")
}
defer os.RemoveAll(root)
// get random bucket name.
bucketName := getRandomBucketName()

37
cmd/server-main.go
View File

@ -211,15 +211,14 @@ func serverMain(ctx *cli.Context) {
// Handle all server environment vars.
serverHandleEnvVars()
// In distributed setup users need to set ENVs always.
if !globalIsEnvCreds && globalIsDistXL {
logger.Fatal(uiErrEnvCredentialsMissingServer(nil), "Unable to initialize minio server in distributed mode")
}
// Create certs path.
logger.FatalIf(createConfigDir(), "Unable to initialize configuration files")
// Initialize server config.
initConfig()
// Load logger subsystem
loadLoggers()
// Check and load SSL certificates.
var err error
globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
@ -246,6 +245,11 @@ func serverMain(ctx *cli.Context) {
checkUpdate(mode)
}
// Enforce ENV credentials for distributed setup such that we can create the first config.
if globalIsDistXL && !globalIsEnvCreds {
logger.Fatal(uiErrInvalidCredentials(nil), "Unable to start the server in distrbuted mode. In distributed mode we require explicit credentials.")
}
// Set system resources to maximum.
logger.LogIf(context.Background(), setMaxResources())
@ -305,9 +309,30 @@ func serverMain(ctx *cli.Context) {
initFederatorBackend(newObject)
}
// Initialize server config.
initConfig()
// Load logger subsystem
loadLoggers()
var cacheConfig = globalServerConfig.GetCacheConfig()
if len(cacheConfig.Drives) > 0 {
// initialize the new disk cache objects.
globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
logger.FatalIf(err, "Unable to initialize disk caching")
}
// Re-enable logging
logger.Disable = false
// Create a new config system.
globalConfigSys = NewConfigSys()
// Initialize config system.
if err := globalConfigSys.Init(newObjectLayerFn()); err != nil {
logger.Fatal(err, "Unable to initialize config system")
}
// Create new policy system.
globalPolicySys = NewPolicySys()

21
cmd/server-startup-msg_test.go
View File

@ -113,11 +113,14 @@ func TestStripStandardPorts(t *testing.T) {
// Test printing server common message.
func TestPrintServerCommonMessage(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printServerCommonMsg(apiEndpoints)
@ -125,11 +128,14 @@ func TestPrintServerCommonMessage(t *testing.T) {
// Tests print cli access message.
func TestPrintCLIAccessMsg(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printCLIAccessMsg(apiEndpoints[0], "myminio")
@ -137,11 +143,14 @@ func TestPrintCLIAccessMsg(t *testing.T) {
// Test print startup message.
func TestPrintStartupMessage(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
apiEndpoints := []string{"http://127.0.0.1:9000"}
printStartupMessage(apiEndpoints)

7
cmd/server_test.go
View File

@ -28,7 +28,6 @@ import (
"math/rand"
"net/http"
"net/url"
"os"
"reflect"
"strings"
"sync"
@ -46,7 +45,6 @@ type TestSuiteCommon struct {
endPoint string
accessKey string
secretKey string
configPath string
signer signerType
secure bool
transport *http.Transport
@ -145,9 +143,6 @@ func TestServerSuite(t *testing.T) {
// Setting up the test suite.
// Starting the Test server with temporary FS backend.
func (s *TestSuiteCommon) SetUpSuite(c *check) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
c.Assert(err, nil)
if s.secure {
cert, key, err := generateTLSCertKey("127.0.0.1")
c.Assert(err, nil)
@ -171,12 +166,10 @@ func (s *TestSuiteCommon) SetUpSuite(c *check) {
s.endPoint = s.testServer.Server.URL
s.accessKey = s.testServer.AccessKey
s.secretKey = s.testServer.SecretKey
s.configPath = rootPath
}
// Called implicitly by "gopkg.in/check.v1" after all tests are run.
func (s *TestSuiteCommon) TearDownSuite(c *check) {
os.RemoveAll(s.configPath)
s.testServer.Stop()
}

28
cmd/signature-v2_test.go
View File

@ -40,11 +40,14 @@ func TestResourceListSorting(t *testing.T) {
// Tests presigned v2 signature.
func TestDoesPresignedV2SignatureMatch(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal("Unable to initialize test config.")
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
now := UTCNow()
@ -157,11 +160,14 @@ func TestDoesPresignedV2SignatureMatch(t *testing.T) {
// TestValidateV2AuthHeader - Tests validate the logic of V2 Authorization header validator.
func TestValidateV2AuthHeader(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal("Unable to initialize test config.")
t.Fatal(err)
}
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
accessID := globalServerConfig.GetCredential().AccessKey
testCases := []struct {
@ -228,11 +234,15 @@ func TestValidateV2AuthHeader(t *testing.T) {
}
func TestDoesPolicySignatureV2Match(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal("Unable to initialize test config.")
t.Fatal(err)
}
defer os.RemoveAll(root)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
creds := globalServerConfig.GetCredential()
policy := "policy"
testCases := []struct {

7
cmd/signature-v4_test.go
View File

@ -92,11 +92,14 @@ func TestDoesPolicySignatureMatch(t *testing.T) {
}
func TestDoesPresignedSignatureMatch(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
obj, fsDir, err := prepareFS()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
defer os.RemoveAll(fsDir)
if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
t.Fatal(err)
}
// sha256 hash of "payload"
payloadSHA256 := "239f59ed55e737c77147cf55ad0c1b030b6d7ee748a7426952f9b852d5a935e5"

207
cmd/test-utils_test.go
View File

@ -221,12 +221,12 @@ func prepareXL16() (ObjectLayer, []string, error) {
// Initialize FS objects.
func initFSObjects(disk string, t *testing.T) (obj ObjectLayer) {
newTestConfig(globalMinioDefaultRegion)
var err error
obj, err = NewFSObjectLayer(disk)
if err != nil {
t.Fatal(err)
}
newTestConfig(globalMinioDefaultRegion, obj)
return obj
}
@ -318,9 +318,9 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
if err != nil {
t.Fatal(err)
}
// set the server configuration.
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("%s", err)
}
@ -332,7 +332,6 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
for _, disk := range disks {
testServer.Disks = append(testServer.Disks, mustGetNewEndpointList(disk)...)
}
testServer.Root = root
testServer.AccessKey = credentials.AccessKey
testServer.SecretKey = credentials.SecretKey
@ -396,98 +395,6 @@ func StartTestServer(t TestErrHandler, instanceType string) TestServer {
return testServer
}
// Initializes storage RPC endpoints.
// The object Layer will be a temp back used for testing purpose.
func initTestStorageRPCEndPoint(endpoints EndpointList) http.Handler {
// Initialize router.
muxRouter := mux.NewRouter().SkipClean(true)
registerStorageRPCRouters(muxRouter, endpoints)
return muxRouter
}
// StartTestStorageRPCServer - Creates a temp XL backend and initializes storage RPC end points,
// then starts a test server with those storage RPC end points registered.
func StartTestStorageRPCServer(t TestErrHandler, instanceType string, diskN int) TestServer {
// create temporary backend for the test server.
disks, err := getRandomDisks(diskN)
if err != nil {
t.Fatal("Failed to create disks for the backend")
}
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("%s", err)
}
// Create an instance of TestServer.
testRPCServer := TestServer{}
// Get credential.
credentials := globalServerConfig.GetCredential()
endpoints := mustGetNewEndpointList(disks...)
testRPCServer.Root = root
testRPCServer.Disks = endpoints
testRPCServer.AccessKey = credentials.AccessKey
testRPCServer.SecretKey = credentials.SecretKey
// Run TestServer.
testRPCServer.Server = httptest.NewServer(initTestStorageRPCEndPoint(endpoints))
return testRPCServer
}
// Sets up a Peers RPC test server.
func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
// create temporary backend for the test server.
nDisks := 16
disks, err := getRandomDisks(nDisks)
if err != nil {
t.Fatal("Failed to create disks for the backend")
}
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("%s", err)
}
// create an instance of TestServer.
testRPCServer := TestServer{}
// Get credential.
credentials := globalServerConfig.GetCredential()
endpoints := mustGetNewEndpointList(disks...)
testRPCServer.Root = root
testRPCServer.Disks = endpoints
testRPCServer.AccessKey = credentials.AccessKey
testRPCServer.SecretKey = credentials.SecretKey
// create temporary backend for the test server.
objLayer, _, err := initObjectLayer(endpoints)
if err != nil {
t.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
}
globalObjLayerMutex.Lock()
globalObjectAPI = objLayer
testRPCServer.Obj = objLayer
globalObjLayerMutex.Unlock()
router := mux.NewRouter().SkipClean(true)
// need storage layer for bucket config storage.
registerStorageRPCRouters(router, endpoints)
// need API layer to send requests, etc.
registerAPIRouter(router)
// module being tested is Peer RPCs router.
registerPeerRPCRouter(router)
// Run TestServer.
testRPCServer.Server = httptest.NewServer(router)
// initialize remainder of serverCmdConfig
testRPCServer.endpoints = endpoints
return testRPCServer
}
// Sets the global config path to empty string.
func resetGlobalConfigPath() {
setConfigDir("")
@ -584,31 +491,17 @@ func resetTestGlobals() {
}
// Configure the server for the test run.
func newTestConfig(bucketLocation string) (rootPath string, err error) {
// Get test root.
rootPath, err = getTestRoot()
if err != nil {
return "", err
}
// Do this only once here.
setConfigDir(rootPath)
func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) {
// Initialize server config.
if err = newConfig(); err != nil {
return "", err
if err = newConfig(obj); err != nil {
return err
}
// Set a default region.
globalServerConfig.SetRegion(bucketLocation)
// Save config.
if err = globalServerConfig.Save(getConfigFile()); err != nil {
return "", err
}
// Return root path.
return rootPath, nil
return saveServerConfig(obj, globalServerConfig)
}
// Deleting the temporary backend and stopping the server.
@ -1985,12 +1878,6 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
// initialize NSLock.
initNSLock(false)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
}
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
@ -1999,7 +1886,15 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
if err != nil {
t.Fatalf("Initialzation of API handler tests failed: <ERROR> %s", err)
}
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("Unable to initialize server config. %s", err)
}
credentials := globalServerConfig.GetCredential()
// Executing the object layer tests for single node setup.
objAPITest(objLayer, FSTestStr, bucketFS, fsAPIRouter, credentials, t)
@ -2014,7 +1909,7 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
// Executing the object layer tests for XL.
objAPITest(objLayer, XLTestStr, bucketXL, xlAPIRouter, credentials, t)
// clean up the temporary test backend.
removeRoots(append(xlDisks, fsDir, rootPath))
removeRoots(append(xlDisks, fsDir))
}
// function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests.
@ -2033,19 +1928,17 @@ type objTestDiskNotFoundType func(obj ObjectLayer, instanceType string, dirs []s
// ExecObjectLayerTest - executes object layer tests.
// Creates single node and XL ObjectLayer instance and runs test for both the layers.
func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Unexpected error", err)
}
defer os.RemoveAll(rootPath)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
}
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatal("Unexpected error", err)
}
// Executing the object layer tests for single node setup.
objTest(objLayer, FSTestStr, t)
@ -2061,42 +1954,34 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
// ExecObjectLayerTestWithDirs - executes object layer tests.
// Creates single node and XL ObjectLayer instance and runs test for both the layers.
func ExecObjectLayerTestWithDirs(t TestErrHandler, objTest objTestTypeWithDirs) {
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Unexpected error", err)
}
defer os.RemoveAll(rootPath)
objLayer, fsDir, err := prepareFS()
if err != nil {
t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
}
objLayer, fsDirs, err := prepareXL16()
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatal("Unexpected error", err)
}
// Executing the object layer tests for XL.
objTest(objLayer, XLTestStr, fsDirs, t)
defer removeRoots(append(fsDirs, fsDir))
defer removeRoots(fsDirs)
}
// ExecObjectLayerDiskAlteredTest - executes object layer tests while altering
// disks in between tests. Creates XL ObjectLayer instance and runs test for XL layer.
func ExecObjectLayerDiskAlteredTest(t *testing.T, objTest objTestDiskNotFoundType) {
configPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Failed to create config directory", err)
}
defer os.RemoveAll(configPath)
objLayer, fsDirs, err := prepareXL16()
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatal("Failed to create config directory", err)
}
// Executing the object layer tests for XL.
objTest(objLayer, XLTestStr, fsDirs, t)
defer removeRoots(fsDirs)
@ -2108,12 +1993,6 @@ type objTestStaleFilesType func(obj ObjectLayer, instanceType string, dirs []str
// ExecObjectLayerStaleFilesTest - executes object layer tests those leaves stale
// files/directories under .minio/tmp. Creates XL ObjectLayer instance and runs test for XL layer.
func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType) {
configPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Failed to create config directory", err)
}
defer os.RemoveAll(configPath)
nDisks := 16
erasureDisks, err := getRandomDisks(nDisks)
if err != nil {
@ -2123,6 +2002,10 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatal("Failed to create config directory", err)
}
// Executing the object layer tests for XL.
objTest(objLayer, XLTestStr, erasureDisks, t)
defer removeRoots(erasureDisks)
@ -2262,7 +2145,8 @@ func initTestWebRPCEndPoint(objLayer ObjectLayer) http.Handler {
}
func StartTestS3PeerRPCServer(t TestErrHandler) (TestServer, []string) {
root, err := newTestConfig(globalMinioDefaultRegion)
// init disks
objLayer, fsDirs, err := prepareXL16()
if err != nil {
t.Fatalf("%s", err)
}
@ -2270,18 +2154,15 @@ func StartTestS3PeerRPCServer(t TestErrHandler) (TestServer, []string) {
// Create an instance of TestServer.
testRPCServer := TestServer{}
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatalf("%s", err)
}
// Fetch credentials for the test server.
credentials := globalServerConfig.GetCredential()
testRPCServer.Root = root
testRPCServer.AccessKey = credentials.AccessKey
testRPCServer.SecretKey = credentials.SecretKey
// init disks
objLayer, fsDirs, err := prepareXL16()
if err != nil {
t.Fatalf("%s", err)
}
// set object layer
testRPCServer.Obj = objLayer
globalObjLayerMutex.Lock()

8
cmd/ui-errors.go
View File

@ -66,12 +66,18 @@ var (
Secret key should be in between 8 and 40 characters.`,
)
uiErrEnvCredentialsMissing = newUIErrFn(
uiErrEnvCredentialsMissingGateway = newUIErrFn(
"Credentials missing",
"Please provide correct credentials",
`Access key and Secret key should be specified in Gateway mode from environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively.`,
)
uiErrEnvCredentialsMissingServer = newUIErrFn(
"Credentials missing",
"Please provide correct credentials",
`Access key and Secret key should be specified in distributed server mode from environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively.`,
)
uiErrInvalidErasureEndpoints = newUIErrFn(
"Invalid endpoint(s) in erasure mode",
"Please provide correct combination of local/remote paths",

4
cmd/web-handlers.go
View File

@ -520,14 +520,14 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
prevCred := globalServerConfig.SetCredential(creds)
// Persist updated credentials.
if err = globalServerConfig.Save(getConfigFile()); err != nil {
if err = saveServerConfig(newObjectLayerFn(), globalServerConfig); err != nil {
// Save the current creds when failed to update.
globalServerConfig.SetCredential(prevCred)
logger.LogIf(context.Background(), err)
return toJSONError(err)
}
if errs := globalNotificationSys.SetCredentials(creds); len(errs) != 0 {
if errs := globalNotificationSys.LoadCredentials(); len(errs) != 0 {
reply.PeerErrMsgs = make(map[string]string)
for host, err := range errs {
err = fmt.Errorf("Unable to update credentials on server %v: %v", host, err)

114
cmd/web-handlers_test.go
View File

@ -28,7 +28,6 @@ import (
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"reflect"
"strconv"
"strings"
@ -1506,14 +1505,13 @@ func TestWebCheckAuthorization(t *testing.T) {
// Register the API end points with XL/FS object layer.
apiRouter := initTestWebRPCEndPoint(obj)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
err = newTestConfig(globalMinioDefaultRegion, obj)
if err != nil {
t.Fatal("Init Test config failed", err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
rec := httptest.NewRecorder()
@ -1585,100 +1583,8 @@ func TestWebCheckAuthorization(t *testing.T) {
}
}
// TestWebObjectLayerNotReady - Test RPCs responses when disks are not ready
func TestWebObjectLayerNotReady(t *testing.T) {
// Initialize web rpc endpoint.
apiRouter := initTestWebRPCEndPoint(nil)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Init Test config failed", err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
rec := httptest.NewRecorder()
credentials := globalServerConfig.GetCredential()
authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
if err != nil {
t.Fatal("Cannot authenticate", err)
}
// Check if web rpc calls return Server not initialized. ServerInfo, GenerateAuth,
// SetAuth and GetAuth are not concerned
webRPCs := []string{"StorageInfo", "MakeBucket", "ListBuckets", "ListObjects", "RemoveObject",
"GetBucketPolicy", "SetBucketPolicy", "ListAllBucketPolicies"}
for _, rpcCall := range webRPCs {
args := &AuthArgs{
RPCVersion: globalRPCAPIVersion,
}
reply := &WebGenericRep{}
req, nerr := newTestWebRPCRequest("Web."+rpcCall, authorization, args)
if nerr != nil {
t.Fatalf("Test %s: Failed to create HTTP request: <ERROR> %v", rpcCall, nerr)
}
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("Test %s: Expected the response status to be 200, but instead found `%d`", rpcCall, rec.Code)
}
err = getTestWebRPCResponse(rec, &reply)
if err == nil {
t.Fatalf("Test %s: Should fail", rpcCall)
} else {
if !strings.EqualFold(err.Error(), errServerNotInitialized.Error()) {
t.Fatalf("Test %s: should fail with %s Found error: %v", rpcCall, errServerNotInitialized, err)
}
}
}
rec = httptest.NewRecorder()
// Test authorization of Web.Download
req, err := http.NewRequest("GET", "/minio/download/bucket/object?token="+authorization, nil)
if err != nil {
t.Fatalf("Cannot create upload request, %v", err)
}
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("Expected the response status to be 503, but instead found `%d`", rec.Code)
}
resp := string(rec.Body.Bytes())
if !strings.EqualFold(resp, errServerNotInitialized.Error()) {
t.Fatalf("Unexpected error message, expected: `%s`, found: `%s`", errServerNotInitialized, resp)
}
rec = httptest.NewRecorder()
// Test authorization of Web.Upload
content := []byte("temporary file's content")
req, err = http.NewRequest("PUT", "/minio/upload/bucket/object", nil)
req.Header.Set("Authorization", "Bearer "+authorization)
req.Header.Set("Content-Length", strconv.Itoa(len(content)))
req.Header.Set("x-amz-date", "20160814T114029Z")
req.Header.Set("Accept", "*/*")
req.Body = ioutil.NopCloser(bytes.NewReader(content))
if err != nil {
t.Fatalf("Cannot create upload request, %v", err)
}
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("Expected the response status to be 503, but instead found `%d`", rec.Code)
}
resp = string(rec.Body.Bytes())
if !strings.EqualFold(resp, errServerNotInitialized.Error()) {
t.Fatalf("Unexpected error message, expected: `%s`, found: `%s`", errServerNotInitialized, resp)
}
}
// TestWebObjectLayerFaultyDisks - Test Web RPC responses with faulty disks
func TestWebObjectLayerFaultyDisks(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
// Prepare XL backend
obj, fsDirs, err := prepareXL16()
if err != nil {
@ -1687,6 +1593,13 @@ func TestWebObjectLayerFaultyDisks(t *testing.T) {
// Executing the object layer tests for XL.
defer removeRoots(fsDirs)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
err = newTestConfig(globalMinioDefaultRegion, obj)
if err != nil {
t.Fatal("Init Test config failed", err)
}
bucketName := "mybucket"
err = obj.MakeBucketWithLocation(context.Background(), bucketName, "")
if err != nil {
@ -1702,15 +1615,6 @@ func TestWebObjectLayerFaultyDisks(t *testing.T) {
// Initialize web rpc endpoint.
apiRouter := initTestWebRPCEndPoint(obj)
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal("Init Test config failed", err)
}
// remove the root directory after the test ends.
defer os.RemoveAll(rootPath)
rec := httptest.NewRecorder()
credentials := globalServerConfig.GetCredential()

6
cmd/xl-sets_test.go
View File

@ -101,12 +101,6 @@ func TestNewXLSets(t *testing.T) {
// TestHashedLayer - tests the hashed layer which will be returned
// consistently for a given object name.
func TestHashedLayer(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
var objs []*xlObjects
for i := 0; i < 16; i++ {

6
cmd/xl-v1-common_test.go
View File

@ -25,12 +25,6 @@ import (
// Tests for if parent directory is object
func TestXLParentDirIsObject(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
obj, fsDisks, err := prepareXL16()
if err != nil {
t.Fatalf("Unable to initialize 'XL' object layer.")

13
cmd/xl-v1-healing-common_test.go
View File

@ -19,7 +19,6 @@ package cmd
import (
"bytes"
"context"
"os"
"path/filepath"
"testing"
"time"
@ -111,12 +110,6 @@ func partsMetaFromModTimes(modTimes []time.Time, algorithm BitrotAlgorithm, chec
// TestListOnlineDisks - checks if listOnlineDisks and outDatedDisks
// are consistent with each other.
func TestListOnlineDisks(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("Failed to initialize config - %v", err)
}
defer os.RemoveAll(rootPath)
obj, disks, err := prepareXL16()
if err != nil {
t.Fatalf("Prepare XL backend failed - %v", err)
@ -280,12 +273,6 @@ func TestListOnlineDisks(t *testing.T) {
func TestDisksWithAllParts(t *testing.T) {
ctx := context.Background()
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("Failed to initialize config - %v", err)
}
defer os.RemoveAll(rootPath)
obj, disks, err := prepareXL16()
if err != nil {
t.Fatalf("Prepare XL backend failed - %v", err)

13
cmd/xl-v1-healing_test.go
View File

@ -19,19 +19,12 @@ package cmd
import (
"bytes"
"context"
"os"
"path/filepath"
"testing"
)
// Tests undoes and validates if the undoing completes successfully.
func TestUndoMakeBucket(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
nDisks := 16
fsDirs, err := getRandomDisks(nDisks)
if err != nil {
@ -65,12 +58,6 @@ func TestUndoMakeBucket(t *testing.T) {
// Tests healing of object.
func TestHealObjectXL(t *testing.T) {
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
nDisks := 16
fsDirs, err := getRandomDisks(nDisks)
if err != nil {

8
cmd/xl-v1-multipart_test.go
View File

@ -18,20 +18,12 @@ package cmd
import (
"context"
"os"
"testing"
"time"
)
// Tests cleanup multipart uploads for erasure coded backend.
func TestXLCleanupStaleMultipartUploads(t *testing.T) {
// Initialize configuration
root, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("%s", err)
}
defer os.RemoveAll(root)
// Create an instance of xl backend
obj, fsDirs, err := prepareXL16()
if err != nil {

6
cmd/xl-v1-object_test.go
View File

@ -262,12 +262,6 @@ func TestPutObjectNoQuorum(t *testing.T) {
// Tests both object and bucket healing.
func TestHealing(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil {
t.Fatalf("Failed to initialize test config %v", err)
}
defer os.RemoveAll(rootPath)
obj, fsDirs, err := prepareXL16()
if err != nil {
t.Fatal(err)

64
pkg/madmin/config-commands.go
View File

@ -18,6 +18,8 @@
package madmin
import (
"bytes"
"crypto/rand"
"encoding/json"
"errors"
"fmt"
@ -26,6 +28,8 @@ import (
"net/http"
"github.com/minio/minio/pkg/quick"
"github.com/minio/sio"
"golang.org/x/crypto/argon2"
)
// NodeSummary - represents the result of an operation part of
@ -43,13 +47,48 @@ type SetConfigResult struct {
Status bool `json:"status"`
}
// GetConfig - returns the config.json of a minio setup.
func (adm *AdminClient) GetConfig() ([]byte, error) {
// No TLS?
if !adm.secure {
return nil, fmt.Errorf("credentials/configuration cannot be retrieved over an insecure connection")
// EncryptServerConfigData - encrypts server config data.
func EncryptServerConfigData(password string, data []byte) ([]byte, error) {
salt := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
return nil, err
}
// derive an encryption key from the master key and the nonce
var key [32]byte
copy(key[:], argon2.IDKey([]byte(password), salt, 1, 64*1024, 4, 32))
encrypted, err := sio.EncryptReader(bytes.NewReader(data), sio.Config{
Key: key[:]},
)
if err != nil {
return nil, err
}
edata, err := ioutil.ReadAll(encrypted)
return append(salt, edata...), err
}
// DecryptServerConfigData - decrypts server config data.
func DecryptServerConfigData(password string, data io.Reader) ([]byte, error) {
salt := make([]byte, 32)
if _, err := io.ReadFull(data, salt); err != nil {
return nil, err
}
// derive an encryption key from the master key and the nonce
var key [32]byte
copy(key[:], argon2.IDKey([]byte(password), salt, 1, 64*1024, 4, 32))
decrypted, err := sio.DecryptReader(data, sio.Config{
Key: key[:]},
)
if err != nil {
return nil, err
}
return ioutil.ReadAll(decrypted)
}
// GetConfig - returns the config.json of a minio setup, incoming data is encrypted.
func (adm *AdminClient) GetConfig() ([]byte, error) {
// Execute GET on /minio/admin/v1/config to get config of a setup.
resp, err := adm.executeMethod("GET",
requestData{relPath: "/v1/config"})
@ -61,19 +100,15 @@ func (adm *AdminClient) GetConfig() ([]byte, error) {
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp)
}
defer resp.Body.Close()
// Return the JSON marshaled bytes to user.
return ioutil.ReadAll(resp.Body)
return DecryptServerConfigData(adm.secretAccessKey, resp.Body)
}
// SetConfig - set config supplied as config.json for the setup.
func (adm *AdminClient) SetConfig(config io.Reader) (r SetConfigResult, err error) {
const maxConfigJSONSize = 256 * 1024 // 256KiB
if !adm.secure { // No TLS?
return r, fmt.Errorf("credentials/configuration cannot be updated over an insecure connection")
}
// Read configuration bytes
configBuf := make([]byte, maxConfigJSONSize+1)
n, err := io.ReadFull(config, configBuf)
@ -104,9 +139,14 @@ func (adm *AdminClient) SetConfig(config io.Reader) (r SetConfigResult, err erro
return r, errors.New("Duplicate key in json file: " + err.Error())
}
econfigBytes, err := EncryptServerConfigData(adm.secretAccessKey, configBytes)
if err != nil {
return r, err
}
reqData := requestData{
relPath: "/v1/config",
content: configBytes,
content: econfigBytes,
}
// Execute PUT on /minio/admin/v1/config to set config.

8
pkg/quick/quick.go
View File

@ -153,9 +153,9 @@ func (d config) DeepDiff(c Config) ([]structs.Field, error) {
return fields, nil
}
// checkData - checks the validity of config data. Data should be of
// CheckData - checks the validity of config data. Data should be of
// type struct and contain a string type field called "Version".
func checkData(data interface{}) error {
func CheckData(data interface{}) error {
if !structs.IsStruct(data) {
return fmt.Errorf("interface must be struct type")
}
@ -211,7 +211,7 @@ func LoadConfig(filename string, clnt *etcd.Client, data interface{}) (qc Config
// SaveConfig - saves given configuration data into given file as JSON.
func SaveConfig(data interface{}, filename string, clnt *etcd.Client) (err error) {
if err = checkData(data); err != nil {
if err = CheckData(data); err != nil {
return err
}
var qc Config
@ -225,7 +225,7 @@ func SaveConfig(data interface{}, filename string, clnt *etcd.Client) (err error
// NewConfig loads config from etcd client if provided, otherwise loads from a local filename.
// fails when all else fails.
func NewConfig(data interface{}, clnt *etcd.Client) (cfg Config, err error) {
if err := checkData(data); err != nil {
if err := CheckData(data); err != nil {
return nil, err
}

8
pkg/quick/quick_test.go
View File

@ -106,7 +106,7 @@ func TestSaveFailOnDir(t *testing.T) {
}
func TestCheckData(t *testing.T) {
err := checkData(nil)
err := CheckData(nil)
if err == nil {
t.Fatal("Unexpected should fail")
}
@ -117,7 +117,7 @@ func TestCheckData(t *testing.T) {
Directories []string
}
saveMeBadNoVersion := myStructBadNoVersion{"guest", "nopassword", []string{"Work", "Documents", "Music"}}
err = checkData(&saveMeBadNoVersion)
err = CheckData(&saveMeBadNoVersion)
if err == nil {
t.Fatal("Unexpected should fail if Version is not set")
}
@ -128,7 +128,7 @@ func TestCheckData(t *testing.T) {
Password string
}
saveMeBadVersionInt := myStructBadVersionInt{1, "guest", "nopassword"}
err = checkData(&saveMeBadVersionInt)
err = CheckData(&saveMeBadVersionInt)
if err == nil {
t.Fatal("Unexpected should fail if Version is integer")
}
@ -141,7 +141,7 @@ func TestCheckData(t *testing.T) {
}
saveMeGood := myStructGood{"1", "guest", "nopassword", []string{"Work", "Documents", "Music"}}
err = checkData(&saveMeGood)
err = CheckData(&saveMeGood)
if err != nil {
t.Fatal(err)
}