MyFavMirrors
/
minio
mirror of https://github.com/minio/minio.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1279 from harshavardhana/backend 

config: Migrate to the new version. Remove backend details.
This commit is contained in:
Anand Babu (AB) Periasamy 2016-04-02 17:59:21 -07:00
parent 6037fe66e9 484ba91b08
commit 2bb262cc56
10 changed files with 255 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -83,15 +83,10 @@ $ make
### How to use Minio?
Initialize minio server at `~/Photos`
~~~
$ minio init fs ~/Photos
~~~
After successfully initializing, start minio server.
Start minio server.
~~~
$ minio server
$ minio server ~/Photos
AccessKey: WLGDGYAQYIGI833EV05A SecretKey: BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF Region: us-east-1

71
config-migrate.go
View File

@ -23,6 +23,7 @@ import (
"github.com/minio/mc/pkg/console"
"github.com/minio/minio/pkg/probe"
"github.com/minio/minio/pkg/quick"
)
func migrateConfig() {
@ -30,6 +31,8 @@ func migrateConfig() {
purgeV1()
// Migrate version '2' to '3'.
migrateV2ToV3()
// Migrate version '3' to '4'.
migrateV3ToV4()
}
// Version '1' is not supported anymore and deprecated, safe to delete.
@ -54,6 +57,8 @@ func purgeV1() {
fatalIf(probe.NewError(errors.New("")), "Unexpected version found ‘"+cv1.Version+"’, cannot migrate.", nil)
}
// Version '2' to '3' config migration adds new fields and re-orders
// previous fields. Simplifies config for future additions.
func migrateV2ToV3() {
cv2, err := loadConfigV2()
if err != nil {
@ -65,23 +70,25 @@ func migrateV2ToV3() {
if cv2.Version != "2" {
return
}
serverConfig.SetAddr(":9000")
serverConfig.SetCredential(credential{
srvConfig := &configV3{}
srvConfig.Version = "3"
srvConfig.Addr = ":9000"
srvConfig.Credential = credential{
AccessKeyID: cv2.Credentials.AccessKeyID,
SecretAccessKey: cv2.Credentials.SecretAccessKey,
})
serverConfig.SetRegion(cv2.Credentials.Region)
serverConfig.SetConsoleLogger(consoleLogger{
}
srvConfig.Region = cv2.Credentials.Region
srvConfig.Logger.Console = consoleLogger{
Enable: true,
Level: "fatal",
})
}
flogger := fileLogger{}
flogger.Level = "error"
if cv2.FileLogger.Filename != "" {
flogger.Enable = true
flogger.Filename = cv2.FileLogger.Filename
}
serverConfig.SetFileLogger(flogger)
srvConfig.Logger.File = flogger
slogger := syslogLogger{}
slogger.Level = "debug"
@ -89,10 +96,52 @@ func migrateV2ToV3() {
slogger.Enable = true
slogger.Addr = cv2.SyslogLogger.Addr
}
serverConfig.SetSyslogLogger(slogger)
srvConfig.Logger.Syslog = slogger
err = serverConfig.Save()
fatalIf(err.Trace(), "Migrating from version ‘"+cv2.Version+"’ to ‘"+serverConfig.GetVersion()+"’ failed.", nil)
qc, err := quick.New(srvConfig)
fatalIf(err.Trace(), "Unable to initialize config.", nil)
console.Println("Migration from version ‘" + cv2.Version + "’ to ‘" + serverConfig.GetVersion() + "’ completed successfully.")
configFile, err := getConfigFile()
fatalIf(err.Trace(), "Unable to get config file.", nil)
// Migrate the config.
err = qc.Save(configFile)
fatalIf(err.Trace(), "Migrating from version ‘"+cv2.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil)
console.Println("Migration from version ‘" + cv2.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
}
// Version '3' to '4' migrates config, removes previous fields related
// to backend types and server address. This change further simplifies
// the config for future additions.
func migrateV3ToV4() {
cv3, err := loadConfigV3()
if err != nil {
if os.IsNotExist(err.ToGoError()) {
return
}
}
fatalIf(err.Trace(), "Unable to load config version ‘3’.", nil)
if cv3.Version != "3" {
return
}
// Save only the new fields, ignore the rest.
srvConfig := &serverConfigV4{}
srvConfig.Version = globalMinioConfigVersion
srvConfig.Credential = cv3.Credential
srvConfig.Region = cv3.Region
srvConfig.Logger.Console = cv3.Logger.Console
srvConfig.Logger.File = cv3.Logger.File
srvConfig.Logger.Syslog = cv3.Logger.Syslog
qc, err := quick.New(srvConfig)
fatalIf(err.Trace(), "Unable to initialize the quick config.", nil)
configFile, err := getConfigFile()
fatalIf(err.Trace(), "Unable to get config file.", nil)
err = qc.Save(configFile)
fatalIf(err.Trace(), "Migrating from version ‘"+cv3.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil)
console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
}

67
config-old.go
View File

@ -79,3 +79,70 @@ func loadConfigV2() (*configV2, *probe.Error) {
}
return qc.Data().(*configV2), nil
}
/////////////////// Config V3 ///////////////////
// backendV3 type.
type backendV3 struct {
Type string `json:"type"`
Disk string `json:"disk,omitempty"`
Disks []string `json:"disks,omitempty"`
}
// loggerV3 type.
type loggerV3 struct {
Console struct {
Enable bool `json:"enable"`
Level string `json:"level"`
}
File struct {
Enable bool `json:"enable"`
Filename string `json:"fileName"`
Level string `json:"level"`
}
Syslog struct {
Enable bool `json:"enable"`
Addr string `json:"address"`
Level string `json:"level"`
} `json:"syslog"`
// Add new loggers here.
}
// configV3 server configuration version '3'.
type configV3 struct {
Version string `json:"version"`
// Backend configuration.
Backend backendV3 `json:"backend"`
// http Server configuration.
Addr string `json:"address"`
// S3 API configuration.
Credential credential `json:"credential"`
Region string `json:"region"`
// Additional error logging configuration.
Logger loggerV3 `json:"logger"`
}
// loadConfigV3 load config version '3'.
func loadConfigV3() (*configV3, *probe.Error) {
configFile, err := getConfigFile()
if err != nil {
return nil, err.Trace()
}
if _, err := os.Stat(configFile); err != nil {
return nil, probe.NewError(err)
}
a := &configV3{}
a.Version = "3"
qc, err := quick.New(a)
if err != nil {
return nil, err.Trace()
}
if err := qc.Load(configFile); err != nil {
return nil, err.Trace()
}
return qc.Data().(*configV3), nil
}

76
config-v3.go → config-v4.go
View File

@ -24,16 +24,10 @@ import (
"github.com/minio/minio/pkg/quick"
)
// serverConfigV3 server configuration version '3'.
type serverConfigV3 struct {
// serverConfigV4 server configuration version '4'.
type serverConfigV4 struct {
Version string `json:"version"`
// Backend configuration.
Backend backend `json:"backend"`
// http Server configuration.
Addr string `json:"address"`
// S3 API configuration.
Credential credential `json:"credential"`
Region string `json:"region"`
@ -45,17 +39,10 @@ type serverConfigV3 struct {
rwMutex *sync.RWMutex
}
// backend type.
type backend struct {
Type string `json:"type"`
Disk string `json:"disk,omitempty"`
Disks []string `json:"disks,omitempty"`
}
// initConfig - initialize server config. config version (called only once).
func initConfig() *probe.Error {
if !isConfigFileExists() {
srvCfg := &serverConfigV3{}
srvCfg := &serverConfigV4{}
srvCfg.Version = globalMinioConfigVersion
srvCfg.Region = "us-east-1"
srvCfg.Credential = mustGenAccessKeys()
@ -89,7 +76,7 @@ func initConfig() *probe.Error {
if _, e := os.Stat(configFile); err != nil {
return probe.NewError(e)
}
srvCfg := &serverConfigV3{}
srvCfg := &serverConfigV4{}
srvCfg.Version = globalMinioConfigVersion
srvCfg.rwMutex = &sync.RWMutex{}
qc, err := quick.New(srvCfg)
@ -100,17 +87,17 @@ func initConfig() *probe.Error {
return err.Trace()
}
// Save the loaded config globally.
serverConfig = qc.Data().(*serverConfigV3)
serverConfig = qc.Data().(*serverConfigV4)
// Set the version properly after the unmarshalled json is loaded.
serverConfig.Version = globalMinioConfigVersion
return nil
}
// serverConfig server config.
var serverConfig *serverConfigV3
var serverConfig *serverConfigV4
// GetVersion get current config version.
func (s serverConfigV3) GetVersion() string {
func (s serverConfigV4) GetVersion() string {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
return s.Version
@ -119,98 +106,77 @@ func (s serverConfigV3) GetVersion() string {
/// Logger related.
// SetFileLogger set new file logger.
func (s *serverConfigV3) SetFileLogger(flogger fileLogger) {
func (s *serverConfigV4) SetFileLogger(flogger fileLogger) {
s.rwMutex.Lock()
defer s.rwMutex.Unlock()
s.Logger.File = flogger
}
// GetFileLogger get current file logger.
func (s serverConfigV3) GetFileLogger() fileLogger {
func (s serverConfigV4) GetFileLogger() fileLogger {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
return s.Logger.File
}
// SetConsoleLogger set new console logger.
func (s *serverConfigV3) SetConsoleLogger(clogger consoleLogger) {
func (s *serverConfigV4) SetConsoleLogger(clogger consoleLogger) {
s.rwMutex.Lock()
defer s.rwMutex.Unlock()
s.Logger.Console = clogger
}
// GetConsoleLogger get current console logger.
func (s serverConfigV3) GetConsoleLogger() consoleLogger {
func (s serverConfigV4) GetConsoleLogger() consoleLogger {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
return s.Logger.Console
}
// SetSyslogLogger set new syslog logger.
func (s *serverConfigV3) SetSyslogLogger(slogger syslogLogger) {
func (s *serverConfigV4) SetSyslogLogger(slogger syslogLogger) {
s.rwMutex.Lock()
defer s.rwMutex.Unlock()
s.Logger.Syslog = slogger
}
// GetSyslogLogger get current syslog logger.
func (s *serverConfigV3) GetSyslogLogger() syslogLogger {
func (s *serverConfigV4) GetSyslogLogger() syslogLogger {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
return s.Logger.Syslog
}
// SetRegion set new region.
func (s *serverConfigV3) SetRegion(region string) {
func (s *serverConfigV4) SetRegion(region string) {
s.rwMutex.Lock()
defer s.rwMutex.Unlock()
s.Region = region
}
// GetRegion get current region.
func (s serverConfigV3) GetRegion() string {
func (s serverConfigV4) GetRegion() string {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
return s.Region
}
// SetAddr set new address.
func (s *serverConfigV3) SetAddr(addr string) {
s.rwMutex.Lock()
defer s.rwMutex.Unlock()
s.Addr = addr
}
// GetAddr get current address.
func (s serverConfigV3) GetAddr() string {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
return s.Addr
}
// SetCredentials set new credentials.
func (s *serverConfigV3) SetCredential(creds credential) {
func (s *serverConfigV4) SetCredential(creds credential) {
s.rwMutex.Lock()
defer s.rwMutex.Unlock()
s.Credential = creds
}
// GetCredentials get current credentials.
func (s serverConfigV3) GetCredential() credential {
func (s serverConfigV4) GetCredential() credential {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
return s.Credential
}
// SetBackend set new backend.
func (s *serverConfigV3) SetBackend(bknd backend) {
s.rwMutex.Lock()
defer s.rwMutex.Unlock()
s.Backend = bknd
}
// Save config.
func (s serverConfigV3) Save() *probe.Error {
func (s serverConfigV4) Save() *probe.Error {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
@ -234,9 +200,3 @@ func (s serverConfigV3) Save() *probe.Error {
// Return success.
return nil
}
func (s serverConfigV3) GetBackend() backend {
s.rwMutex.RLock()
defer s.rwMutex.RUnlock()
return s.Backend
}

12
docker/start.sh
View File

@ -1,14 +1,10 @@
#!/bin/bash
if [ "$1" = 'fs' ]; then
if [ -z "$2" ]; then
## Fix this check when we arrive at XL.
if [ -z "$1" ]; then
echo "Invalid arguments"
echo "Usage: fs <export_dir>"
echo "Usage: <export_dir>"
exit 1
else
/bin/mkdir -p "$2" && /minio init fs "$2" && /minio server
fi
else
echo "Usage: fs <export_dir>"
exit 0
/bin/mkdir -p "$2" && /minio server "$2"
fi

2
globals.go
View File

@ -29,7 +29,7 @@ const (
// minio configuration related constants.
const (
globalMinioConfigVersion = "3"
globalMinioConfigVersion = "4"
globalMinioConfigDir = ".minio"
globalMinioCertsDir = ".minio/certs"
globalMinioCertFile = "public.crt"

9
main.go
View File

@ -73,10 +73,6 @@ func init() {
if !isContainerized() && os.Geteuid() == 0 {
console.Fatalln("Please run ‘minio’ as a non-root user.")
}
// Initialize config.
err := initConfig()
fatalIf(err.Trace(), "Unable to initialize minio config.", nil)
}
func migrate() {
@ -142,7 +138,6 @@ func findClosestCommands(command string) []string {
func registerApp() *cli.App {
// Register all commands.
registerCommand(initCmd)
registerCommand(serverCmd)
registerCommand(versionCmd)
registerCommand(updateCmd)
@ -199,6 +194,10 @@ func main() {
// Migrate any old version of config / state files to newer format.
migrate()
// Initialize config.
err := initConfig()
fatalIf(err.Trace(), "Unable to initialize minio config.", nil)
// Enable all loggers by now.
enableLoggers()

62
minio.md
View File

@ -1,32 +1,40 @@
# Server command line SPEC. - Tue Mar 22 06:04:42 UTC 2016
# Server command line SPEC - Fri Apr 1 19:49:56 PDT 2016
Minio initialize filesystem backend.
~~~
$ minio init fs <path>
~~~
## Single disk
Regular single server, single disk mode.
```
$ minio server <disk>
```
Minio initialize XL backend.
~~~
$ minio init xl <url1>...<url16>
~~~
## Multi disk
```
$ minio controller start - Start minio controller.
```
Prints a secret token to be used by all parties.
For 'fs' backend it starts the server.
~~~
$ minio server
~~~
Start all servers without disk with `MINIO_CONTROLLER` env set, start in cluster mode.
```
$ MINIO_CONTROLLER=<host>:<token> minio server
```
For 'xl' backend it waits for servers to join.
~~~
$ minio server
... [PROGRESS BAR] of servers connecting
~~~
## Minio Controller cli.
Now on other servers execute 'join' and they connect.
~~~
....
minio join <url1> -- from <url2> && minio server
minio join <url1> -- from <url3> && minio server
...
...
minio join <url1> -- from <url16> && minio server
~~~
Set controller host and token.
```
$ export MINIO_CONTROLLER=<host>:<token>
```
Create a cluster from the pool of nodes.
```
$ minioctl new <clustername> <ip1>:/mnt/disk1 .. <ip16>:/mnt/disk1
```
Start the cluster.
```
$ minioctl start <clustername>
```
Stop the cluster
```
$ minioctl stop <clustername>
```

160
minio-main.go → server-main.go
View File

@ -34,67 +34,46 @@ import (
"github.com/minio/minio/pkg/probe"
)
var initCmd = cli.Command{
Name: "init",
Usage: "Initialize Minio cloud storage server.",
var serverCmd = cli.Command{
Name: "server",
Usage: "Start Minio cloud storage server.",
Flags: []cli.Flag{
cli.StringFlag{
Name: "address",
Value: ":9000",
},
},
Action: initMain,
CustomHelpTemplate: `NAME:
minio {{.Name}} - {{.Usage}}
USAGE:
minio {{.Name}} [OPTION VALUE] PATH
OPTIONS:
{{range .Flags}}{{.}}
{{end}}
ENVIRONMENT VARIABLES:
MINIO_ACCESS_KEY, MINIO_SECRET_KEY: Access and secret key to use.
EXAMPLES:
1. Start minio server on Linux.
$ minio {{.Name}} fs /home/shared
2. Start minio server on Windows.
$ minio {{.Name}} fs C:\MyShare
3. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
$ minio {{.Name}} --address 192.168.1.101:9000 fs /home/shared
4. Start minio server with minimum free disk threshold to 5%
$ minio {{.Name}} fs /home/shared/Pictures
`,
}
var serverCmd = cli.Command{
Name: "server",
Usage: "Start Minio cloud storage server.",
Flags: []cli.Flag{},
Action: serverMain,
CustomHelpTemplate: `NAME:
minio {{.Name}} - {{.Usage}}
USAGE:
minio {{.Name}}
minio {{.Name}} [OPTIONS] PATH
OPTIONS:
{{range .Flags}}{{.}}
{{end}}
ENVIRONMENT VARIABLES:
MINIO_ACCESS_KEY: Access key string of 5 to 20 characters in length.
MINIO_SECRET_KEY: Secret key string of 8 to 40 characters in length.
EXAMPLES:
1. Start minio server.
$ minio {{.Name}}
$ minio {{.Name}} /home/shared
2. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
$ minio {{.Name}} --address 192.168.1.101:9000 /home/shared
3. Start minio server on Windows.
$ minio {{.Name}} C:\MyShare
`,
}
// configureServer configure a new server instance
func configureServer(objectAPI ObjectAPI) *http.Server {
func configureServer(serverAddr string, objectAPI ObjectAPI) *http.Server {
// Minio server config
apiServer := &http.Server{
Addr: serverConfig.GetAddr(),
Addr: serverAddr,
Handler: configureServerHandler(objectAPI),
MaxHeaderBytes: 1 << 20,
}
@ -142,33 +121,11 @@ func printListenIPs(httpServerConf *http.Server) {
}
}
// initServer initialize server
func initServer(c *cli.Context) {
host, port, _ := net.SplitHostPort(c.String("address"))
// If port empty, default to port '80'
if port == "" {
port = "80"
// if SSL is enabled, choose port as "443" instead.
if isSSL() {
port = "443"
}
}
// Join host and port.
serverConfig.SetAddr(net.JoinHostPort(host, port))
// Set backend FS type.
if c.Args().Get(0) == "fs" {
fsPath := strings.TrimSpace(c.Args().Get(1))
// Last argument is always a file system path, verify if it exists and is accessible.
_, e := os.Stat(fsPath)
fatalIf(probe.NewError(e), "Unable to validate the path", nil)
serverConfig.SetBackend(backend{
Type: "fs",
Disk: fsPath,
})
} // else { Add backend XL type here.
// initServerConfig initialize server config.
func initServerConfig(c *cli.Context) {
// Save new config.
err := serverConfig.Save()
fatalIf(err.Trace(), "Unable to save config.", nil)
// Fetch access keys from environment variables if any and update the config.
accessKey := os.Getenv("MINIO_ACCESS_KEY")
@ -187,29 +144,15 @@ func initServer(c *cli.Context) {
SecretAccessKey: secretKey,
})
}
// Save new config.
err := serverConfig.Save()
fatalIf(err.Trace(), "Unable to save config.", nil)
// Successfully written.
backend := serverConfig.GetBackend()
if backend.Type == "fs" {
console.Println(colorGreen("Successfully initialized Minio at %s", backend.Disk))
}
}
// Check init arguments.
func checkInitSyntax(c *cli.Context) {
// Check server arguments.
func checkServerSyntax(c *cli.Context) {
if !c.Args().Present() || c.Args().First() == "help" {
cli.ShowCommandHelpAndExit(c, "init", 1)
cli.ShowCommandHelpAndExit(c, "server", 1)
}
if len(c.Args()) > 2 {
fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘minio init --help’.", nil)
}
path := strings.TrimSpace(c.Args().Last())
if path == "" {
fatalIf(probe.NewError(errInvalidArgument), "Path argument cannot be empty.", nil)
fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘minio server --help’.", nil)
}
}
@ -292,34 +235,45 @@ func checkPortAvailability(port int) {
}
}
func initMain(c *cli.Context) {
// check 'init' cli arguments.
checkInitSyntax(c)
// Initialize server.
initServer(c)
}
func serverMain(c *cli.Context) {
if c.Args().Present() || c.Args().First() == "help" {
cli.ShowCommandHelpAndExit(c, "server", 1)
// check 'server' cli arguments.
checkServerSyntax(c)
// Initialize server config.
initServerConfig(c)
// Server address.
serverAddress := c.String("address")
host, port, _ := net.SplitHostPort(serverAddress)
// If port empty, default to port '80'
if port == "" {
port = "80"
// if SSL is enabled, choose port as "443" instead.
if isSSL() {
port = "443"
}
}
// Check configured ports.
checkPortAvailability(getPort(net.JoinHostPort(host, port)))
var objectAPI ObjectAPI
var err *probe.Error
// get backend.
backend := serverConfig.GetBackend()
if backend.Type == "fs" {
// Set backend FS type.
fsPath := strings.TrimSpace(c.Args().Get(0))
if fsPath != "" {
// Last argument is always a file system path, verify if it exists and is accessible.
_, e := os.Stat(fsPath)
fatalIf(probe.NewError(e), "Unable to validate the path", nil)
// Initialize filesystem storage layer.
objectAPI, err = newFS(backend.Disk)
fatalIf(err.Trace(backend.Type, backend.Disk), "Initializing filesystem failed.", nil)
} else { // else if backend.Type == "xl" { here.
console.Fatalln("No known backends configured, please use ‘minio init --help’ to initialize a backend.")
objectAPI, err = newFS(fsPath)
fatalIf(err.Trace(fsPath), "Initializing filesystem failed.", nil)
}
// Configure server.
apiServer := configureServer(objectAPI)
apiServer := configureServer(serverAddress, objectAPI)
// Credential.
cred := serverConfig.GetCredential()

10
server_fs_test.go
View File

@ -81,15 +81,15 @@ func (s *MyAPISuite) SetUpSuite(c *C) {
// Initialize server config.
initConfig()
// Set port.
addr := ":" + strconv.Itoa(getFreePort())
// Get credential.
s.credential = serverConfig.GetCredential()
// Set a default region.
serverConfig.SetRegion("us-east-1")
// Set a new address.
serverConfig.SetAddr(":" + strconv.Itoa(getFreePort()))
// Do this only once here
setGlobalConfigPath(root)
@ -99,8 +99,8 @@ func (s *MyAPISuite) SetUpSuite(c *C) {
fs, err := newFS(fsroot)
c.Assert(err, IsNil)
httpHandler := configureServerHandler(fs)
testAPIFSCacheServer = httptest.NewServer(httpHandler)
apiServer := configureServer(addr, fs)
testAPIFSCacheServer = httptest.NewServer(apiServer.Handler)
}
func (s *MyAPISuite) TearDownSuite(c *C) {