From e3de237eb811611a4dc186ee914736781f01984d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 10 Sep 2016 00:51:25 -0700 Subject: [PATCH] Move config to v7 to fix previous v6 bug on null Notification entries. (#2650) Thanks to @koolhead17 for reporting this. --- cmd/config-migrate.go | 94 ++++++++++++++++++++++++------ cmd/config-old.go | 36 ++++++++++++ cmd/{config-v6.go => config-v7.go} | 52 ++++++++--------- cmd/config-v7_test.go | 91 +++++++++++++++++++++++++++++ cmd/globals.go | 2 +- cmd/utils_test.go | 6 +- 6 files changed, 233 insertions(+), 48 deletions(-) rename cmd/{config-v6.go => config-v7.go} (78%) create mode 100644 cmd/config-v7_test.go diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index 2731c7413..87ecf0591 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -36,6 +36,8 @@ func migrateConfig() { migrateV4ToV5() // Migrate version '5' to '6. migrateV5ToV6() + // Migrate version '6' to '7'. + migrateV6ToV7() } // Version '1' is not supported anymore and deprecated, safe to delete. @@ -151,6 +153,46 @@ func migrateV3ToV4() { console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.") } +// Version '4' to '5' migrates config, removes previous fields related +// to backend types and server address. This change further simplifies +// the config for future additions. +func migrateV4ToV5() { + cv4, err := loadConfigV4() + if err != nil && os.IsNotExist(err) { + return + } + fatalIf(err, "Unable to load config version ‘4’.") + if cv4.Version != "4" { + return + } + + // Save only the new fields, ignore the rest. + srvConfig := &configV5{} + srvConfig.Version = "5" + srvConfig.Credential = cv4.Credential + srvConfig.Region = cv4.Region + if srvConfig.Region == "" { + // Region needs to be set for AWS Signature Version 4. + srvConfig.Region = "us-east-1" + } + srvConfig.Logger.Console = cv4.Logger.Console + srvConfig.Logger.File = cv4.Logger.File + srvConfig.Logger.Syslog = cv4.Logger.Syslog + srvConfig.Logger.AMQP.Enable = false + srvConfig.Logger.ElasticSearch.Enable = false + srvConfig.Logger.Redis.Enable = false + + qc, err := quick.New(srvConfig) + fatalIf(err, "Unable to initialize the quick config.") + configFile, err := getConfigFile() + fatalIf(err, "Unable to get config file.") + + err = qc.Save(configFile) + fatalIf(err, "Failed to migrate config from ‘"+cv4.Version+"’ to ‘"+srvConfig.Version+"’ failed.") + + console.Println("Migration from version ‘" + cv4.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.") +} + // Version '5' to '6' migrates config, removes previous fields related // to backend types and server address. This change further simplifies // the config for future additions. @@ -165,8 +207,8 @@ func migrateV5ToV6() { } // Save only the new fields, ignore the rest. - srvConfig := &serverConfigV6{} - srvConfig.Version = globalMinioConfigVersion + srvConfig := &configV6{} + srvConfig.Version = "6" srvConfig.Credential = cv5.Credential srvConfig.Region = cv5.Region if srvConfig.Region == "" { @@ -176,6 +218,7 @@ func migrateV5ToV6() { srvConfig.Logger.Console = cv5.Logger.Console srvConfig.Logger.File = cv5.Logger.File srvConfig.Logger.Syslog = cv5.Logger.Syslog + srvConfig.Notify.AMQP = map[string]amqpNotify{ "1": { Enable: cv5.Logger.AMQP.Enable, @@ -217,34 +260,49 @@ func migrateV5ToV6() { console.Println("Migration from version ‘" + cv5.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.") } -// Version '4' to '5' migrates config, removes previous fields related +// Version '6' to '7' migrates config, removes previous fields related // to backend types and server address. This change further simplifies // the config for future additions. -func migrateV4ToV5() { - cv4, err := loadConfigV4() +func migrateV6ToV7() { + cv6, err := loadConfigV6() if err != nil && os.IsNotExist(err) { return } - fatalIf(err, "Unable to load config version ‘4’.") - if cv4.Version != "4" { + fatalIf(err, "Unable to load config version ‘6’.") + if cv6.Version != "6" { return } // Save only the new fields, ignore the rest. - srvConfig := &configV5{} + srvConfig := &serverConfigV7{} srvConfig.Version = globalMinioConfigVersion - srvConfig.Credential = cv4.Credential - srvConfig.Region = cv4.Region + srvConfig.Credential = cv6.Credential + srvConfig.Region = cv6.Region if srvConfig.Region == "" { // Region needs to be set for AWS Signature Version 4. srvConfig.Region = "us-east-1" } - srvConfig.Logger.Console = cv4.Logger.Console - srvConfig.Logger.File = cv4.Logger.File - srvConfig.Logger.Syslog = cv4.Logger.Syslog - srvConfig.Logger.AMQP.Enable = false - srvConfig.Logger.ElasticSearch.Enable = false - srvConfig.Logger.Redis.Enable = false + srvConfig.Logger.Console = cv6.Logger.Console + srvConfig.Logger.File = cv6.Logger.File + srvConfig.Logger.Syslog = cv6.Logger.Syslog + srvConfig.Notify.AMQP = make(map[string]amqpNotify) + srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify) + srvConfig.Notify.Redis = make(map[string]redisNotify) + if len(cv6.Notify.AMQP) == 0 { + srvConfig.Notify.AMQP["1"] = amqpNotify{} + } else { + srvConfig.Notify.AMQP = cv6.Notify.AMQP + } + if len(cv6.Notify.ElasticSearch) == 0 { + srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{} + } else { + srvConfig.Notify.ElasticSearch = cv6.Notify.ElasticSearch + } + if len(cv6.Notify.Redis) == 0 { + srvConfig.Notify.Redis["1"] = redisNotify{} + } else { + srvConfig.Notify.Redis = cv6.Notify.Redis + } qc, err := quick.New(srvConfig) fatalIf(err, "Unable to initialize the quick config.") @@ -252,7 +310,7 @@ func migrateV4ToV5() { fatalIf(err, "Unable to get config file.") err = qc.Save(configFile) - fatalIf(err, "Failed to migrate config from ‘"+cv4.Version+"’ to ‘"+srvConfig.Version+"’ failed.") + fatalIf(err, "Failed to migrate config from ‘"+cv6.Version+"’ to ‘"+srvConfig.Version+"’ failed.") - console.Println("Migration from version ‘" + cv4.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.") + console.Println("Migration from version ‘" + cv6.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.") } diff --git a/cmd/config-old.go b/cmd/config-old.go index 840019e6f..87b781bf5 100644 --- a/cmd/config-old.go +++ b/cmd/config-old.go @@ -274,3 +274,39 @@ func loadConfigV5() (*configV5, error) { } return c, nil } + +// configV6 server configuration version '6'. +type configV6 struct { + Version string `json:"version"` + + // S3 API configuration. + Credential credential `json:"credential"` + Region string `json:"region"` + + // Additional error logging configuration. + Logger logger `json:"logger"` + + // Notification queue configuration. + Notify notifier `json:"notify"` +} + +// loadConfigV6 load config version '6'. +func loadConfigV6() (*configV6, error) { + configFile, err := getConfigFile() + if err != nil { + return nil, err + } + if _, err = os.Stat(configFile); err != nil { + return nil, err + } + c := &configV6{} + c.Version = "6" + qc, err := quick.New(c) + if err != nil { + return nil, err + } + if err := qc.Load(configFile); err != nil { + return nil, err + } + return c, nil +} diff --git a/cmd/config-v6.go b/cmd/config-v7.go similarity index 78% rename from cmd/config-v6.go rename to cmd/config-v7.go index 6c742227c..a2c15ef8f 100644 --- a/cmd/config-v6.go +++ b/cmd/config-v7.go @@ -23,8 +23,8 @@ import ( "github.com/minio/minio/pkg/quick" ) -// serverConfigV6 server configuration version '5'. -type serverConfigV6 struct { +// serverConfigV7 server configuration version '7'. +type serverConfigV7 struct { Version string `json:"version"` // S3 API configuration. @@ -45,7 +45,7 @@ type serverConfigV6 struct { func initConfig() error { if !isConfigFileExists() { // Initialize server config. - srvCfg := &serverConfigV6{} + srvCfg := &serverConfigV7{} srvCfg.Version = globalMinioConfigVersion srvCfg.Region = "us-east-1" srvCfg.Credential = mustGenAccessKeys() @@ -84,7 +84,7 @@ func initConfig() error { if _, err = os.Stat(configFile); err != nil { return err } - srvCfg := &serverConfigV6{} + srvCfg := &serverConfigV7{} srvCfg.Version = globalMinioConfigVersion srvCfg.rwMutex = &sync.RWMutex{} qc, err := quick.New(srvCfg) @@ -103,10 +103,10 @@ func initConfig() error { } // serverConfig server config. -var serverConfig *serverConfigV6 +var serverConfig *serverConfigV7 // GetVersion get current config version. -func (s serverConfigV6) GetVersion() string { +func (s serverConfigV7) GetVersion() string { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Version @@ -114,135 +114,135 @@ func (s serverConfigV6) GetVersion() string { /// Logger related. -func (s *serverConfigV6) SetAMQPNotifyByID(accountID string, amqpn amqpNotify) { +func (s *serverConfigV7) SetAMQPNotifyByID(accountID string, amqpn amqpNotify) { s.rwMutex.Lock() defer s.rwMutex.Unlock() s.Notify.AMQP[accountID] = amqpn } -func (s serverConfigV6) GetAMQP() map[string]amqpNotify { +func (s serverConfigV7) GetAMQP() map[string]amqpNotify { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Notify.AMQP } // GetAMQPNotify get current AMQP logger. -func (s serverConfigV6) GetAMQPNotifyByID(accountID string) amqpNotify { +func (s serverConfigV7) GetAMQPNotifyByID(accountID string) amqpNotify { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Notify.AMQP[accountID] } -func (s *serverConfigV6) SetElasticSearchNotifyByID(accountID string, esNotify elasticSearchNotify) { +func (s *serverConfigV7) SetElasticSearchNotifyByID(accountID string, esNotify elasticSearchNotify) { s.rwMutex.Lock() defer s.rwMutex.Unlock() s.Notify.ElasticSearch[accountID] = esNotify } -func (s serverConfigV6) GetElasticSearch() map[string]elasticSearchNotify { +func (s serverConfigV7) GetElasticSearch() map[string]elasticSearchNotify { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Notify.ElasticSearch } // GetElasticSearchNotify get current ElasicSearch logger. -func (s serverConfigV6) GetElasticSearchNotifyByID(accountID string) elasticSearchNotify { +func (s serverConfigV7) GetElasticSearchNotifyByID(accountID string) elasticSearchNotify { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Notify.ElasticSearch[accountID] } -func (s *serverConfigV6) SetRedisNotifyByID(accountID string, rNotify redisNotify) { +func (s *serverConfigV7) SetRedisNotifyByID(accountID string, rNotify redisNotify) { s.rwMutex.Lock() defer s.rwMutex.Unlock() s.Notify.Redis[accountID] = rNotify } -func (s serverConfigV6) GetRedis() map[string]redisNotify { +func (s serverConfigV7) GetRedis() map[string]redisNotify { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Notify.Redis } // GetRedisNotify get current Redis logger. -func (s serverConfigV6) GetRedisNotifyByID(accountID string) redisNotify { +func (s serverConfigV7) GetRedisNotifyByID(accountID string) redisNotify { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Notify.Redis[accountID] } // SetFileLogger set new file logger. -func (s *serverConfigV6) SetFileLogger(flogger fileLogger) { +func (s *serverConfigV7) SetFileLogger(flogger fileLogger) { s.rwMutex.Lock() defer s.rwMutex.Unlock() s.Logger.File = flogger } // GetFileLogger get current file logger. -func (s serverConfigV6) GetFileLogger() fileLogger { +func (s serverConfigV7) GetFileLogger() fileLogger { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Logger.File } // SetConsoleLogger set new console logger. -func (s *serverConfigV6) SetConsoleLogger(clogger consoleLogger) { +func (s *serverConfigV7) SetConsoleLogger(clogger consoleLogger) { s.rwMutex.Lock() defer s.rwMutex.Unlock() s.Logger.Console = clogger } // GetConsoleLogger get current console logger. -func (s serverConfigV6) GetConsoleLogger() consoleLogger { +func (s serverConfigV7) GetConsoleLogger() consoleLogger { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Logger.Console } // SetSyslogLogger set new syslog logger. -func (s *serverConfigV6) SetSyslogLogger(slogger syslogLogger) { +func (s *serverConfigV7) SetSyslogLogger(slogger syslogLogger) { s.rwMutex.Lock() defer s.rwMutex.Unlock() s.Logger.Syslog = slogger } // GetSyslogLogger get current syslog logger. -func (s *serverConfigV6) GetSyslogLogger() syslogLogger { +func (s *serverConfigV7) GetSyslogLogger() syslogLogger { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Logger.Syslog } // SetRegion set new region. -func (s *serverConfigV6) SetRegion(region string) { +func (s *serverConfigV7) SetRegion(region string) { s.rwMutex.Lock() defer s.rwMutex.Unlock() s.Region = region } // GetRegion get current region. -func (s serverConfigV6) GetRegion() string { +func (s serverConfigV7) GetRegion() string { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Region } // SetCredentials set new credentials. -func (s *serverConfigV6) SetCredential(creds credential) { +func (s *serverConfigV7) SetCredential(creds credential) { s.rwMutex.Lock() defer s.rwMutex.Unlock() s.Credential = creds } // GetCredentials get current credentials. -func (s serverConfigV6) GetCredential() credential { +func (s serverConfigV7) GetCredential() credential { s.rwMutex.RLock() defer s.rwMutex.RUnlock() return s.Credential } // Save config. -func (s serverConfigV6) Save() error { +func (s serverConfigV7) Save() error { s.rwMutex.RLock() defer s.rwMutex.RUnlock() diff --git a/cmd/config-v7_test.go b/cmd/config-v7_test.go new file mode 100644 index 000000000..46ce85823 --- /dev/null +++ b/cmd/config-v7_test.go @@ -0,0 +1,91 @@ +package cmd + +import ( + "reflect" + "testing" +) + +func TestServerConfig(t *testing.T) { + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + if serverConfig.GetRegion() != "us-east-1" { + t.Errorf("Expecting region `us-east-1` found %s", serverConfig.GetRegion()) + } + + // Set new region and verify. + serverConfig.SetRegion("us-west-1") + if serverConfig.GetRegion() != "us-west-1" { + t.Errorf("Expecting region `us-west-1` found %s", serverConfig.GetRegion()) + } + + // Set new amqp notification id. + serverConfig.SetAMQPNotifyByID("2", amqpNotify{}) + savedNotifyCfg1 := serverConfig.GetAMQPNotifyByID("2") + if !reflect.DeepEqual(savedNotifyCfg1, amqpNotify{}) { + t.Errorf("Expecting AMQP config %#v found %#v", amqpNotify{}, savedNotifyCfg1) + } + + // Set new elastic search notification id. + serverConfig.SetElasticSearchNotifyByID("2", elasticSearchNotify{}) + savedNotifyCfg2 := serverConfig.GetElasticSearchNotifyByID("2") + if !reflect.DeepEqual(savedNotifyCfg2, elasticSearchNotify{}) { + t.Errorf("Expecting Elasticsearch config %#v found %#v", elasticSearchNotify{}, savedNotifyCfg2) + } + + // Set new redis notification id. + serverConfig.SetRedisNotifyByID("2", redisNotify{}) + savedNotifyCfg3 := serverConfig.GetRedisNotifyByID("2") + if !reflect.DeepEqual(savedNotifyCfg3, redisNotify{}) { + t.Errorf("Expecting Redis config %#v found %#v", redisNotify{}, savedNotifyCfg3) + } + + // Set new console logger. + serverConfig.SetConsoleLogger(consoleLogger{ + Enable: true, + }) + consoleCfg := serverConfig.GetConsoleLogger() + if !reflect.DeepEqual(consoleCfg, consoleLogger{Enable: true}) { + t.Errorf("Expecting console logger config %#v found %#v", consoleLogger{Enable: true}, consoleCfg) + } + + // Set new file logger. + serverConfig.SetFileLogger(fileLogger{ + Enable: true, + }) + fileCfg := serverConfig.GetFileLogger() + if !reflect.DeepEqual(fileCfg, fileLogger{Enable: true}) { + t.Errorf("Expecting file logger config %#v found %#v", fileLogger{Enable: true}, consoleCfg) + } + + // Set new syslog logger. + serverConfig.SetSyslogLogger(syslogLogger{ + Enable: true, + }) + sysLogCfg := serverConfig.GetSyslogLogger() + if !reflect.DeepEqual(sysLogCfg, syslogLogger{Enable: true}) { + t.Errorf("Expecting syslog logger config %#v found %#v", syslogLogger{Enable: true}, sysLogCfg) + } + + // Match version. + if serverConfig.GetVersion() != globalMinioConfigVersion { + t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), globalMinioConfigVersion) + } + + // Attempt to save. + if err := serverConfig.Save(); err != nil { + t.Fatalf("Unable to save updated config file %s", err) + } + + // Do this only once here. + setGlobalConfigPath(rootPath) + + // Initialize server config. + if err := initConfig(); err != nil { + t.Fatalf("Unable to initialize from updated config file %s", err) + } +} diff --git a/cmd/globals.go b/cmd/globals.go index 625ba1925..154b519c9 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -30,7 +30,7 @@ const ( // minio configuration related constants. const ( - globalMinioConfigVersion = "6" + globalMinioConfigVersion = "7" globalMinioConfigDir = ".minio" globalMinioCertsDir = "certs" globalMinioCertFile = "public.crt" diff --git a/cmd/utils_test.go b/cmd/utils_test.go index 5b30bd38b..3b6215378 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -25,14 +25,14 @@ import ( // Tests http.Header clone. func TestCloneHeader(t *testing.T) { headers := []http.Header{ - http.Header{ + { "Content-Type": {"text/html; charset=UTF-8"}, "Content-Length": {"0"}, }, - http.Header{ + { "Content-Length": {"0", "1", "2"}, }, - http.Header{ + { "Expires": {"-1"}, "Content-Length": {"0"}, "Content-Encoding": {"gzip"},