Add deliveryMode parameter for AMQP notfication target (#4008)

Configuration migration was done.

Also adds documentation about AMQP configuration parameters.

Fixes #3982
This commit is contained in:
Aditya Manthramurthy 2017-03-31 16:04:26 +05:30 committed by Harshavardhana
parent 5cec6bd80d
commit 096427f973
10 changed files with 205 additions and 46 deletions

View File

@ -89,6 +89,10 @@ func migrateConfig() error {
if err := migrateV16ToV17(); err != nil { if err := migrateV16ToV17(); err != nil {
return err return err
} }
// Migration version '17' to '18'.
if err := migrateV17ToV18(); err != nil {
return err
}
return nil return nil
} }
@ -1207,3 +1211,107 @@ func migrateV16ToV17() error {
log.Printf("Migration from version %s to %s completed successfully.\n", cv16.Version, srvConfig.Version) log.Printf("Migration from version %s to %s completed successfully.\n", cv16.Version, srvConfig.Version)
return nil return nil
} }
// Version '17' to '18' migration. Adds "deliveryMode" configuration
// parameter for AMQP notification target
func migrateV17ToV18() error {
configFile := getConfigFile()
cv17 := &serverConfigV17{}
_, err := quick.Load(configFile, cv17)
if os.IsNotExist(err) {
return nil
} else if err != nil {
return fmt.Errorf("Unable to load config version 17. %v", err)
}
if cv17.Version != "17" {
return nil
}
// Copy over fields from V17 into V18 config struct
srvConfig := &serverConfigV17{
Logger: &loggers{},
Notify: &notifier{},
}
srvConfig.Version = "18"
srvConfig.Credential = cv17.Credential
srvConfig.Region = cv17.Region
if srvConfig.Region == "" {
// Region needs to be set for AWS Signature Version 4.
srvConfig.Region = globalMinioDefaultRegion
}
srvConfig.Logger.Console = cv17.Logger.Console
srvConfig.Logger.File = cv17.Logger.File
// check and set notifiers config
if len(cv17.Notify.AMQP) == 0 {
srvConfig.Notify.AMQP = make(map[string]amqpNotify)
srvConfig.Notify.AMQP["1"] = amqpNotify{}
} else {
// New deliveryMode parameter is added for AMQP,
// default value is already 0, so nothing to
// explicitly migrate here.
srvConfig.Notify.AMQP = cv17.Notify.AMQP
}
if len(cv17.Notify.ElasticSearch) == 0 {
srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
Format: formatNamespace,
}
} else {
srvConfig.Notify.ElasticSearch = cv17.Notify.ElasticSearch
}
if len(cv17.Notify.Redis) == 0 {
srvConfig.Notify.Redis = make(map[string]redisNotify)
srvConfig.Notify.Redis["1"] = redisNotify{
Format: formatNamespace,
}
} else {
srvConfig.Notify.Redis = cv17.Notify.Redis
}
if len(cv17.Notify.PostgreSQL) == 0 {
srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
Format: formatNamespace,
}
} else {
srvConfig.Notify.PostgreSQL = cv17.Notify.PostgreSQL
}
if len(cv17.Notify.Kafka) == 0 {
srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
srvConfig.Notify.Kafka["1"] = kafkaNotify{}
} else {
srvConfig.Notify.Kafka = cv17.Notify.Kafka
}
if len(cv17.Notify.NATS) == 0 {
srvConfig.Notify.NATS = make(map[string]natsNotify)
srvConfig.Notify.NATS["1"] = natsNotify{}
} else {
srvConfig.Notify.NATS = cv17.Notify.NATS
}
if len(cv17.Notify.Webhook) == 0 {
srvConfig.Notify.Webhook = make(map[string]webhookNotify)
srvConfig.Notify.Webhook["1"] = webhookNotify{}
} else {
srvConfig.Notify.Webhook = cv17.Notify.Webhook
}
if len(cv17.Notify.MySQL) == 0 {
srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
srvConfig.Notify.MySQL["1"] = mySQLNotify{
Format: formatNamespace,
}
} else {
srvConfig.Notify.MySQL = cv17.Notify.MySQL
}
// Load browser config from existing config in the file.
srvConfig.Browser = cv17.Browser
if err = quick.Save(configFile, srvConfig); err != nil {
return fmt.Errorf("Failed to migrate config from %s to %s. %v", cv17.Version, srvConfig.Version, err)
}
log.Printf("Migration from version %s to %s completed successfully.\n", cv17.Version, srvConfig.Version)
return nil
}

View File

@ -118,11 +118,14 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
if err := migrateV16ToV17(); err != nil { if err := migrateV16ToV17(); err != nil {
t.Fatal("migrate v16 to v17 should succeed when no config file is found") t.Fatal("migrate v16 to v17 should succeed when no config file is found")
} }
if err := migrateV17ToV18(); err != nil {
t.Fatal("migrate v17 to v18 should succeed when no config file is found")
}
} }
// Test if a config migration from v2 to v17 is successfully done // Test if a config migration from v2 to v18 is successfully done
func TestServerConfigMigrateV2toV16(t *testing.T) { func TestServerConfigMigrateV2toV18(t *testing.T) {
rootPath, err := newTestConfig(globalMinioDefaultRegion) rootPath, err := newTestConfig(globalMinioDefaultRegion)
if err != nil { if err != nil {
t.Fatalf("Init Test config failed") t.Fatalf("Init Test config failed")
@ -161,7 +164,7 @@ func TestServerConfigMigrateV2toV16(t *testing.T) {
} }
// Check the version number in the upgraded config file // Check the version number in the upgraded config file
expectedVersion := v17 expectedVersion := v18
if serverConfig.Version != expectedVersion { if serverConfig.Version != expectedVersion {
t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version) t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
} }
@ -238,4 +241,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
if err := migrateV16ToV17(); err == nil { if err := migrateV16ToV17(); err == nil {
t.Fatal("migrateConfigV16ToV17() should fail with a corrupted json") t.Fatal("migrateConfigV16ToV17() should fail with a corrupted json")
} }
if err := migrateV17ToV18(); err == nil {
t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
}
} }

View File

@ -430,3 +430,22 @@ type serverConfigV16 struct {
// Notification queue configuration. // Notification queue configuration.
Notify *notifier `json:"notify"` Notify *notifier `json:"notify"`
} }
// serverConfigV17 server configuration version '17' which is like
// version '16' except it adds support for "format" parameter in
// database event notification targets: PostgreSQL, MySQL, Redis and
// Elasticsearch.
type serverConfigV17 struct {
Version string `json:"version"`
// S3 API configuration.
Credential credential `json:"credential"`
Region string `json:"region"`
Browser BrowserFlag `json:"browser"`
// Additional error logging configuration.
Logger *loggers `json:"logger"`
// Notification queue configuration.
Notify *notifier `json:"notify"`
}

View File

@ -27,19 +27,18 @@ import (
) )
// Config version // Config version
const v17 = "17" const v18 = "18"
var ( var (
// serverConfig server config. // serverConfig server config.
serverConfig *serverConfigV17 serverConfig *serverConfigV18
serverConfigMu sync.RWMutex serverConfigMu sync.RWMutex
) )
// serverConfigV17 server configuration version '17' which is like // serverConfigV18 server configuration version '18' which is like
// version '16' except it adds support for "format" parameter in // version '17' except it adds support for "deliveryMode" parameter in
// database event notification targets: PostgreSQL, MySQL, Redis and // the AMQP notification target.
// Elasticsearch. type serverConfigV18 struct {
type serverConfigV17 struct {
sync.RWMutex sync.RWMutex
Version string `json:"version"` Version string `json:"version"`
@ -56,7 +55,7 @@ type serverConfigV17 struct {
} }
// GetVersion get current config version. // GetVersion get current config version.
func (s *serverConfigV17) GetVersion() string { func (s *serverConfigV18) GetVersion() string {
s.RLock() s.RLock()
defer s.RUnlock() defer s.RUnlock()
@ -64,7 +63,7 @@ func (s *serverConfigV17) GetVersion() string {
} }
// SetRegion set new region. // SetRegion set new region.
func (s *serverConfigV17) SetRegion(region string) { func (s *serverConfigV18) SetRegion(region string) {
s.Lock() s.Lock()
defer s.Unlock() defer s.Unlock()
@ -72,7 +71,7 @@ func (s *serverConfigV17) SetRegion(region string) {
} }
// GetRegion get current region. // GetRegion get current region.
func (s *serverConfigV17) GetRegion() string { func (s *serverConfigV18) GetRegion() string {
s.RLock() s.RLock()
defer s.RUnlock() defer s.RUnlock()
@ -80,7 +79,7 @@ func (s *serverConfigV17) GetRegion() string {
} }
// SetCredentials set new credentials. // SetCredentials set new credentials.
func (s *serverConfigV17) SetCredential(creds credential) { func (s *serverConfigV18) SetCredential(creds credential) {
s.Lock() s.Lock()
defer s.Unlock() defer s.Unlock()
@ -89,7 +88,7 @@ func (s *serverConfigV17) SetCredential(creds credential) {
} }
// GetCredentials get current credentials. // GetCredentials get current credentials.
func (s *serverConfigV17) GetCredential() credential { func (s *serverConfigV18) GetCredential() credential {
s.RLock() s.RLock()
defer s.RUnlock() defer s.RUnlock()
@ -97,7 +96,7 @@ func (s *serverConfigV17) GetCredential() credential {
} }
// SetBrowser set if browser is enabled. // SetBrowser set if browser is enabled.
func (s *serverConfigV17) SetBrowser(b bool) { func (s *serverConfigV18) SetBrowser(b bool) {
s.Lock() s.Lock()
defer s.Unlock() defer s.Unlock()
@ -106,7 +105,7 @@ func (s *serverConfigV17) SetBrowser(b bool) {
} }
// GetCredentials get current credentials. // GetCredentials get current credentials.
func (s *serverConfigV17) GetBrowser() bool { func (s *serverConfigV18) GetBrowser() bool {
s.RLock() s.RLock()
defer s.RUnlock() defer s.RUnlock()
@ -114,7 +113,7 @@ func (s *serverConfigV17) GetBrowser() bool {
} }
// Save config. // Save config.
func (s *serverConfigV17) Save() error { func (s *serverConfigV18) Save() error {
s.RLock() s.RLock()
defer s.RUnlock() defer s.RUnlock()
@ -122,9 +121,9 @@ func (s *serverConfigV17) Save() error {
return quick.Save(getConfigFile(), s) return quick.Save(getConfigFile(), s)
} }
func newServerConfigV17() *serverConfigV17 { func newServerConfigV18() *serverConfigV18 {
srvCfg := &serverConfigV17{ srvCfg := &serverConfigV18{
Version: v17, Version: v18,
Credential: mustGetNewCredential(), Credential: mustGetNewCredential(),
Region: globalMinioDefaultRegion, Region: globalMinioDefaultRegion,
Browser: true, Browser: true,
@ -160,12 +159,13 @@ func newServerConfigV17() *serverConfigV17 {
// found, otherwise use default parameters // found, otherwise use default parameters
func newConfig() error { func newConfig() error {
// Initialize server config. // Initialize server config.
srvCfg := newServerConfigV17() srvCfg := newServerConfigV18()
// If env is set override the credentials from config file. // If env is set override the credentials from config file.
if globalIsEnvCreds { if globalIsEnvCreds {
srvCfg.SetCredential(globalActiveCred) srvCfg.SetCredential(globalActiveCred)
} }
if globalIsEnvBrowser { if globalIsEnvBrowser {
srvCfg.SetBrowser(globalIsBrowserEnabled) srvCfg.SetBrowser(globalIsBrowserEnabled)
} }
@ -233,8 +233,8 @@ func checkDupJSONKeys(json string) error {
} }
// getValidConfig - returns valid server configuration // getValidConfig - returns valid server configuration
func getValidConfig() (*serverConfigV17, error) { func getValidConfig() (*serverConfigV18, error) {
srvCfg := &serverConfigV17{ srvCfg := &serverConfigV18{
Region: globalMinioDefaultRegion, Region: globalMinioDefaultRegion,
Browser: true, Browser: true,
} }
@ -244,8 +244,8 @@ func getValidConfig() (*serverConfigV17, error) {
return nil, err return nil, err
} }
if srvCfg.Version != v17 { if srvCfg.Version != v18 {
return nil, fmt.Errorf("configuration version mismatch. Expected: %s, Got: %s", v17, srvCfg.Version) return nil, fmt.Errorf("configuration version mismatch. Expected: %s, Got: %s", v18, srvCfg.Version)
} }
// Load config file json and check for duplication json keys // Load config file json and check for duplication json keys

View File

@ -109,8 +109,8 @@ func TestServerConfig(t *testing.T) {
serverConfig.Logger.SetFile(fileLogger) serverConfig.Logger.SetFile(fileLogger)
// Match version. // Match version.
if serverConfig.GetVersion() != v17 { if serverConfig.GetVersion() != v18 {
t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v17) t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v18)
} }
// Attempt to save. // Attempt to save.
@ -217,7 +217,7 @@ func TestValidateConfig(t *testing.T) {
configPath := filepath.Join(rootPath, minioConfigFile) configPath := filepath.Join(rootPath, minioConfigFile)
v := v17 v := v18
testCases := []struct { testCases := []struct {
configData string configData string

View File

@ -96,7 +96,7 @@ func newGatewayLayer(backendType, accessKey, secretKey string) (GatewayLayer, er
// only used in memory. // only used in memory.
func newGatewayConfig(accessKey, secretKey, region string) error { func newGatewayConfig(accessKey, secretKey, region string) error {
// Initialize server config. // Initialize server config.
srvCfg := newServerConfigV17() srvCfg := newServerConfigV18()
// If env is set for a fresh start, save them to config file. // If env is set for a fresh start, save them to config file.
srvCfg.SetCredential(credential{ srvCfg.SetCredential(credential{

View File

@ -32,6 +32,7 @@ type amqpNotify struct {
Exchange string `json:"exchange"` Exchange string `json:"exchange"`
RoutingKey string `json:"routingKey"` RoutingKey string `json:"routingKey"`
ExchangeType string `json:"exchangeType"` ExchangeType string `json:"exchangeType"`
DeliveryMode uint8 `json:"deliveryMode"`
Mandatory bool `json:"mandatory"` Mandatory bool `json:"mandatory"`
Immediate bool `json:"immediate"` Immediate bool `json:"immediate"`
Durable bool `json:"durable"` Durable bool `json:"durable"`
@ -145,8 +146,9 @@ func (q amqpConn) Fire(entry *logrus.Entry) error {
q.params.Mandatory, q.params.Mandatory,
q.params.Immediate, q.params.Immediate,
amqp.Publishing{ amqp.Publishing{
ContentType: "application/json", ContentType: "application/json",
Body: []byte(body), DeliveryMode: q.params.DeliveryMode,
Body: []byte(body),
}) })
if err != nil { if err != nil {
return err return err

View File

@ -26,27 +26,50 @@ Install RabbitMQ from [here](https://www.rabbitmq.com/).
### Step 1: Add AMQP endpoint to Minio ### Step 1: Add AMQP endpoint to Minio
The default location of Minio server configuration file is ``~/.minio/config.json``. Update the AMQP configuration block in ``config.json`` as follows: The default location of Minio server configuration file is ``~/.minio/config.json``. The AMQP configuration is located in the `amqp` key under the `notify` top-level key. Create a configuration key-value pair here for your AMQP instance. The key is a name for your AMQP endpoint, and the value is a collection of key-value parameters described in the table below.
| Parameter | Type | Description |
|:---|:---|:---|
| `enable` | _bool_ | (Required) Is this server endpoint configuration active/enabled? |
| `url` | _string_ | (Required) AMQP server endpoint, e.g. `amqp://myuser:mypassword@localhost:5672` |
| `exchange` | _string_ | Name of the exchange. |
| `routingKey` | _string_ | Routing key for publishing. |
| `exchangeType` | _string_ | Kind of exchange. |
| `deliveryMode` | _uint8_ | Delivery mode for publishing. 0 or 1 - transient; 2 - persistent. |
| `mandatory` | _bool_ | Publishing related bool. |
| `immediate` | _bool_ | Publishing related bool. |
| `durable` | _bool_ | Exchange declaration related bool. |
| `internal` | _bool_ | Exchange declaration related bool. |
| `noWait` | _bool_ | Exchange declaration related bool. |
| `autoDeleted` | _bool_ | Exchange declaration related bool. |
An example configuration for RabbitMQ is shown below:
```json ```json
"amqp": { "amqp": {
"1": { "1": {
"enable": true, "enable": true,
"url": "amqp://myuser:mypassword@localhost:5672", "url": "amqp://myuser:mypassword@localhost:5672",
"exchange": "bucketevents", "exchange": "bucketevents",
"routingKey": "bucketlogs", "routingKey": "bucketlogs",
"exchangeType": "fanout", "exchangeType": "fanout",
"mandatory": false, "deliveryMode": 0,
"immediate": false, "mandatory": false,
"durable": false, "immediate": false,
"internal": false, "durable": false,
"noWait": false, "internal": false,
"autoDeleted": false "noWait": false,
"autoDeleted": false
} }
} }
``` ```
Restart Minio server to reflect config changes. Minio supports all the exchanges available in [RabbitMQ](https://www.rabbitmq.com/). For this setup, we are using ``fanout`` exchange. After updating the configuration file, restart the Minio server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs:us-east-1:1:amqp` at start-up if there were no errors.
Minio supports all the exchanges available in [RabbitMQ](https://www.rabbitmq.com/). For this setup, we are using ``fanout`` exchange.
Note that, you can add as many AMQP server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the AMQP instance and an object of per-server configuration parameters.
### Step 2: Enable bucket notification using Minio client ### Step 2: Enable bucket notification using Minio client

View File

@ -1,4 +1,4 @@
# Minio Server `config.json` (v17) Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio) # Minio Server `config.json` (v18) Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
Minio server stores all its configuration data in `${HOME}/.minio/config.json` file by default. Following sections provide detailed explanation of each fields and how to customize them. A complete example of `config.json` is available [here](https://raw.githubusercontent.com/minio/minio/master/docs/config/config.sample.json) Minio server stores all its configuration data in `${HOME}/.minio/config.json` file by default. Following sections provide detailed explanation of each fields and how to customize them. A complete example of `config.json` is available [here](https://raw.githubusercontent.com/minio/minio/master/docs/config/config.sample.json)

View File

@ -23,6 +23,7 @@
"exchange": "bucketevents", "exchange": "bucketevents",
"routingKey": "bucketlogs", "routingKey": "bucketlogs",
"exchangeType": "fanout", "exchangeType": "fanout",
"deliveryMode": 0,
"mandatory": false, "mandatory": false,
"immediate": false, "immediate": false,
"durable": false, "durable": false,