Migrate config to KV data format (#8392)

- adding oauth support to MinIO browser (#8400) by @kanagaraj - supports multi-line get/set/del for all config fields - add support for comments, allow toggle - add extensive validation of config before saving - support MinIO browser to support proper claims, using STS tokens - env support for all config parameters, legacy envs are also supported with all documentation now pointing to latest ENVs - preserve accessKey/secretKey from FS mode setups - add history support implements three APIs - ClearHistory - RestoreHistory - ListHistory - add help command support for each config parameters - all the bug fixes after migration to KV, and other bug fixes encountered during testing.
2025-11-09 21:49:46 -05:00 · 2019-10-22 22:59:13 -07:00
parent 8836d57e3c
commit ee4a6a823d
185 changed files with 8228 additions and 3597 deletions
--- a/pkg/event/target/amqp.go
+++ b/pkg/event/target/amqp.go
@@ -49,6 +49,43 @@ type AMQPArgs struct {
 	QueueLimit   uint64   `json:"queueLimit"`
 }

+// AMQP input constants.
+const (
+	AmqpQueueDir   = "queue_dir"
+	AmqpQueueLimit = "queue_limit"
+
+	AmqpURL               = "url"
+	AmqpExchange          = "exchange"
+	AmqpRoutingKey        = "routing_key"
+	AmqpExchangeType      = "exchange_type"
+	AmqpDeliveryMode      = "delivery_mode"
+	AmqpMandatory         = "mandatory"
+	AmqpImmediate         = "immediate"
+	AmqpDurable           = "durable"
+	AmqpInternal          = "internal"
+	AmqpNoWait            = "no_wait"
+	AmqpAutoDeleted       = "auto_deleted"
+	AmqpArguments         = "arguments"
+	AmqpPublishingHeaders = "publishing_headers"
+
+	EnvAMQPState             = "MINIO_NOTIFY_AMQP_STATE"
+	EnvAMQPURL               = "MINIO_NOTIFY_AMQP_URL"
+	EnvAMQPExchange          = "MINIO_NOTIFY_AMQP_EXCHANGE"
+	EnvAMQPRoutingKey        = "MINIO_NOTIFY_AMQP_ROUTING_KEY"
+	EnvAMQPExchangeType      = "MINIO_NOTIFY_AMQP_EXCHANGE_TYPE"
+	EnvAMQPDeliveryMode      = "MINIO_NOTIFY_AMQP_DELIVERY_MODE"
+	EnvAMQPMandatory         = "MINIO_NOTIFY_AMQP_MANDATORY"
+	EnvAMQPImmediate         = "MINIO_NOTIFY_AMQP_IMMEDIATE"
+	EnvAMQPDurable           = "MINIO_NOTIFY_AMQP_DURABLE"
+	EnvAMQPInternal          = "MINIO_NOTIFY_AMQP_INTERNAL"
+	EnvAMQPNoWait            = "MINIO_NOTIFY_AMQP_NO_WAIT"
+	EnvAMQPAutoDeleted       = "MINIO_NOTIFY_AMQP_AUTO_DELETED"
+	EnvAMQPArguments         = "MINIO_NOTIFY_AMQP_ARGUMENTS"
+	EnvAMQPPublishingHeaders = "MINIO_NOTIFY_AMQP_PUBLISHING_HEADERS"
+	EnvAMQPQueueDir          = "MINIO_NOTIFY_AMQP_QUEUE_DIR"
+	EnvAMQPQueueLimit        = "MINIO_NOTIFY_AMQP_QUEUE_LIMIT"
+)
+
 // Validate AMQP arguments
 func (a *AMQPArgs) Validate() error {
 	if !a.Enable {
--- a/pkg/event/target/elasticsearch.go
+++ b/pkg/event/target/elasticsearch.go
@@ -31,6 +31,22 @@ import (
 	"gopkg.in/olivere/elastic.v5"
 )

+// Elastic constants
+const (
+	ElasticFormat     = "format"
+	ElasticURL        = "url"
+	ElasticIndex      = "index"
+	ElasticQueueDir   = "queue_dir"
+	ElasticQueueLimit = "queue_limit"
+
+	EnvElasticState      = "MINIO_NOTIFY_ELASTICSEARCH_STATE"
+	EnvElasticFormat     = "MINIO_NOTIFY_ELASTICSEARCH_FORMAT"
+	EnvElasticURL        = "MINIO_NOTIFY_ELASTICSEARCH_URL"
+	EnvElasticIndex      = "MINIO_NOTIFY_ELASTICSEARCH_INDEX"
+	EnvElasticQueueDir   = "MINIO_NOTIFY_ELASTICSEARCH_QUEUE_DIR"
+	EnvElasticQueueLimit = "MINIO_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT"
+)
+
 // ElasticsearchArgs - Elasticsearch target arguments.
 type ElasticsearchArgs struct {
 	Enable     bool     `json:"enable"`
--- a/pkg/event/target/kafka.go
+++ b/pkg/event/target/kafka.go
@@ -33,6 +33,32 @@ import (
 	sarama "gopkg.in/Shopify/sarama.v1"
 )

+// MQTT input constants
+const (
+	KafkaBrokers       = "brokers"
+	KafkaTopic         = "topic"
+	KafkaQueueDir      = "queue_dir"
+	KafkaQueueLimit    = "queue_limit"
+	KafkaTLSEnable     = "tls_enable"
+	KafkaTLSSkipVerify = "tls_skip_verify"
+	KafkaTLSClientAuth = "tls_client_auth"
+	KafkaSASLEnable    = "sasl_enable"
+	KafkaSASLUsername  = "sasl_username"
+	KafkaSASLPassword  = "sasl_password"
+
+	EnvKafkaState         = "MINIO_NOTIFY_KAFKA_STATE"
+	EnvKafkaBrokers       = "MINIO_NOTIFY_KAFKA_BROKERS"
+	EnvKafkaTopic         = "MINIO_NOTIFY_KAFKA_TOPIC"
+	EnvKafkaQueueDir      = "MINIO_NOTIFY_KAFKA_QUEUE_DIR"
+	EnvKafkaQueueLimit    = "MINIO_NOTIFY_KAFKA_QUEUE_LIMIT"
+	EnvKafkaTLSEnable     = "MINIO_NOTIFY_KAFKA_TLS_ENABLE"
+	EnvKafkaTLSSkipVerify = "MINIO_NOTIFY_KAFKA_TLS_SKIP_VERIFY"
+	EnvKafkaTLSClientAuth = "MINIO_NOTIFY_KAFKA_TLS_CLIENT_AUTH"
+	EnvKafkaSASLEnable    = "MINIO_NOTIFY_KAFKA_SASL_ENABLE"
+	EnvKafkaSASLUsername  = "MINIO_NOTIFY_KAFKA_SASL_USERNAME"
+	EnvKafkaSASLPassword  = "MINIO_NOTIFY_KAFKA_SASL_PASSWORD"
+)
+
 // KafkaArgs - Kafka target arguments.
 type KafkaArgs struct {
 	Enable     bool        `json:"enable"`
--- a/pkg/event/target/mqtt.go
+++ b/pkg/event/target/mqtt.go
@@ -38,6 +38,30 @@ const (
 	storePrefix       = "minio"
 )

+// MQTT input constants
+const (
+	MqttBroker            = "broker"
+	MqttTopic             = "topic"
+	MqttQoS               = "qos"
+	MqttUsername          = "username"
+	MqttPassword          = "password"
+	MqttReconnectInterval = "reconnect_interval"
+	MqttKeepAliveInterval = "keep_alive_interval"
+	MqttQueueDir          = "queue_dir"
+	MqttQueueLimit        = "queue_limit"
+
+	EnvMQTTState             = "MINIO_NOTIFY_MQTT_STATE"
+	EnvMQTTBroker            = "MINIO_NOTIFY_MQTT_BROKER"
+	EnvMQTTTopic             = "MINIO_NOTIFY_MQTT_TOPIC"
+	EnvMQTTQoS               = "MINIO_NOTIFY_MQTT_QOS"
+	EnvMQTTUsername          = "MINIO_NOTIFY_MQTT_USERNAME"
+	EnvMQTTPassword          = "MINIO_NOTIFY_MQTT_PASSWORD"
+	EnvMQTTReconnectInterval = "MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL"
+	EnvMQTTKeepAliveInterval = "MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL"
+	EnvMQTTQueueDir          = "MINIO_NOTIFY_MQTT_QUEUE_DIR"
+	EnvMQTTQueueLimit        = "MINIO_NOTIFY_MQTT_QUEUE_LIMIT"
+)
+
 // MQTTArgs - MQTT target arguments.
 type MQTTArgs struct {
 	Enable               bool           `json:"enable"`
--- a/pkg/event/target/mysql.go
+++ b/pkg/event/target/mysql.go
@@ -81,6 +81,32 @@ const (
 	mysqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES (?, ?);`
 )

+// MySQL related constants
+const (
+	MySQLFormat     = "format"
+	MySQLDSNString  = "dsn_string"
+	MySQLTable      = "table"
+	MySQLHost       = "host"
+	MySQLPort       = "port"
+	MySQLUsername   = "username"
+	MySQLPassword   = "password"
+	MySQLDatabase   = "database"
+	MySQLQueueLimit = "queue_limit"
+	MySQLQueueDir   = "queue_dir"
+
+	EnvMySQLState      = "MINIO_NOTIFY_MYSQL_STATE"
+	EnvMySQLFormat     = "MINIO_NOTIFY_MYSQL_FORMAT"
+	EnvMySQLDSNString  = "MINIO_NOTIFY_MYSQL_DSN_STRING"
+	EnvMySQLTable      = "MINIO_NOTIFY_MYSQL_TABLE"
+	EnvMySQLHost       = "MINIO_NOTIFY_MYSQL_HOST"
+	EnvMySQLPort       = "MINIO_NOTIFY_MYSQL_PORT"
+	EnvMySQLUsername   = "MINIO_NOTIFY_MYSQL_USERNAME"
+	EnvMySQLPassword   = "MINIO_NOTIFY_MYSQL_PASSWORD"
+	EnvMySQLDatabase   = "MINIO_NOTIFY_MYSQL_DATABASE"
+	EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT"
+	EnvMySQLQueueDir   = "MINIO_NOTIFY_MYSQL_QUEUE_DIR"
+)
+
 // MySQLArgs - MySQL target arguments.
 type MySQLArgs struct {
 	Enable     bool     `json:"enable"`
--- a/pkg/event/target/nats.go
+++ b/pkg/event/target/nats.go
@@ -30,6 +30,42 @@ import (
 	"github.com/nats-io/stan.go"
 )

+// NATS related constants
+const (
+	NATSAddress      = "address"
+	NATSSubject      = "subject"
+	NATSUsername     = "username"
+	NATSPassword     = "password"
+	NATSToken        = "token"
+	NATSSecure       = "secure"
+	NATSPingInterval = "ping_interval"
+	NATSQueueDir     = "queue_dir"
+	NATSQueueLimit   = "queue_limit"
+
+	// Streaming constants
+	NATSStreamingEnable             = "streaming_enable"
+	NATSStreamingClusterID          = "streaming_cluster_id"
+	NATSStreamingAsync              = "streaming_async"
+	NATSStreamingMaxPubAcksInFlight = "streaming_max_pub_acks_in_flight"
+
+	EnvNATSState        = "MINIO_NOTIFY_NATS_STATE"
+	EnvNATSAddress      = "MINIO_NOTIFY_NATS_ADDRESS"
+	EnvNATSSubject      = "MINIO_NOTIFY_NATS_SUBJECT"
+	EnvNATSUsername     = "MINIO_NOTIFY_NATS_USERNAME"
+	EnvNATSPassword     = "MINIO_NOTIFY_NATS_PASSWORD"
+	EnvNATSToken        = "MINIO_NOTIFY_NATS_TOKEN"
+	EnvNATSSecure       = "MINIO_NOTIFY_NATS_SECURE"
+	EnvNATSPingInterval = "MINIO_NOTIFY_NATS_PING_INTERVAL"
+	EnvNATSQueueDir     = "MINIO_NOTIFY_NATS_QUEUE_DIR"
+	EnvNATSQueueLimit   = "MINIO_NOTIFY_NATS_QUEUE_LIMIT"
+
+	// Streaming constants
+	EnvNATSStreamingEnable             = "MINIO_NOTIFY_NATS_STREAMING_ENABLE"
+	EnvNATSStreamingClusterID          = "MINIO_NOTIFY_NATS_STREAMING_CLUSTER_ID"
+	EnvNATSStreamingAsync              = "MINIO_NOTIFY_NATS_STREAMING_ASYNC"
+	EnvNATSStreamingMaxPubAcksInFlight = "MINIO_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT"
+)
+
 // NATSArgs - NATS target arguments.
 type NATSArgs struct {
 	Enable       bool      `json:"enable"`
--- a/pkg/event/target/nsq.go
+++ b/pkg/event/target/nsq.go
@@ -31,6 +31,24 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )

+// NSQ constants
+const (
+	NSQAddress       = "nsqd_address"
+	NSQTopic         = "topic"
+	NSQTLSEnable     = "tls_enable"
+	NSQTLSSkipVerify = "tls_skip_verify"
+	NSQQueueDir      = "queue_dir"
+	NSQQueueLimit    = "queue_limit"
+
+	EnvNSQState         = "MINIO_NOTIFY_NSQ"
+	EnvNSQAddress       = "MINIO_NOTIFY_NSQ_NSQD_ADDRESS"
+	EnvNSQTopic         = "MINIO_NOTIFY_NSQ_TOPIC"
+	EnvNSQTLSEnable     = "MINIO_NOTIFY_NSQ_TLS_ENABLE"
+	EnvNSQTLSSkipVerify = "MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY"
+	EnvNSQQueueDir      = "MINIO_NOTIFY_NSQ_QUEUE_DIR"
+	EnvNSQQueueLimit    = "MINIO_NOTIFY_NSQ_QUEUE_LIMIT"
+)
+
 // NSQArgs - NSQ target arguments.
 type NSQArgs struct {
 	Enable      bool      `json:"enable"`
--- a/pkg/event/target/postgresql.go
+++ b/pkg/event/target/postgresql.go
@@ -82,6 +82,32 @@ const (
 	psqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES ($1, $2);`
 )

+// Postgres constants
+const (
+	PostgresFormat           = "format"
+	PostgresConnectionString = "connection_string"
+	PostgresTable            = "table"
+	PostgresHost             = "host"
+	PostgresPort             = "port"
+	PostgresUsername         = "username"
+	PostgresPassword         = "password"
+	PostgresDatabase         = "database"
+	PostgresQueueDir         = "queue_dir"
+	PostgresQueueLimit       = "queue_limit"
+
+	EnvPostgresState            = "MINIO_NOTIFY_POSTGRES_STATE"
+	EnvPostgresFormat           = "MINIO_NOTIFY_POSTGRES_FORMAT"
+	EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING"
+	EnvPostgresTable            = "MINIO_NOTIFY_POSTGRES_TABLE"
+	EnvPostgresHost             = "MINIO_NOTIFY_POSTGRES_HOST"
+	EnvPostgresPort             = "MINIO_NOTIFY_POSTGRES_PORT"
+	EnvPostgresUsername         = "MINIO_NOTIFY_POSTGRES_USERNAME"
+	EnvPostgresPassword         = "MINIO_NOTIFY_POSTGRES_PASSWORD"
+	EnvPostgresDatabase         = "MINIO_NOTIFY_POSTGRES_DATABASE"
+	EnvPostgresQueueDir         = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR"
+	EnvPostgresQueueLimit       = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT"
+)
+
 // PostgreSQLArgs - PostgreSQL target arguments.
 type PostgreSQLArgs struct {
 	Enable           bool      `json:"enable"`
--- a/pkg/event/target/redis.go
+++ b/pkg/event/target/redis.go
@@ -32,6 +32,24 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )

+// Redis constants
+const (
+	RedisFormat     = "format"
+	RedisAddress    = "address"
+	RedisPassword   = "password"
+	RedisKey        = "key"
+	RedisQueueDir   = "queue_dir"
+	RedisQueueLimit = "queue_limit"
+
+	EnvRedisState      = "MINIO_NOTIFY_REDIS_STATE"
+	EnvRedisFormat     = "MINIO_NOTIFY_REDIS_FORMAT"
+	EnvRedisAddress    = "MINIO_NOTIFY_REDIS_ADDRESS"
+	EnvRedisPassword   = "MINIO_NOTIFY_REDIS_PASSWORD"
+	EnvRedisKey        = "MINIO_NOTIFY_REDIS_KEY"
+	EnvRedisQueueDir   = "MINIO_NOTIFY_REDIS_QUEUE_DIR"
+	EnvRedisQueueLimit = "MINIO_NOTIFY_REDIS_QUEUE_LIMIT"
+)
+
 // RedisArgs - Redis target arguments.
 type RedisArgs struct {
 	Enable     bool      `json:"enable"`
--- a/pkg/event/target/webhook.go
+++ b/pkg/event/target/webhook.go
@@ -37,10 +37,25 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )

+// Webhook constants
+const (
+	WebhookEndpoint   = "endpoint"
+	WebhookAuthToken  = "auth_token"
+	WebhookQueueDir   = "queue_dir"
+	WebhookQueueLimit = "queue_limit"
+
+	EnvWebhookState      = "MINIO_NOTIFY_WEBHOOK_STATE"
+	EnvWebhookEndpoint   = "MINIO_NOTIFY_WEBHOOK_ENDPOINT"
+	EnvWebhookAuthToken  = "MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN"
+	EnvWebhookQueueDir   = "MINIO_NOTIFY_WEBHOOK_QUEUE_DIR"
+	EnvWebhookQueueLimit = "MINIO_NOTIFY_WEBHOOK_QUEUE_LIMIT"
+)
+
 // WebhookArgs - Webhook target arguments.
 type WebhookArgs struct {
 	Enable     bool           `json:"enable"`
 	Endpoint   xnet.URL       `json:"endpoint"`
+	AuthToken  string         `json:"authToken"`
 	RootCAs    *x509.CertPool `json:"-"`
 	QueueDir   string         `json:"queueDir"`
 	QueueLimit uint64         `json:"queueLimit"`
@@ -114,6 +129,10 @@ func (target *WebhookTarget) send(eventData event.Event) error {
 		return err
 	}

+	if target.args.AuthToken != "" {
+		req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
+	}
+
 	req.Header.Set("Content-Type", "application/json")

 	resp, err := target.httpClient.Do(req)
@@ -172,15 +191,15 @@ func (target *WebhookTarget) Close() error {
 }

 // NewWebhookTarget - creates new Webhook target.
-func NewWebhookTarget(id string, args WebhookArgs, doneCh <-chan struct{}, loggerOnce func(ctx context.Context, err error, id interface{}, kind ...interface{})) *WebhookTarget {
+func NewWebhookTarget(id string, args WebhookArgs, doneCh <-chan struct{}, loggerOnce func(ctx context.Context, err error, id interface{}, kind ...interface{})) (*WebhookTarget, error) {

 	var store Store

 	if args.QueueDir != "" {
 		queueDir := filepath.Join(args.QueueDir, storePrefix+"-webhook-"+id)
 		store = NewQueueStore(queueDir, args.QueueLimit)
-		if oErr := store.Open(); oErr != nil {
-			return nil
+		if err := store.Open(); err != nil {
+			return nil, err
 		}
 	}

@@ -209,5 +228,5 @@ func NewWebhookTarget(id string, args WebhookArgs, doneCh <-chan struct{}, logge
 		go sendEvents(target, eventKeyCh, doneCh, loggerOnce)
 	}

-	return target
+	return target, nil
 }