MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 21:49:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

Migrate config to KV data format (#8392)

Browse Source 
- adding oauth support to MinIO browser (#8400) by @kanagaraj
- supports multi-line get/set/del for all config fields
- add support for comments, allow toggle
- add extensive validation of config before saving
- support MinIO browser to support proper claims, using STS tokens
- env support for all config parameters, legacy envs are also
  supported with all documentation now pointing to latest ENVs
- preserve accessKey/secretKey from FS mode setups
- add history support implements three APIs
  - ClearHistory
  - RestoreHistory
  - ListHistory
- add help command support for each config parameters
- all the bug fixes after migration to KV, and other bug
  fixes encountered during testing.
This commit is contained in:
Harshavardhana
2019-10-22 22:59:13 -07:00
committed by kannappanr
parent 8836d57e3c
commit ee4a6a823d
185 changed files with 8228 additions and 3597 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/event/config.go
View File

@@ -167,6 +167,13 @@ func (q *Queue) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Validate - checks whether queue has valid values or not.
func (q Queue) Validate(region string, targetList *TargetList) error {
if q.ARN.region == "" {
if !targetList.Exists(q.ARN.TargetID) {
return &ErrARNNotFound{q.ARN}
}
return nil
}
if region != "" && q.ARN.region != region {
return &ErrUnknownRegion{q.ARN.region}
}

37
pkg/event/target/amqp.go
View File

@@ -49,6 +49,43 @@ type AMQPArgs struct {
QueueLimit uint64 `json:"queueLimit"`
}
// AMQP input constants.
const (
AmqpQueueDir = "queue_dir"
AmqpQueueLimit = "queue_limit"
AmqpURL = "url"
AmqpExchange = "exchange"
AmqpRoutingKey = "routing_key"
AmqpExchangeType = "exchange_type"
AmqpDeliveryMode = "delivery_mode"
AmqpMandatory = "mandatory"
AmqpImmediate = "immediate"
AmqpDurable = "durable"
AmqpInternal = "internal"
AmqpNoWait = "no_wait"
AmqpAutoDeleted = "auto_deleted"
AmqpArguments = "arguments"
AmqpPublishingHeaders = "publishing_headers"
EnvAMQPState = "MINIO_NOTIFY_AMQP_STATE"
EnvAMQPURL = "MINIO_NOTIFY_AMQP_URL"
EnvAMQPExchange = "MINIO_NOTIFY_AMQP_EXCHANGE"
EnvAMQPRoutingKey = "MINIO_NOTIFY_AMQP_ROUTING_KEY"
EnvAMQPExchangeType = "MINIO_NOTIFY_AMQP_EXCHANGE_TYPE"
EnvAMQPDeliveryMode = "MINIO_NOTIFY_AMQP_DELIVERY_MODE"
EnvAMQPMandatory = "MINIO_NOTIFY_AMQP_MANDATORY"
EnvAMQPImmediate = "MINIO_NOTIFY_AMQP_IMMEDIATE"
EnvAMQPDurable = "MINIO_NOTIFY_AMQP_DURABLE"
EnvAMQPInternal = "MINIO_NOTIFY_AMQP_INTERNAL"
EnvAMQPNoWait = "MINIO_NOTIFY_AMQP_NO_WAIT"
EnvAMQPAutoDeleted = "MINIO_NOTIFY_AMQP_AUTO_DELETED"
EnvAMQPArguments = "MINIO_NOTIFY_AMQP_ARGUMENTS"
EnvAMQPPublishingHeaders = "MINIO_NOTIFY_AMQP_PUBLISHING_HEADERS"
EnvAMQPQueueDir = "MINIO_NOTIFY_AMQP_QUEUE_DIR"
EnvAMQPQueueLimit = "MINIO_NOTIFY_AMQP_QUEUE_LIMIT"
)
// Validate AMQP arguments
func (a *AMQPArgs) Validate() error {
if !a.Enable {

16
pkg/event/target/elasticsearch.go
View File

@@ -31,6 +31,22 @@ import (
"gopkg.in/olivere/elastic.v5"
)
// Elastic constants
const (
ElasticFormat = "format"
ElasticURL = "url"
ElasticIndex = "index"
ElasticQueueDir = "queue_dir"
ElasticQueueLimit = "queue_limit"
EnvElasticState = "MINIO_NOTIFY_ELASTICSEARCH_STATE"
EnvElasticFormat = "MINIO_NOTIFY_ELASTICSEARCH_FORMAT"
EnvElasticURL = "MINIO_NOTIFY_ELASTICSEARCH_URL"
EnvElasticIndex = "MINIO_NOTIFY_ELASTICSEARCH_INDEX"
EnvElasticQueueDir = "MINIO_NOTIFY_ELASTICSEARCH_QUEUE_DIR"
EnvElasticQueueLimit = "MINIO_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT"
)
// ElasticsearchArgs - Elasticsearch target arguments.
type ElasticsearchArgs struct {
Enable bool `json:"enable"`

26
pkg/event/target/kafka.go
View File

@@ -33,6 +33,32 @@ import (
sarama "gopkg.in/Shopify/sarama.v1"
)
// MQTT input constants
const (
KafkaBrokers = "brokers"
KafkaTopic = "topic"
KafkaQueueDir = "queue_dir"
KafkaQueueLimit = "queue_limit"
KafkaTLSEnable = "tls_enable"
KafkaTLSSkipVerify = "tls_skip_verify"
KafkaTLSClientAuth = "tls_client_auth"
KafkaSASLEnable = "sasl_enable"
KafkaSASLUsername = "sasl_username"
KafkaSASLPassword = "sasl_password"
EnvKafkaState = "MINIO_NOTIFY_KAFKA_STATE"
EnvKafkaBrokers = "MINIO_NOTIFY_KAFKA_BROKERS"
EnvKafkaTopic = "MINIO_NOTIFY_KAFKA_TOPIC"
EnvKafkaQueueDir = "MINIO_NOTIFY_KAFKA_QUEUE_DIR"
EnvKafkaQueueLimit = "MINIO_NOTIFY_KAFKA_QUEUE_LIMIT"
EnvKafkaTLSEnable = "MINIO_NOTIFY_KAFKA_TLS_ENABLE"
EnvKafkaTLSSkipVerify = "MINIO_NOTIFY_KAFKA_TLS_SKIP_VERIFY"
EnvKafkaTLSClientAuth = "MINIO_NOTIFY_KAFKA_TLS_CLIENT_AUTH"
EnvKafkaSASLEnable = "MINIO_NOTIFY_KAFKA_SASL_ENABLE"
EnvKafkaSASLUsername = "MINIO_NOTIFY_KAFKA_SASL_USERNAME"
EnvKafkaSASLPassword = "MINIO_NOTIFY_KAFKA_SASL_PASSWORD"
)
// KafkaArgs - Kafka target arguments.
type KafkaArgs struct {
Enable bool `json:"enable"`

24
pkg/event/target/mqtt.go
View File

@@ -38,6 +38,30 @@ const (
storePrefix = "minio"
)
// MQTT input constants
const (
MqttBroker = "broker"
MqttTopic = "topic"
MqttQoS = "qos"
MqttUsername = "username"
MqttPassword = "password"
MqttReconnectInterval = "reconnect_interval"
MqttKeepAliveInterval = "keep_alive_interval"
MqttQueueDir = "queue_dir"
MqttQueueLimit = "queue_limit"
EnvMQTTState = "MINIO_NOTIFY_MQTT_STATE"
EnvMQTTBroker = "MINIO_NOTIFY_MQTT_BROKER"
EnvMQTTTopic = "MINIO_NOTIFY_MQTT_TOPIC"
EnvMQTTQoS = "MINIO_NOTIFY_MQTT_QOS"
EnvMQTTUsername = "MINIO_NOTIFY_MQTT_USERNAME"
EnvMQTTPassword = "MINIO_NOTIFY_MQTT_PASSWORD"
EnvMQTTReconnectInterval = "MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL"
EnvMQTTKeepAliveInterval = "MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL"
EnvMQTTQueueDir = "MINIO_NOTIFY_MQTT_QUEUE_DIR"
EnvMQTTQueueLimit = "MINIO_NOTIFY_MQTT_QUEUE_LIMIT"
)
// MQTTArgs - MQTT target arguments.
type MQTTArgs struct {
Enable bool `json:"enable"`

26
pkg/event/target/mysql.go
View File

@@ -81,6 +81,32 @@ const (
mysqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES (?, ?);`
)
// MySQL related constants
const (
MySQLFormat = "format"
MySQLDSNString = "dsn_string"
MySQLTable = "table"
MySQLHost = "host"
MySQLPort = "port"
MySQLUsername = "username"
MySQLPassword = "password"
MySQLDatabase = "database"
MySQLQueueLimit = "queue_limit"
MySQLQueueDir = "queue_dir"
EnvMySQLState = "MINIO_NOTIFY_MYSQL_STATE"
EnvMySQLFormat = "MINIO_NOTIFY_MYSQL_FORMAT"
EnvMySQLDSNString = "MINIO_NOTIFY_MYSQL_DSN_STRING"
EnvMySQLTable = "MINIO_NOTIFY_MYSQL_TABLE"
EnvMySQLHost = "MINIO_NOTIFY_MYSQL_HOST"
EnvMySQLPort = "MINIO_NOTIFY_MYSQL_PORT"
EnvMySQLUsername = "MINIO_NOTIFY_MYSQL_USERNAME"
EnvMySQLPassword = "MINIO_NOTIFY_MYSQL_PASSWORD"
EnvMySQLDatabase = "MINIO_NOTIFY_MYSQL_DATABASE"
EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT"
EnvMySQLQueueDir = "MINIO_NOTIFY_MYSQL_QUEUE_DIR"
)
// MySQLArgs - MySQL target arguments.
type MySQLArgs struct {
Enable bool `json:"enable"`

36
pkg/event/target/nats.go
View File

@@ -30,6 +30,42 @@ import (
"github.com/nats-io/stan.go"
)
// NATS related constants
const (
NATSAddress = "address"
NATSSubject = "subject"
NATSUsername = "username"
NATSPassword = "password"
NATSToken = "token"
NATSSecure = "secure"
NATSPingInterval = "ping_interval"
NATSQueueDir = "queue_dir"
NATSQueueLimit = "queue_limit"
// Streaming constants
NATSStreamingEnable = "streaming_enable"
NATSStreamingClusterID = "streaming_cluster_id"
NATSStreamingAsync = "streaming_async"
NATSStreamingMaxPubAcksInFlight = "streaming_max_pub_acks_in_flight"
EnvNATSState = "MINIO_NOTIFY_NATS_STATE"
EnvNATSAddress = "MINIO_NOTIFY_NATS_ADDRESS"
EnvNATSSubject = "MINIO_NOTIFY_NATS_SUBJECT"
EnvNATSUsername = "MINIO_NOTIFY_NATS_USERNAME"
EnvNATSPassword = "MINIO_NOTIFY_NATS_PASSWORD"
EnvNATSToken = "MINIO_NOTIFY_NATS_TOKEN"
EnvNATSSecure = "MINIO_NOTIFY_NATS_SECURE"
EnvNATSPingInterval = "MINIO_NOTIFY_NATS_PING_INTERVAL"
EnvNATSQueueDir = "MINIO_NOTIFY_NATS_QUEUE_DIR"
EnvNATSQueueLimit = "MINIO_NOTIFY_NATS_QUEUE_LIMIT"
// Streaming constants
EnvNATSStreamingEnable = "MINIO_NOTIFY_NATS_STREAMING_ENABLE"
EnvNATSStreamingClusterID = "MINIO_NOTIFY_NATS_STREAMING_CLUSTER_ID"
EnvNATSStreamingAsync = "MINIO_NOTIFY_NATS_STREAMING_ASYNC"
EnvNATSStreamingMaxPubAcksInFlight = "MINIO_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT"
)
// NATSArgs - NATS target arguments.
type NATSArgs struct {
Enable bool `json:"enable"`

18
pkg/event/target/nsq.go
View File

@@ -31,6 +31,24 @@ import (
xnet "github.com/minio/minio/pkg/net"
)
// NSQ constants
const (
NSQAddress = "nsqd_address"
NSQTopic = "topic"
NSQTLSEnable = "tls_enable"
NSQTLSSkipVerify = "tls_skip_verify"
NSQQueueDir = "queue_dir"
NSQQueueLimit = "queue_limit"
EnvNSQState = "MINIO_NOTIFY_NSQ"
EnvNSQAddress = "MINIO_NOTIFY_NSQ_NSQD_ADDRESS"
EnvNSQTopic = "MINIO_NOTIFY_NSQ_TOPIC"
EnvNSQTLSEnable = "MINIO_NOTIFY_NSQ_TLS_ENABLE"
EnvNSQTLSSkipVerify = "MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY"
EnvNSQQueueDir = "MINIO_NOTIFY_NSQ_QUEUE_DIR"
EnvNSQQueueLimit = "MINIO_NOTIFY_NSQ_QUEUE_LIMIT"
)
// NSQArgs - NSQ target arguments.
type NSQArgs struct {
Enable bool `json:"enable"`

26
pkg/event/target/postgresql.go
View File

@@ -82,6 +82,32 @@ const (
psqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES ($1, $2);`
)
// Postgres constants
const (
PostgresFormat = "format"
PostgresConnectionString = "connection_string"
PostgresTable = "table"
PostgresHost = "host"
PostgresPort = "port"
PostgresUsername = "username"
PostgresPassword = "password"
PostgresDatabase = "database"
PostgresQueueDir = "queue_dir"
PostgresQueueLimit = "queue_limit"
EnvPostgresState = "MINIO_NOTIFY_POSTGRES_STATE"
EnvPostgresFormat = "MINIO_NOTIFY_POSTGRES_FORMAT"
EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING"
EnvPostgresTable = "MINIO_NOTIFY_POSTGRES_TABLE"
EnvPostgresHost = "MINIO_NOTIFY_POSTGRES_HOST"
EnvPostgresPort = "MINIO_NOTIFY_POSTGRES_PORT"
EnvPostgresUsername = "MINIO_NOTIFY_POSTGRES_USERNAME"
EnvPostgresPassword = "MINIO_NOTIFY_POSTGRES_PASSWORD"
EnvPostgresDatabase = "MINIO_NOTIFY_POSTGRES_DATABASE"
EnvPostgresQueueDir = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR"
EnvPostgresQueueLimit = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT"
)
// PostgreSQLArgs - PostgreSQL target arguments.
type PostgreSQLArgs struct {
Enable bool `json:"enable"`

18
pkg/event/target/redis.go
View File

@@ -32,6 +32,24 @@ import (
xnet "github.com/minio/minio/pkg/net"
)
// Redis constants
const (
RedisFormat = "format"
RedisAddress = "address"
RedisPassword = "password"
RedisKey = "key"
RedisQueueDir = "queue_dir"
RedisQueueLimit = "queue_limit"
EnvRedisState = "MINIO_NOTIFY_REDIS_STATE"
EnvRedisFormat = "MINIO_NOTIFY_REDIS_FORMAT"
EnvRedisAddress = "MINIO_NOTIFY_REDIS_ADDRESS"
EnvRedisPassword = "MINIO_NOTIFY_REDIS_PASSWORD"
EnvRedisKey = "MINIO_NOTIFY_REDIS_KEY"
EnvRedisQueueDir = "MINIO_NOTIFY_REDIS_QUEUE_DIR"
EnvRedisQueueLimit = "MINIO_NOTIFY_REDIS_QUEUE_LIMIT"
)
// RedisArgs - Redis target arguments.
type RedisArgs struct {
Enable bool `json:"enable"`

27
pkg/event/target/webhook.go
View File

@@ -37,10 +37,25 @@ import (
xnet "github.com/minio/minio/pkg/net"
)
// Webhook constants
const (
WebhookEndpoint = "endpoint"
WebhookAuthToken = "auth_token"
WebhookQueueDir = "queue_dir"
WebhookQueueLimit = "queue_limit"
EnvWebhookState = "MINIO_NOTIFY_WEBHOOK_STATE"
EnvWebhookEndpoint = "MINIO_NOTIFY_WEBHOOK_ENDPOINT"
EnvWebhookAuthToken = "MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN"
EnvWebhookQueueDir = "MINIO_NOTIFY_WEBHOOK_QUEUE_DIR"
EnvWebhookQueueLimit = "MINIO_NOTIFY_WEBHOOK_QUEUE_LIMIT"
)
// WebhookArgs - Webhook target arguments.
type WebhookArgs struct {
Enable bool `json:"enable"`
Endpoint xnet.URL `json:"endpoint"`
AuthToken string `json:"authToken"`
RootCAs *x509.CertPool `json:"-"`
QueueDir string `json:"queueDir"`
QueueLimit uint64 `json:"queueLimit"`
@@ -114,6 +129,10 @@ func (target *WebhookTarget) send(eventData event.Event) error {
return err
}
if target.args.AuthToken != "" {
req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
}
req.Header.Set("Content-Type", "application/json")
resp, err := target.httpClient.Do(req)
@@ -172,15 +191,15 @@ func (target *WebhookTarget) Close() error {
}
// NewWebhookTarget - creates new Webhook target.
func NewWebhookTarget(id string, args WebhookArgs, doneCh <-chan struct{}, loggerOnce func(ctx context.Context, err error, id interface{}, kind ...interface{})) *WebhookTarget {
func NewWebhookTarget(id string, args WebhookArgs, doneCh <-chan struct{}, loggerOnce func(ctx context.Context, err error, id interface{}, kind ...interface{})) (*WebhookTarget, error) {
var store Store
if args.QueueDir != "" {
queueDir := filepath.Join(args.QueueDir, storePrefix+"-webhook-"+id)
store = NewQueueStore(queueDir, args.QueueLimit)
if oErr := store.Open(); oErr != nil {
return nil
if err := store.Open(); err != nil {
return nil, err
}
}
@@ -209,5 +228,5 @@ func NewWebhookTarget(id string, args WebhookArgs, doneCh <-chan struct{}, logge
go sendEvents(target, eventKeyCh, doneCh, loggerOnce)
}
return target
return target, nil
}