make notification as separate package (#5294)

* Remove old notification files

* Add net package

* Add event package

* Modify minio to take new notification system

pkg/event/target/amqp.go

@@ -0,0 +1,150 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"encoding/json"
+	"net"
+	"net/url"
+	"sync"
+
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+	"github.com/streadway/amqp"
+)
+
+// AMQPArgs - AMQP target arguments.
+type AMQPArgs struct {
+	Enable       bool     `json:"enable"`
+	URL          xnet.URL `json:"url"`
+	Exchange     string   `json:"exchange"`
+	RoutingKey   string   `json:"routingKey"`
+	ExchangeType string   `json:"exchangeType"`
+	DeliveryMode uint8    `json:"deliveryMode"`
+	Mandatory    bool     `json:"mandatory"`
+	Immediate    bool     `json:"immediate"`
+	Durable      bool     `json:"durable"`
+	Internal     bool     `json:"internal"`
+	NoWait       bool     `json:"noWait"`
+	AutoDeleted  bool     `json:"autoDeleted"`
+}
+
+// AMQPTarget - AMQP target
+type AMQPTarget struct {
+	id        event.TargetID
+	args      AMQPArgs
+	conn      *amqp.Connection
+	connMutex sync.Mutex
+}
+
+// ID - returns TargetID.
+func (target *AMQPTarget) ID() event.TargetID {
+	return target.id
+}
+
+func (target *AMQPTarget) channel() (*amqp.Channel, error) {
+	isAMQPClosedErr := func(err error) bool {
+		if err == amqp.ErrClosed {
+			return true
+		}
+
+		if nerr, ok := err.(*net.OpError); ok {
+			return (nerr.Err.Error() == "use of closed network connection")
+		}
+
+		return false
+	}
+
+	target.connMutex.Lock()
+	defer target.connMutex.Unlock()
+
+	ch, err := target.conn.Channel()
+	if err == nil {
+		return ch, nil
+	}
+
+	if !isAMQPClosedErr(err) {
+		return nil, err
+	}
+
+	var conn *amqp.Connection
+	if conn, err = amqp.Dial(target.args.URL.String()); err != nil {
+		return nil, err
+	}
+
+	if ch, err = conn.Channel(); err != nil {
+		return nil, err
+	}
+
+	target.conn = conn
+
+	return ch, nil
+}
+
+// Send - sends event to AMQP.
+func (target *AMQPTarget) Send(eventData event.Event) error {
+	ch, err := target.channel()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		// FIXME: log returned error. ignore time being.
+		_ = ch.Close()
+	}()
+
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{eventData.EventName, key, []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	if err = ch.ExchangeDeclare(target.args.Exchange, target.args.ExchangeType, target.args.Durable,
+		target.args.AutoDeleted, target.args.Internal, target.args.NoWait, nil); err != nil {
+		return err
+	}
+
+	return ch.Publish(target.args.Exchange, target.args.RoutingKey, target.args.Mandatory,
+		target.args.Immediate, amqp.Publishing{
+			ContentType:  "application/json",
+			DeliveryMode: target.args.DeliveryMode,
+			Body:         data,
+		})
+}
+
+// Close - does nothing and available for interface compatibility.
+func (target *AMQPTarget) Close() error {
+	return nil
+}
+
+// NewAMQPTarget - creates new AMQP target.
+func NewAMQPTarget(id string, args AMQPArgs) (*AMQPTarget, error) {
+	conn, err := amqp.Dial(args.URL.String())
+	if err != nil {
+		return nil, err
+	}
+
+	return &AMQPTarget{
+		id:   event.TargetID{id, "amqp"},
+		args: args,
+		conn: conn,
+	}, nil
+}

pkg/event/target/elasticsearch.go

@@ -0,0 +1,132 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+
+	"gopkg.in/olivere/elastic.v5"
+)
+
+// ElasticsearchArgs - Elasticsearch target arguments.
+type ElasticsearchArgs struct {
+	Enable bool     `json:"enable"`
+	Format string   `json:"format"`
+	URL    xnet.URL `json:"url"`
+	Index  string   `json:"index"`
+}
+
+// ElasticsearchTarget - Elasticsearch target.
+type ElasticsearchTarget struct {
+	id     event.TargetID
+	args   ElasticsearchArgs
+	client *elastic.Client
+}
+
+// ID - returns target ID.
+func (target *ElasticsearchTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Send - sends event to Elasticsearch.
+func (target *ElasticsearchTarget) Send(eventData event.Event) (err error) {
+	var key string
+
+	remove := func() error {
+		_, err := target.client.Delete().Index(target.args.Index).Type("event").Id(key).Do(context.Background())
+		return err
+	}
+
+	update := func() error {
+		_, err := target.client.Index().Index(target.args.Index).Type("event").BodyJson(map[string]interface{}{"Records": []event.Event{eventData}}).Id(key).Do(context.Background())
+		return err
+	}
+
+	add := func() error {
+		eventTime, err := time.Parse(event.AMZTimeFormat, eventData.EventTime)
+		if err != nil {
+			return err
+		}
+
+		eventTimeMS := fmt.Sprintf("%d", eventTime.UnixNano()/1000000)
+		_, err = target.client.Index().Index(target.args.Index).Type("event").Timestamp(eventTimeMS).BodyJson(map[string]interface{}{"Records": []event.Event{eventData}}).Do(context.Background())
+		return err
+	}
+
+	if target.args.Format == event.NamespaceFormat {
+		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+		if err != nil {
+			return err
+		}
+
+		key = eventData.S3.Bucket.Name + "/" + objectName
+		if eventData.EventName == event.ObjectRemovedDelete {
+			err = remove()
+		} else {
+			err = update()
+		}
+
+		return err
+	}
+
+	if target.args.Format == event.AccessFormat {
+		return add()
+	}
+
+	return nil
+}
+
+// Close - does nothing and available for interface compatibility.
+func (target *ElasticsearchTarget) Close() error {
+	return nil
+}
+
+// NewElasticsearchTarget - creates new Elasticsearch target.
+func NewElasticsearchTarget(id string, args ElasticsearchArgs) (*ElasticsearchTarget, error) {
+	client, err := elastic.NewClient(elastic.SetURL(args.URL.String()), elastic.SetSniff(false), elastic.SetMaxRetries(10))
+	if err != nil {
+		return nil, err
+	}
+
+	exists, err := client.IndexExists(args.Index).Do(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	if !exists {
+		var createIndex *elastic.IndicesCreateResult
+		if createIndex, err = client.CreateIndex(args.Index).Do(context.Background()); err != nil {
+			return nil, err
+		}
+
+		if !createIndex.Acknowledged {
+			return nil, fmt.Errorf("index %v not created", args.Index)
+		}
+	}
+
+	return &ElasticsearchTarget{
+		id:     event.TargetID{id, "elasticsearch"},
+		args:   args,
+		client: client,
+	}, nil
+}

pkg/event/target/httpclient.go

@@ -0,0 +1,141 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+	"github.com/skyrings/skyring-common/tools/uuid"
+)
+
+// HTTPClientTarget - HTTP client target.
+type HTTPClientTarget struct {
+	id        event.TargetID
+	w         http.ResponseWriter
+	eventCh   chan []byte
+	DoneCh    chan struct{}
+	stopCh    chan struct{}
+	isStopped uint32
+	isRunning uint32
+}
+
+// ID - returns target ID.
+func (target HTTPClientTarget) ID() event.TargetID {
+	return target.id
+}
+
+func (target *HTTPClientTarget) start() {
+	go func() {
+		defer func() {
+			atomic.AddUint32(&target.isRunning, 1)
+
+			// Close DoneCh to indicate we are done.
+			close(target.DoneCh)
+		}()
+
+		write := func(event []byte) error {
+			if _, err := target.w.Write(event); err != nil {
+				return err
+			}
+
+			target.w.(http.Flusher).Flush()
+			return nil
+		}
+
+		for {
+			keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+			select {
+			case <-target.stopCh:
+				// We are asked to stop.
+				return
+			case event, ok := <-target.eventCh:
+				if !ok {
+					// Got read error.  Exit the goroutine.
+					return
+				}
+				if err := write(event); err != nil {
+					// Got write error to the client.  Exit the goroutine.
+					return
+				}
+			case <-keepAliveTicker.C:
+				if err := write([]byte(" ")); err != nil {
+					// Got write error to the client.  Exit the goroutine.
+					return
+				}
+			}
+		}
+	}()
+}
+
+// Send - sends event to HTTP client.
+func (target *HTTPClientTarget) Send(eventData event.Event) error {
+	if atomic.LoadUint32(&target.isRunning) != 0 {
+		return errors.New("closed http connection")
+	}
+
+	data, err := json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+	data = append(data, byte('\n'))
+
+	select {
+	case target.eventCh <- data:
+		return nil
+	case <-target.DoneCh:
+		return errors.New("error in sending event")
+	}
+}
+
+// Close - closes underneath goroutine.
+func (target *HTTPClientTarget) Close() error {
+	atomic.AddUint32(&target.isStopped, 1)
+	if atomic.LoadUint32(&target.isStopped) == 1 {
+		close(target.stopCh)
+	}
+
+	return nil
+}
+
+func mustGetNewUUID() string {
+	uuid, err := uuid.New()
+	if err != nil {
+		panic(fmt.Sprintf("%s. Unable to generate random UUID", err))
+	}
+
+	return uuid.String()
+}
+
+// NewHTTPClientTarget - creates new HTTP client target.
+func NewHTTPClientTarget(host xnet.Host, w http.ResponseWriter) *HTTPClientTarget {
+	c := &HTTPClientTarget{
+		id:      event.TargetID{"httpclient" + "+" + mustGetNewUUID() + "+" + host.Name, host.Port.String()},
+		w:       w,
+		eventCh: make(chan []byte),
+		DoneCh:  make(chan struct{}),
+		stopCh:  make(chan struct{}),
+	}
+	c.start()
+	return c
+}

pkg/event/target/kafka.go

@@ -0,0 +1,97 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"encoding/json"
+	"net/url"
+
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+
+	sarama "gopkg.in/Shopify/sarama.v1"
+)
+
+// KafkaArgs - Kafka target arguments.
+type KafkaArgs struct {
+	Enable  bool        `json:"enable"`
+	Brokers []xnet.Host `json:"brokers"`
+	Topic   string      `json:"topic"`
+}
+
+// KafkaTarget - Kafka target.
+type KafkaTarget struct {
+	id       event.TargetID
+	args     KafkaArgs
+	producer sarama.SyncProducer
+}
+
+// ID - returns target ID.
+func (target *KafkaTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Send - sends event to Kafka.
+func (target *KafkaTarget) Send(eventData event.Event) error {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{eventData.EventName, key, []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	msg := sarama.ProducerMessage{
+		Topic: target.args.Topic,
+		Key:   sarama.StringEncoder(key),
+		Value: sarama.ByteEncoder(data),
+	}
+	_, _, err = target.producer.SendMessage(&msg)
+
+	return err
+}
+
+// Close - closes underneath kafka connection.
+func (target *KafkaTarget) Close() error {
+	return target.producer.Close()
+}
+
+// NewKafkaTarget - creates new Kafka target.
+func NewKafkaTarget(id string, args KafkaArgs) (*KafkaTarget, error) {
+	config := sarama.NewConfig()
+	config.Producer.RequiredAcks = sarama.WaitForAll
+	config.Producer.Retry.Max = 10
+	config.Producer.Return.Successes = true
+
+	brokers := []string{}
+	for _, broker := range args.Brokers {
+		brokers = append(brokers, broker.String())
+	}
+	producer, err := sarama.NewSyncProducer(brokers, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &KafkaTarget{
+		id:       event.TargetID{id, "kafka"},
+		args:     args,
+		producer: producer,
+	}, nil
+}

pkg/event/target/mqtt.go

@@ -0,0 +1,117 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"net/url"
+	"time"
+
+	"github.com/eclipse/paho.mqtt.golang"
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+// MQTTArgs - MQTT target arguments.
+type MQTTArgs struct {
+	Enable               bool           `json:"enable"`
+	Broker               xnet.URL       `json:"broker"`
+	Topic                string         `json:"topic"`
+	QoS                  byte           `json:"qos"`
+	ClientID             string         `json:"clientId"`
+	User                 string         `json:"username"`
+	Password             string         `json:"password"`
+	MaxReconnectInterval time.Duration  `json:"reconnectInterval"`
+	KeepAlive            time.Duration  `json:"keepAliveInterval"`
+	RootCAs              *x509.CertPool `json:"-"`
+}
+
+// MQTTTarget - MQTT target.
+type MQTTTarget struct {
+	id     event.TargetID
+	args   MQTTArgs
+	client mqtt.Client
+}
+
+// ID - returns target ID.
+func (target *MQTTTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Send - sends event to MQTT.
+func (target *MQTTTarget) Send(eventData event.Event) error {
+	if !target.client.IsConnected() {
+		token := target.client.Connect()
+		if token.Wait() {
+			if err := token.Error(); err != nil {
+				return err
+			}
+		}
+	}
+
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{eventData.EventName, key, []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	token := target.client.Publish(target.args.Topic, target.args.QoS, false, string(data))
+
+	if token.Wait() {
+		return token.Error()
+	}
+
+	return nil
+}
+
+// Close - does nothing and available for interface compatibility.
+func (target *MQTTTarget) Close() error {
+	return nil
+}
+
+// NewMQTTTarget - creates new MQTT target.
+func NewMQTTTarget(id string, args MQTTArgs) (*MQTTTarget, error) {
+	options := &mqtt.ClientOptions{
+		ClientID:             args.ClientID,
+		CleanSession:         true,
+		Username:             args.User,
+		Password:             args.Password,
+		MaxReconnectInterval: args.MaxReconnectInterval,
+		KeepAlive:            args.KeepAlive,
+		TLSConfig:            tls.Config{RootCAs: args.RootCAs},
+	}
+	options.AddBroker(args.Broker.String())
+
+	client := mqtt.NewClient(options)
+	token := client.Connect()
+	if token.Wait() && token.Error() != nil {
+		return nil, token.Error()
+	}
+
+	return &MQTTTarget{
+		id:     event.TargetID{id, "mqtt"},
+		args:   args,
+		client: client,
+	}, nil
+}

pkg/event/target/mysql.go

@@ -0,0 +1,226 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// MySQL Notifier implementation. Two formats, "namespace" and
+// "access" are supported.
+//
+// * Namespace format
+//
+// On each create or update object event in Minio Object storage
+// server, a row is created or updated in the table in MySQL. On each
+// object removal, the corresponding row is deleted from the table.
+//
+// A table with a specific structure (column names, column types, and
+// primary key/uniqueness constraint) is used. The user may set the
+// table name in the configuration. A sample SQL command that creates
+// a command with the required structure is:
+//
+//     CREATE TABLE myminio (
+//         key_name VARCHAR(2048),
+//         value JSONB,
+//         PRIMARY KEY (key_name),
+//     );
+//
+// MySQL's "INSERT ... ON DUPLICATE ..." feature (UPSERT) is used
+// here. The implementation has been tested with MySQL Ver 14.14
+// Distrib 5.7.17.
+//
+// * Access format
+//
+// On each event, a row is appended to the configured table. There is
+// no deletion or modification of existing rows.
+//
+// A different table schema is used for this format. A sample SQL
+// commant that creates a table with the required structure is:
+//
+// CREATE TABLE myminio (
+//     event_time TIMESTAMP WITH TIME ZONE NOT NULL,
+//     event_data JSONB
+// );
+
+package target
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/go-sql-driver/mysql"
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+const (
+	mysqlTableExists          = `SELECT 1 FROM %s;`
+	mysqlCreateNamespaceTable = `CREATE TABLE %s (key_name VARCHAR(2048), value JSON, PRIMARY KEY (key_name));`
+	mysqlCreateAccessTable    = `CREATE TABLE %s (event_time DATETIME NOT NULL, event_data JSON);`
+
+	mysqlUpdateRow = `INSERT INTO %s (key_name, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value);`
+	mysqlDeleteRow = `DELETE FROM %s WHERE key_name = ?;`
+	mysqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES (?, ?);`
+)
+
+// MySQLArgs - MySQL target arguments.
+type MySQLArgs struct {
+	Enable   bool     `json:"enable"`
+	Format   string   `json:"format"`
+	DSN      string   `json:"dsnString"`
+	Table    string   `json:"table"`
+	Host     xnet.URL `json:"host"`
+	Port     string   `json:"port"`
+	User     string   `json:"user"`
+	Password string   `json:"password"`
+	Database string   `json:"database"`
+}
+
+// MySQLTarget - MySQL target.
+type MySQLTarget struct {
+	id         event.TargetID
+	args       MySQLArgs
+	updateStmt *sql.Stmt
+	deleteStmt *sql.Stmt
+	insertStmt *sql.Stmt
+	db         *sql.DB
+}
+
+// ID - returns target ID.
+func (target *MySQLTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Send - sends event to MySQL.
+func (target *MySQLTarget) Send(eventData event.Event) error {
+	if target.args.Format == event.NamespaceFormat {
+		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+		if err != nil {
+			return err
+		}
+		key := eventData.S3.Bucket.Name + "/" + objectName
+
+		if eventData.EventName == event.ObjectRemovedDelete {
+			_, err = target.deleteStmt.Exec(key)
+		} else {
+			var data []byte
+			if data, err = json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}); err != nil {
+				return err
+			}
+
+			_, err = target.updateStmt.Exec(key, data)
+		}
+		return err
+	}
+
+	if target.args.Format == event.AccessFormat {
+		eventTime, err := time.Parse(event.AMZTimeFormat, eventData.EventTime)
+		if err != nil {
+			return err
+		}
+
+		data, err := json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}})
+		if err != nil {
+			return err
+		}
+
+		_, err = target.insertStmt.Exec(eventTime, data)
+		return err
+	}
+
+	return nil
+}
+
+// Close - closes underneath connections to MySQL database.
+func (target *MySQLTarget) Close() error {
+	if target.updateStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.updateStmt.Close()
+	}
+
+	if target.deleteStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.deleteStmt.Close()
+	}
+
+	if target.insertStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.insertStmt.Close()
+	}
+
+	return target.db.Close()
+}
+
+// NewMySQLTarget - creates new MySQL target.
+func NewMySQLTarget(id string, args MySQLArgs) (*MySQLTarget, error) {
+	if args.DSN == "" {
+		config := mysql.Config{
+			User:   args.User,
+			Passwd: args.Password,
+			Net:    "tcp",
+			Addr:   args.Host.String() + ":" + args.Port,
+			DBName: args.Database,
+		}
+
+		args.DSN = config.FormatDSN()
+	}
+
+	db, err := sql.Open("mysql", args.DSN)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = db.Ping(); err != nil {
+		return nil, err
+	}
+
+	if _, err = db.Exec(fmt.Sprintf(mysqlTableExists, args.Table)); err != nil {
+		createStmt := mysqlCreateNamespaceTable
+		if args.Format == event.AccessFormat {
+			createStmt = mysqlCreateAccessTable
+		}
+
+		if _, err = db.Exec(fmt.Sprintf(createStmt, args.Table)); err != nil {
+			return nil, err
+		}
+	}
+
+	var updateStmt, deleteStmt, insertStmt *sql.Stmt
+	switch args.Format {
+	case event.NamespaceFormat:
+		// insert or update statement
+		if updateStmt, err = db.Prepare(fmt.Sprintf(mysqlUpdateRow, args.Table)); err != nil {
+			return nil, err
+		}
+		// delete statement
+		if deleteStmt, err = db.Prepare(fmt.Sprintf(mysqlDeleteRow, args.Table)); err != nil {
+			return nil, err
+		}
+	case event.AccessFormat:
+		// insert statement
+		if insertStmt, err = db.Prepare(fmt.Sprintf(mysqlInsertRow, args.Table)); err != nil {
+			return nil, err
+		}
+	}
+
+	return &MySQLTarget{
+		id:         event.TargetID{id, "mysql"},
+		args:       args,
+		updateStmt: updateStmt,
+		deleteStmt: deleteStmt,
+		insertStmt: insertStmt,
+		db:         db,
+	}, nil
+}

pkg/event/target/nats.go

@@ -0,0 +1,143 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"encoding/json"
+	"net/url"
+
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+	"github.com/nats-io/go-nats-streaming"
+	"github.com/nats-io/nats"
+)
+
+// NATSArgs - NATS target arguments.
+type NATSArgs struct {
+	Enable       bool      `json:"enable"`
+	Address      xnet.Host `json:"address"`
+	Subject      string    `json:"subject"`
+	Username     string    `json:"username"`
+	Password     string    `json:"password"`
+	Token        string    `json:"token"`
+	Secure       bool      `json:"secure"`
+	PingInterval int64     `json:"pingInterval"`
+	Streaming    struct {
+		Enable             bool   `json:"enable"`
+		ClusterID          string `json:"clusterID"`
+		ClientID           string `json:"clientID"`
+		Async              bool   `json:"async"`
+		MaxPubAcksInflight int    `json:"maxPubAcksInflight"`
+	} `json:"streaming"`
+}
+
+// NATSTarget - NATS target.
+type NATSTarget struct {
+	id       event.TargetID
+	args     NATSArgs
+	natsConn *nats.Conn
+	stanConn stan.Conn
+}
+
+// ID - returns target ID.
+func (target *NATSTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Send - sends event to NATS.
+func (target *NATSTarget) Send(eventData event.Event) (err error) {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{eventData.EventName, key, []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	if target.stanConn != nil {
+		if target.args.Streaming.Async {
+			_, err = target.stanConn.PublishAsync(target.args.Subject, data, nil)
+		} else {
+			err = target.stanConn.Publish(target.args.Subject, data)
+		}
+	} else {
+		err = target.natsConn.Publish(target.args.Subject, data)
+	}
+
+	return err
+}
+
+// Close - closes underneath connections to NATS server.
+func (target *NATSTarget) Close() (err error) {
+	if target.stanConn != nil {
+		err = target.stanConn.Close()
+	}
+
+	if target.natsConn != nil {
+		target.natsConn.Close()
+	}
+
+	return err
+}
+
+// NewNATSTarget - creates new NATS target.
+func NewNATSTarget(id string, args NATSArgs) (*NATSTarget, error) {
+	var natsConn *nats.Conn
+	var stanConn stan.Conn
+	var err error
+
+	if args.Streaming.Enable {
+		scheme := "nats"
+		if args.Secure {
+			scheme = "tls"
+		}
+		addressURL := scheme + "://" + args.Username + ":" + args.Password + "@" + args.Address.String()
+
+		clientID := args.Streaming.ClientID
+		if clientID == "" {
+			clientID = mustGetNewUUID()
+		}
+
+		connOpts := []stan.Option{stan.NatsURL(addressURL)}
+		if args.Streaming.MaxPubAcksInflight > 0 {
+			connOpts = append(connOpts, stan.MaxPubAcksInflight(args.Streaming.MaxPubAcksInflight))
+		}
+
+		stanConn, err = stan.Connect(args.Streaming.ClusterID, clientID, connOpts...)
+	} else {
+		options := nats.DefaultOptions
+		options.Url = "nats://" + args.Address.String()
+		options.User = args.Username
+		options.Password = args.Password
+		options.Token = args.Token
+		options.Secure = args.Secure
+		natsConn, err = options.Connect()
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &NATSTarget{
+		id:       event.TargetID{id, "nats"},
+		args:     args,
+		stanConn: stanConn,
+		natsConn: natsConn,
+	}, nil
+}

pkg/event/target/postgresql.go

@@ -0,0 +1,233 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// PostgreSQL Notifier implementation. Two formats, "namespace" and
+// "access" are supported.
+//
+// * Namespace format
+//
+// On each create or update object event in Minio Object storage
+// server, a row is created or updated in the table in Postgres. On
+// each object removal, the corresponding row is deleted from the
+// table.
+//
+// A table with a specific structure (column names, column types, and
+// primary key/uniqueness constraint) is used. The user may set the
+// table name in the configuration. A sample SQL command that creates
+// a table with the required structure is:
+//
+//     CREATE TABLE myminio (
+//         key VARCHAR PRIMARY KEY,
+//         value JSONB
+//     );
+//
+// PostgreSQL's "INSERT ... ON CONFLICT ... DO UPDATE ..." feature
+// (UPSERT) is used here, so the minimum version of PostgreSQL
+// required is 9.5.
+//
+// * Access format
+//
+// On each event, a row is appended to the configured table. There is
+// no deletion or modification of existing rows.
+//
+// A different table schema is used for this format. A sample SQL
+// commant that creates a table with the required structure is:
+//
+// CREATE TABLE myminio (
+//     event_time TIMESTAMP WITH TIME ZONE NOT NULL,
+//     event_data JSONB
+// );
+
+package target
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	_ "github.com/lib/pq" // Register postgres driver
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+const (
+	psqlTableExists          = `SELECT 1 FROM %s;`
+	psqlCreateNamespaceTable = `CREATE TABLE %s (key VARCHAR PRIMARY KEY, value JSONB);`
+	psqlCreateAccessTable    = `CREATE TABLE %s (event_time TIMESTAMP WITH TIME ZONE NOT NULL, event_data JSONB);`
+
+	psqlUpdateRow = `INSERT INTO %s (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;`
+	psqlDeleteRow = `DELETE FROM %s WHERE key = $1;`
+	psqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES ($1, $2);`
+)
+
+// PostgreSQLArgs - PostgreSQL target arguments.
+type PostgreSQLArgs struct {
+	Enable           bool     `json:"enable"`
+	Format           string   `json:"format"`
+	ConnectionString string   `json:"connectionString"`
+	Table            string   `json:"table"`
+	Host             xnet.URL `json:"host"`     // default: localhost
+	Port             string   `json:"port"`     // default: 5432
+	User             string   `json:"user"`     // default: user running minio
+	Password         string   `json:"password"` // default: no password
+	Database         string   `json:"database"` // default: same as user
+}
+
+// PostgreSQLTarget - PostgreSQL target.
+type PostgreSQLTarget struct {
+	id         event.TargetID
+	args       PostgreSQLArgs
+	updateStmt *sql.Stmt
+	deleteStmt *sql.Stmt
+	insertStmt *sql.Stmt
+	db         *sql.DB
+}
+
+// ID - returns target ID.
+func (target *PostgreSQLTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Send - sends event to PostgreSQL.
+func (target *PostgreSQLTarget) Send(eventData event.Event) error {
+	if target.args.Format == event.NamespaceFormat {
+		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+		if err != nil {
+			return err
+		}
+		key := eventData.S3.Bucket.Name + "/" + objectName
+
+		if eventData.EventName == event.ObjectRemovedDelete {
+			_, err = target.deleteStmt.Exec(key)
+		} else {
+			var data []byte
+			if data, err = json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}); err != nil {
+				return err
+			}
+
+			_, err = target.updateStmt.Exec(key, data)
+		}
+		return err
+	}
+
+	if target.args.Format == event.AccessFormat {
+		eventTime, err := time.Parse(event.AMZTimeFormat, eventData.EventTime)
+		if err != nil {
+			return err
+		}
+
+		data, err := json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}})
+		if err != nil {
+			return err
+		}
+
+		_, err = target.insertStmt.Exec(eventTime, data)
+		return err
+	}
+
+	return nil
+}
+
+// Close - closes underneath connections to PostgreSQL database.
+func (target *PostgreSQLTarget) Close() error {
+	if target.updateStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.updateStmt.Close()
+	}
+
+	if target.deleteStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.deleteStmt.Close()
+	}
+
+	if target.insertStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.insertStmt.Close()
+	}
+
+	return target.db.Close()
+}
+
+// NewPostgreSQLTarget - creates new PostgreSQL target.
+func NewPostgreSQLTarget(id string, args PostgreSQLArgs) (*PostgreSQLTarget, error) {
+	params := []string{args.ConnectionString}
+	if !args.Host.IsEmpty() {
+		params = append(params, "host="+args.Host.String())
+	}
+	if args.Port != "" {
+		params = append(params, "port="+args.Port)
+	}
+	if args.User != "" {
+		params = append(params, "user="+args.User)
+	}
+	if args.Password != "" {
+		params = append(params, "password="+args.Password)
+	}
+	if args.Database != "" {
+		params = append(params, "dbname="+args.Database)
+	}
+	connStr := strings.Join(params, " ")
+
+	db, err := sql.Open("postgres", connStr)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = db.Ping(); err != nil {
+		return nil, err
+	}
+
+	if _, err = db.Exec(fmt.Sprintf(psqlTableExists, args.Table)); err != nil {
+		createStmt := psqlCreateNamespaceTable
+		if args.Format == event.AccessFormat {
+			createStmt = psqlCreateAccessTable
+		}
+
+		if _, err = db.Exec(fmt.Sprintf(createStmt, args.Table)); err != nil {
+			return nil, err
+		}
+	}
+
+	var updateStmt, deleteStmt, insertStmt *sql.Stmt
+	switch args.Format {
+	case event.NamespaceFormat:
+		// insert or update statement
+		if updateStmt, err = db.Prepare(fmt.Sprintf(psqlUpdateRow, args.Table)); err != nil {
+			return nil, err
+		}
+		// delete statement
+		if deleteStmt, err = db.Prepare(fmt.Sprintf(psqlDeleteRow, args.Table)); err != nil {
+			return nil, err
+		}
+	case event.AccessFormat:
+		// insert statement
+		if insertStmt, err = db.Prepare(fmt.Sprintf(psqlInsertRow, args.Table)); err != nil {
+			return nil, err
+		}
+	}
+
+	return &PostgreSQLTarget{
+		id:         event.TargetID{id, "postgresql"},
+		args:       args,
+		updateStmt: updateStmt,
+		deleteStmt: deleteStmt,
+		insertStmt: insertStmt,
+		db:         db,
+	}, nil
+}

pkg/event/target/redis.go

@@ -0,0 +1,156 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/garyburd/redigo/redis"
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+// RedisArgs - Redis target arguments.
+type RedisArgs struct {
+	Enable   bool      `json:"enable"`
+	Format   string    `json:"format"`
+	Addr     xnet.Host `json:"address"`
+	Password string    `json:"password"`
+	Key      string    `json:"key"`
+}
+
+// RedisTarget - Redis target.
+type RedisTarget struct {
+	id   event.TargetID
+	args RedisArgs
+	pool *redis.Pool
+}
+
+// ID - returns target ID.
+func (target *RedisTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Send - sends event to Redis.
+func (target *RedisTarget) Send(eventData event.Event) error {
+	conn := target.pool.Get()
+	defer func() {
+		// FIXME: log returned error. ignore time being.
+		_ = conn.Close()
+	}()
+
+	if target.args.Format == event.NamespaceFormat {
+		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+		if err != nil {
+			return err
+		}
+		key := eventData.S3.Bucket.Name + "/" + objectName
+
+		if eventData.EventName == event.ObjectRemovedDelete {
+			_, err = conn.Do("HDEL", target.args.Key, key)
+		} else {
+			var data []byte
+			if data, err = json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}); err != nil {
+				return err
+			}
+
+			_, err = conn.Do("HSET", target.args.Key, key, data)
+		}
+		return err
+	}
+
+	if target.args.Format == event.AccessFormat {
+		data, err := json.Marshal([]interface{}{eventData.EventTime, []event.Event{eventData}})
+		if err != nil {
+			return err
+		}
+		_, err = conn.Do("RPUSH", target.args.Key, data)
+		return err
+	}
+
+	return nil
+}
+
+// Close - does nothing and available for interface compatibility.
+func (target *RedisTarget) Close() error {
+	return nil
+}
+
+// NewRedisTarget - creates new Redis target.
+func NewRedisTarget(id string, args RedisArgs) (*RedisTarget, error) {
+	pool := &redis.Pool{
+		MaxIdle:     3,
+		IdleTimeout: 2 * 60 * time.Second,
+		Dial: func() (redis.Conn, error) {
+			conn, err := redis.Dial("tcp", args.Addr.String())
+			if err != nil {
+				return nil, err
+			}
+
+			if args.Password == "" {
+				return conn, nil
+			}
+
+			if _, err = conn.Do("AUTH", args.Password); err != nil {
+				// FIXME: log returned error. ignore time being.
+				_ = conn.Close()
+				return nil, err
+			}
+
+			return conn, nil
+		},
+		TestOnBorrow: func(c redis.Conn, t time.Time) error {
+			_, err := c.Do("PING")
+			return err
+		},
+	}
+
+	conn := pool.Get()
+	defer func() {
+		// FIXME: log returned error. ignore time being.
+		_ = conn.Close()
+	}()
+
+	if _, err := conn.Do("PING"); err != nil {
+		return nil, err
+	}
+
+	typeAvailable, err := redis.String(conn.Do("TYPE", args.Key))
+	if err != nil {
+		return nil, err
+	}
+
+	if typeAvailable != "none" {
+		expectedType := "hash"
+		if args.Format == event.AccessFormat {
+			expectedType = "list"
+		}
+
+		if typeAvailable != expectedType {
+			return nil, fmt.Errorf("expected type %v does not match with available type %v", expectedType, typeAvailable)
+		}
+	}
+
+	return &RedisTarget{
+		id:   event.TargetID{id, "redis"},
+		args: args,
+		pool: pool,
+	}, nil
+}

pkg/event/target/webhook.go

@@ -0,0 +1,113 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+// WebhookArgs - Webhook target arguments.
+type WebhookArgs struct {
+	Enable   bool           `json:"enable"`
+	Endpoint xnet.URL       `json:"endpoint"`
+	RootCAs  *x509.CertPool `json:"-"`
+}
+
+// WebhookTarget - Webhook target.
+type WebhookTarget struct {
+	id         event.TargetID
+	args       WebhookArgs
+	httpClient *http.Client
+}
+
+// ID - returns target ID.
+func (target WebhookTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Send - sends event to Webhook.
+func (target *WebhookTarget) Send(eventData event.Event) error {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{eventData.EventName, key, []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("POST", target.args.Endpoint.String(), bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	// req.Header.Set("User-Agent", globalServerUserAgent)
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := target.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	// FIXME: log returned error. ignore time being.
+	_ = resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK, http.StatusAccepted, http.StatusContinue:
+		return nil
+	default:
+		return fmt.Errorf("sending event failed with %v", resp.Status)
+	}
+}
+
+// Close - does nothing and available for interface compatibility.
+func (target *WebhookTarget) Close() error {
+	return nil
+}
+
+// NewWebhookTarget - creates new Webhook target.
+func NewWebhookTarget(id string, args WebhookArgs) *WebhookTarget {
+	return &WebhookTarget{
+		id:   event.TargetID{id, "webhook"},
+		args: args,
+		httpClient: &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{RootCAs: args.RootCAs},
+				DialContext: (&net.Dialer{
+					Timeout:   5 * time.Second,
+					KeepAlive: 5 * time.Second,
+				}).DialContext,
+				TLSHandshakeTimeout:   3 * time.Second,
+				ResponseHeaderTimeout: 3 * time.Second,
+				ExpectContinueTimeout: 2 * time.Second,
+			},
+		},
+	}
+}