2016-08-05 01:01:58 -04:00
|
|
|
/*
|
2017-03-18 14:28:41 -04:00
|
|
|
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
2016-08-05 01:01:58 -04:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2016-08-18 19:23:42 -04:00
|
|
|
package cmd
|
2016-08-05 01:01:58 -04:00
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2016-10-12 04:03:50 -04:00
|
|
|
"encoding/json"
|
2016-08-05 01:01:58 -04:00
|
|
|
"encoding/xml"
|
|
|
|
"fmt"
|
2016-08-25 02:04:34 -04:00
|
|
|
"net"
|
2016-08-05 01:01:58 -04:00
|
|
|
"net/url"
|
|
|
|
"path"
|
|
|
|
"sync"
|
|
|
|
|
|
|
|
"github.com/Sirupsen/logrus"
|
|
|
|
)
|
|
|
|
|
2017-03-22 21:44:35 -04:00
|
|
|
const (
|
|
|
|
minioEventSource = "minio:s3"
|
|
|
|
)
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
type externalNotifier struct {
|
|
|
|
// Per-bucket notification config. This is updated via
|
|
|
|
// PutBucketNotification API.
|
|
|
|
notificationConfigs map[string]*notificationConfig
|
|
|
|
|
|
|
|
// An external target keeps a connection to an external
|
|
|
|
// service to which events are to be sent. It is a mapping
|
|
|
|
// from an ARN to a log object
|
|
|
|
targets map[string]*logrus.Logger
|
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
rwMutex *sync.RWMutex
|
2016-10-12 04:03:50 -04:00
|
|
|
}
|
2016-08-05 01:01:58 -04:00
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
type internalNotifier struct {
|
|
|
|
// per-bucket listener configuration. This is updated
|
|
|
|
// when listeners connect or disconnect.
|
|
|
|
listenerConfigs map[string][]listenerConfig
|
|
|
|
|
|
|
|
// An internal target is a peer Minio server, that is
|
|
|
|
// connected to a listening client. Here, targets is a map of
|
|
|
|
// listener ARN to log object.
|
|
|
|
targets map[string]*listenerLogger
|
|
|
|
|
|
|
|
// Connected listeners is a map of listener ARNs to channels
|
|
|
|
// on which the ListenBucket API handler go routine is waiting
|
|
|
|
// for events to send to a client.
|
|
|
|
connectedListeners map[string]chan []NotificationEvent
|
|
|
|
|
|
|
|
rwMutex *sync.RWMutex
|
|
|
|
}
|
|
|
|
|
|
|
|
// Global event notification configuration. This structure has state
|
|
|
|
// about configured external notifications, and run-time configuration
|
|
|
|
// for listener notifications.
|
|
|
|
type eventNotifier struct {
|
|
|
|
|
|
|
|
// `external` here refers to notification configuration to
|
|
|
|
// send events to supported external systems
|
|
|
|
external externalNotifier
|
|
|
|
|
|
|
|
// `internal` refers to notification configuration for live
|
|
|
|
// listening clients. Events for a client are send from all
|
|
|
|
// servers, internally to a particular server that is
|
|
|
|
// connected to the client.
|
|
|
|
internal internalNotifier
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
// Represents data to be sent with notification event.
|
|
|
|
type eventData struct {
|
|
|
|
Type EventName
|
|
|
|
Bucket string
|
|
|
|
ObjInfo ObjectInfo
|
|
|
|
ReqParams map[string]string
|
2017-03-22 21:44:35 -04:00
|
|
|
Host string
|
|
|
|
Port string
|
|
|
|
UserAgent string
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
// New notification event constructs a new notification event message from
|
|
|
|
// input request metadata which completed successfully.
|
|
|
|
func newNotificationEvent(event eventData) NotificationEvent {
|
2017-01-10 19:43:48 -05:00
|
|
|
// Fetch the region.
|
2016-08-05 01:01:58 -04:00
|
|
|
region := serverConfig.GetRegion()
|
2017-01-10 19:43:48 -05:00
|
|
|
|
|
|
|
// Fetch the credentials.
|
|
|
|
creds := serverConfig.GetCredential()
|
|
|
|
|
|
|
|
// Time when Minio finished processing the request.
|
2017-03-18 14:28:41 -04:00
|
|
|
eventTime := UTCNow()
|
2017-01-10 19:43:48 -05:00
|
|
|
|
|
|
|
// API endpoint is captured here to be returned back
|
|
|
|
// to the client for it to differentiate from which
|
|
|
|
// server the request came from.
|
|
|
|
var apiEndpoint string
|
|
|
|
if len(globalAPIEndpoints) >= 1 {
|
|
|
|
apiEndpoint = globalAPIEndpoints[0]
|
|
|
|
}
|
|
|
|
|
|
|
|
// Fetch a hexadecimal representation of event time in nano seconds.
|
|
|
|
uniqueID := mustGetRequestID(eventTime)
|
|
|
|
|
|
|
|
/// Construct a new object created event.
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
// Following blocks fills in all the necessary details of s3
|
|
|
|
// event message structure.
|
2016-08-05 01:01:58 -04:00
|
|
|
// http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
|
|
|
|
nEvent := NotificationEvent{
|
2017-01-10 19:43:48 -05:00
|
|
|
EventVersion: eventVersion,
|
2017-03-22 21:44:35 -04:00
|
|
|
EventSource: minioEventSource,
|
2016-08-05 01:01:58 -04:00
|
|
|
AwsRegion: region,
|
2017-01-10 19:43:48 -05:00
|
|
|
EventTime: eventTime.Format(timeFormatAMZ),
|
2016-08-05 01:01:58 -04:00
|
|
|
EventName: event.Type.String(),
|
2017-01-10 19:43:48 -05:00
|
|
|
UserIdentity: identity{creds.AccessKey},
|
2016-08-05 01:01:58 -04:00
|
|
|
RequestParameters: event.ReqParams,
|
2017-01-10 19:43:48 -05:00
|
|
|
ResponseElements: map[string]string{
|
|
|
|
responseRequestIDKey: uniqueID,
|
|
|
|
// Following is a custom response element to indicate
|
|
|
|
// event origin server endpoint.
|
|
|
|
responseOriginEndpointKey: apiEndpoint,
|
|
|
|
},
|
2016-08-05 01:01:58 -04:00
|
|
|
S3: eventMeta{
|
2017-01-10 19:43:48 -05:00
|
|
|
SchemaVersion: eventSchemaVersion,
|
|
|
|
ConfigurationID: eventConfigID,
|
2016-08-05 01:01:58 -04:00
|
|
|
Bucket: bucketMeta{
|
|
|
|
Name: event.Bucket,
|
2017-01-10 19:43:48 -05:00
|
|
|
OwnerIdentity: identity{creds.AccessKey},
|
|
|
|
ARN: bucketARNPrefix + event.Bucket,
|
2016-08-05 01:01:58 -04:00
|
|
|
},
|
|
|
|
},
|
2017-03-22 21:44:35 -04:00
|
|
|
Source: sourceInfo{
|
|
|
|
Host: event.Host,
|
|
|
|
Port: event.Port,
|
|
|
|
UserAgent: event.UserAgent,
|
|
|
|
},
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
2017-01-10 19:43:48 -05:00
|
|
|
// Escape the object name. For example "red flower.jpg" becomes "red+flower.jpg".
|
2016-08-05 01:01:58 -04:00
|
|
|
escapedObj := url.QueryEscape(event.ObjInfo.Name)
|
2017-01-10 19:43:48 -05:00
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
// For delete object event type, we do not need to set ETag and Size.
|
|
|
|
if event.Type == ObjectRemovedDelete {
|
|
|
|
nEvent.S3.Object = objectMeta{
|
|
|
|
Key: escapedObj,
|
2017-03-30 11:58:14 -04:00
|
|
|
VersionID: "1",
|
2017-01-10 19:43:48 -05:00
|
|
|
Sequencer: uniqueID,
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
return nEvent
|
|
|
|
}
|
2017-01-10 19:43:48 -05:00
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
// For all other events we should set ETag and Size.
|
|
|
|
nEvent.S3.Object = objectMeta{
|
2017-03-30 11:58:14 -04:00
|
|
|
Key: escapedObj,
|
|
|
|
ETag: event.ObjInfo.MD5Sum,
|
|
|
|
Size: event.ObjInfo.Size,
|
|
|
|
ContentType: event.ObjInfo.ContentType,
|
|
|
|
UserDefined: event.ObjInfo.UserDefined,
|
|
|
|
VersionID: "1",
|
|
|
|
Sequencer: uniqueID,
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
2017-01-10 19:43:48 -05:00
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
// Success.
|
|
|
|
return nEvent
|
|
|
|
}
|
|
|
|
|
2017-02-15 13:45:45 -05:00
|
|
|
// Fetch all external targets. This returns a copy of the current map of
|
|
|
|
// external notification targets.
|
|
|
|
func (en eventNotifier) GetAllExternalTargets() map[string]*logrus.Logger {
|
|
|
|
en.external.rwMutex.RLock()
|
|
|
|
defer en.external.rwMutex.RUnlock()
|
|
|
|
targetsCopy := make(map[string]*logrus.Logger)
|
|
|
|
for k, v := range en.external.targets {
|
|
|
|
targetsCopy[k] = v
|
|
|
|
}
|
|
|
|
return targetsCopy
|
|
|
|
}
|
|
|
|
|
|
|
|
// Fetch the external target.
|
2016-10-12 04:03:50 -04:00
|
|
|
func (en eventNotifier) GetExternalTarget(queueARN string) *logrus.Logger {
|
2017-02-15 13:45:45 -05:00
|
|
|
en.external.rwMutex.RLock()
|
|
|
|
defer en.external.rwMutex.RUnlock()
|
2016-10-12 04:03:50 -04:00
|
|
|
return en.external.targets[queueARN]
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
func (en eventNotifier) GetInternalTarget(arn string) *listenerLogger {
|
|
|
|
en.internal.rwMutex.RLock()
|
|
|
|
defer en.internal.rwMutex.RUnlock()
|
|
|
|
return en.internal.targets[arn]
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
2016-08-15 23:56:43 -04:00
|
|
|
// Set a new sns target for an input sns ARN.
|
2016-10-12 04:03:50 -04:00
|
|
|
func (en *eventNotifier) AddListenerChan(snsARN string, listenerCh chan []NotificationEvent) error {
|
2016-08-05 01:01:58 -04:00
|
|
|
if listenerCh == nil {
|
2016-09-10 05:23:28 -04:00
|
|
|
return errInvalidArgument
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
2016-10-12 04:03:50 -04:00
|
|
|
en.internal.rwMutex.Lock()
|
|
|
|
defer en.internal.rwMutex.Unlock()
|
|
|
|
en.internal.connectedListeners[snsARN] = listenerCh
|
2016-08-05 01:01:58 -04:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-08-15 23:56:43 -04:00
|
|
|
// Remove sns target for an input sns ARN.
|
2016-10-12 04:03:50 -04:00
|
|
|
func (en *eventNotifier) RemoveListenerChan(snsARN string) {
|
|
|
|
en.internal.rwMutex.Lock()
|
|
|
|
defer en.internal.rwMutex.Unlock()
|
|
|
|
if en.internal.connectedListeners != nil {
|
|
|
|
delete(en.internal.connectedListeners, snsARN)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (en *eventNotifier) SendListenerEvent(arn string, event []NotificationEvent) error {
|
|
|
|
en.internal.rwMutex.Lock()
|
|
|
|
defer en.internal.rwMutex.Unlock()
|
|
|
|
|
|
|
|
ch, ok := en.internal.connectedListeners[arn]
|
2016-08-05 01:01:58 -04:00
|
|
|
if ok {
|
2016-10-12 04:03:50 -04:00
|
|
|
ch <- event
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
2016-10-12 04:03:50 -04:00
|
|
|
// If the channel is not present we ignore the event.
|
|
|
|
return nil
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
// Fetch bucket notification config for an input bucket.
|
|
|
|
func (en eventNotifier) GetBucketNotificationConfig(bucket string) *notificationConfig {
|
2016-10-12 04:03:50 -04:00
|
|
|
en.external.rwMutex.RLock()
|
|
|
|
defer en.external.rwMutex.RUnlock()
|
|
|
|
return en.external.notificationConfigs[bucket]
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
func (en *eventNotifier) SetBucketNotificationConfig(bucket string, ncfg *notificationConfig) {
|
|
|
|
en.external.rwMutex.Lock()
|
|
|
|
if ncfg == nil {
|
|
|
|
delete(en.external.notificationConfigs, bucket)
|
|
|
|
} else {
|
|
|
|
en.external.notificationConfigs[bucket] = ncfg
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
2016-10-12 04:03:50 -04:00
|
|
|
en.external.rwMutex.Unlock()
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
func (en *eventNotifier) GetBucketListenerConfig(bucket string) []listenerConfig {
|
|
|
|
en.internal.rwMutex.RLock()
|
|
|
|
defer en.internal.rwMutex.RUnlock()
|
|
|
|
return en.internal.listenerConfigs[bucket]
|
|
|
|
}
|
|
|
|
|
|
|
|
func (en *eventNotifier) SetBucketListenerConfig(bucket string, lcfg []listenerConfig) error {
|
|
|
|
en.internal.rwMutex.Lock()
|
|
|
|
defer en.internal.rwMutex.Unlock()
|
2016-10-20 19:09:19 -04:00
|
|
|
if len(lcfg) == 0 {
|
2016-10-12 04:03:50 -04:00
|
|
|
delete(en.internal.listenerConfigs, bucket)
|
|
|
|
} else {
|
|
|
|
en.internal.listenerConfigs[bucket] = lcfg
|
|
|
|
}
|
2016-10-20 19:09:19 -04:00
|
|
|
for _, elcArr := range en.internal.listenerConfigs {
|
|
|
|
for _, elcElem := range elcArr {
|
|
|
|
currArn := elcElem.TopicConfig.TopicARN
|
|
|
|
logger, err := newListenerLogger(currArn, elcElem.TargetServer)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
en.internal.targets[currArn] = logger
|
2016-09-29 01:46:19 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-10-22 01:56:27 -04:00
|
|
|
func eventNotifyForBucketNotifications(eventType, objectName, bucketName string, nEvent []NotificationEvent) {
|
2016-10-12 04:03:50 -04:00
|
|
|
nConfig := globalEventNotifier.GetBucketNotificationConfig(bucketName)
|
2016-09-10 05:23:28 -04:00
|
|
|
if nConfig == nil {
|
|
|
|
return
|
|
|
|
}
|
2016-08-05 01:01:58 -04:00
|
|
|
// Validate if the event and object match the queue configs.
|
|
|
|
for _, qConfig := range nConfig.QueueConfigs {
|
|
|
|
eventMatch := eventMatch(eventType, qConfig.Events)
|
|
|
|
ruleMatch := filterRuleMatch(objectName, qConfig.Filter.Key.FilterRules)
|
|
|
|
if eventMatch && ruleMatch {
|
2016-10-12 04:03:50 -04:00
|
|
|
targetLog := globalEventNotifier.GetExternalTarget(qConfig.QueueARN)
|
2016-08-05 01:01:58 -04:00
|
|
|
if targetLog != nil {
|
|
|
|
targetLog.WithFields(logrus.Fields{
|
2016-10-12 04:03:50 -04:00
|
|
|
"Key": path.Join(bucketName, objectName),
|
2016-09-20 05:11:17 -04:00
|
|
|
"EventType": eventType,
|
2016-10-12 04:03:50 -04:00
|
|
|
"Records": nEvent,
|
2016-08-05 01:01:58 -04:00
|
|
|
}).Info()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-10-12 04:03:50 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
func eventNotifyForBucketListeners(eventType, objectName, bucketName string,
|
|
|
|
nEvent []NotificationEvent) {
|
|
|
|
lCfgs := globalEventNotifier.GetBucketListenerConfig(bucketName)
|
|
|
|
if lCfgs == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Validate if the event and object match listener configs
|
|
|
|
for _, lcfg := range lCfgs {
|
|
|
|
ruleMatch := filterRuleMatch(objectName, lcfg.TopicConfig.Filter.Key.FilterRules)
|
|
|
|
eventMatch := eventMatch(eventType, lcfg.TopicConfig.Events)
|
2016-08-05 01:01:58 -04:00
|
|
|
if eventMatch && ruleMatch {
|
2016-10-12 04:03:50 -04:00
|
|
|
targetLog := globalEventNotifier.GetInternalTarget(
|
|
|
|
lcfg.TopicConfig.TopicARN)
|
|
|
|
if targetLog != nil && targetLog.log != nil {
|
|
|
|
targetLog.log.WithFields(logrus.Fields{
|
|
|
|
"Key": path.Join(bucketName, objectName),
|
|
|
|
"EventType": eventType,
|
|
|
|
"Records": nEvent,
|
|
|
|
}).Info()
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-10-12 04:03:50 -04:00
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// eventNotify notifies an event to relevant targets based on their
|
|
|
|
// bucket configuration (notifications and listeners).
|
|
|
|
func eventNotify(event eventData) {
|
2017-03-16 15:21:58 -04:00
|
|
|
if globalEventNotifier == nil {
|
|
|
|
return
|
|
|
|
}
|
2016-10-12 04:03:50 -04:00
|
|
|
// Notifies a new event.
|
|
|
|
// List of events reported through this function are
|
|
|
|
// - s3:ObjectCreated:Put
|
|
|
|
// - s3:ObjectCreated:Post
|
|
|
|
// - s3:ObjectCreated:Copy
|
|
|
|
// - s3:ObjectCreated:CompleteMultipartUpload
|
|
|
|
// - s3:ObjectRemoved:Delete
|
|
|
|
|
|
|
|
// Event type.
|
|
|
|
eventType := event.Type.String()
|
|
|
|
|
|
|
|
// Object name.
|
|
|
|
objectName := event.ObjInfo.Name
|
|
|
|
|
|
|
|
// Save the notification event to be sent.
|
|
|
|
notificationEvent := []NotificationEvent{newNotificationEvent(event)}
|
|
|
|
|
|
|
|
// Notify external targets.
|
2016-10-22 01:56:27 -04:00
|
|
|
eventNotifyForBucketNotifications(eventType, objectName, event.Bucket, notificationEvent)
|
2016-10-12 04:03:50 -04:00
|
|
|
|
|
|
|
// Notify internal targets.
|
2016-10-22 01:56:27 -04:00
|
|
|
eventNotifyForBucketListeners(eventType, objectName, event.Bucket, notificationEvent)
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
// loads notification config if any for a given bucket, returns
|
|
|
|
// structured notification config.
|
2016-08-05 01:01:58 -04:00
|
|
|
func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationConfig, error) {
|
|
|
|
// Construct the notification config path.
|
2016-12-10 19:15:12 -05:00
|
|
|
ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
|
|
|
|
|
|
|
|
// Acquire a write lock on notification config before modifying.
|
|
|
|
objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
|
|
|
|
objLock.RLock()
|
|
|
|
defer objLock.RUnlock()
|
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
var buffer bytes.Buffer
|
2016-12-21 14:29:32 -05:00
|
|
|
err := objAPI.GetObject(minioMetaBucket, ncPath, 0, -1, &buffer) // Read everything.
|
2016-08-05 01:01:58 -04:00
|
|
|
if err != nil {
|
2016-10-12 04:03:50 -04:00
|
|
|
// 'notification.xml' not found return
|
|
|
|
// 'errNoSuchNotifications'. This is default when no
|
|
|
|
// bucket notifications are found on the bucket.
|
2016-11-19 20:37:57 -05:00
|
|
|
if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
|
2016-08-05 01:01:58 -04:00
|
|
|
return nil, errNoSuchNotifications
|
|
|
|
}
|
2016-09-06 16:30:05 -04:00
|
|
|
errorIf(err, "Unable to load bucket-notification for bucket %s", bucket)
|
2016-08-05 01:01:58 -04:00
|
|
|
// Returns error for other errors.
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Unmarshal notification bytes.
|
|
|
|
notificationConfigBytes := buffer.Bytes()
|
|
|
|
notificationCfg := ¬ificationConfig{}
|
|
|
|
if err = xml.Unmarshal(notificationConfigBytes, ¬ificationCfg); err != nil {
|
|
|
|
return nil, err
|
2016-10-12 04:03:50 -04:00
|
|
|
}
|
2016-08-05 01:01:58 -04:00
|
|
|
|
|
|
|
// Return success.
|
|
|
|
return notificationCfg, nil
|
|
|
|
}
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
// loads notification config if any for a given bucket, returns
|
|
|
|
// structured notification config.
|
|
|
|
func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, error) {
|
2016-10-26 23:13:00 -04:00
|
|
|
// in single node mode, there are no peers, so in this case
|
|
|
|
// there is no configuration to load, as any previously
|
|
|
|
// connected listen clients have been disconnected
|
2016-11-07 15:09:24 -05:00
|
|
|
if !globalIsDistXL {
|
2016-10-26 23:13:00 -04:00
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
// Construct the notification config path.
|
2016-12-10 19:15:12 -05:00
|
|
|
lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
|
|
|
|
|
|
|
|
// Acquire a write lock on notification config before modifying.
|
|
|
|
objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
|
|
|
|
objLock.RLock()
|
|
|
|
defer objLock.RUnlock()
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
var buffer bytes.Buffer
|
2016-12-21 14:29:32 -05:00
|
|
|
err := objAPI.GetObject(minioMetaBucket, lcPath, 0, -1, &buffer)
|
2016-10-12 04:03:50 -04:00
|
|
|
if err != nil {
|
|
|
|
// 'notification.xml' not found return
|
|
|
|
// 'errNoSuchNotifications'. This is default when no
|
2016-10-26 11:46:14 -04:00
|
|
|
// bucket listeners are found on the bucket.
|
2016-11-16 19:42:23 -05:00
|
|
|
if isErrObjectNotFound(err) {
|
2016-10-12 04:03:50 -04:00
|
|
|
return nil, errNoSuchNotifications
|
|
|
|
}
|
|
|
|
errorIf(err, "Unable to load bucket-listeners for bucket %s", bucket)
|
|
|
|
// Returns error for other errors.
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Unmarshal notification bytes.
|
|
|
|
var lCfg []listenerConfig
|
|
|
|
lConfigBytes := buffer.Bytes()
|
|
|
|
if err = json.Unmarshal(lConfigBytes, &lCfg); err != nil {
|
|
|
|
errorIf(err, "Unable to unmarshal listener config from JSON.")
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return success.
|
|
|
|
return lCfg, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func persistNotificationConfig(bucket string, ncfg *notificationConfig, obj ObjectLayer) error {
|
|
|
|
// marshal to xml
|
|
|
|
buf, err := xml.Marshal(ncfg)
|
|
|
|
if err != nil {
|
|
|
|
errorIf(err, "Unable to marshal notification configuration into XML")
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// build path
|
|
|
|
ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
|
2016-12-10 19:15:12 -05:00
|
|
|
// Acquire a write lock on notification config before modifying.
|
|
|
|
objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
|
|
|
|
objLock.Lock()
|
|
|
|
defer objLock.Unlock()
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
// write object to path
|
2016-11-21 16:51:05 -05:00
|
|
|
sha256Sum := getSHA256Hash(buf)
|
2016-10-21 04:25:17 -04:00
|
|
|
_, err = obj.PutObject(minioMetaBucket, ncPath, int64(len(buf)), bytes.NewReader(buf), nil, sha256Sum)
|
2016-10-12 04:03:50 -04:00
|
|
|
if err != nil {
|
|
|
|
errorIf(err, "Unable to write bucket notification configuration.")
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Persists validated listener config to object layer.
|
|
|
|
func persistListenerConfig(bucket string, lcfg []listenerConfig, obj ObjectLayer) error {
|
|
|
|
buf, err := json.Marshal(lcfg)
|
|
|
|
if err != nil {
|
|
|
|
errorIf(err, "Unable to marshal listener config to JSON.")
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// build path
|
|
|
|
lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
|
2016-12-10 19:15:12 -05:00
|
|
|
// Acquire a write lock on notification config before modifying.
|
|
|
|
objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
|
|
|
|
objLock.Lock()
|
|
|
|
defer objLock.Unlock()
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
// write object to path
|
2016-11-21 16:51:05 -05:00
|
|
|
sha256Sum := getSHA256Hash(buf)
|
2016-10-21 04:25:17 -04:00
|
|
|
_, err = obj.PutObject(minioMetaBucket, lcPath, int64(len(buf)), bytes.NewReader(buf), nil, sha256Sum)
|
2016-10-12 04:03:50 -04:00
|
|
|
if err != nil {
|
|
|
|
errorIf(err, "Unable to write bucket listener configuration to object layer.")
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-12-10 19:15:12 -05:00
|
|
|
// Removes notification.xml for a given bucket, only used during DeleteBucket.
|
|
|
|
func removeNotificationConfig(bucket string, objAPI ObjectLayer) error {
|
|
|
|
// Verify bucket is valid.
|
|
|
|
if !IsValidBucketName(bucket) {
|
|
|
|
return BucketNameInvalid{Bucket: bucket}
|
|
|
|
}
|
|
|
|
|
|
|
|
ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
|
|
|
|
|
|
|
|
// Acquire a write lock on notification config before modifying.
|
|
|
|
objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
|
|
|
|
objLock.Lock()
|
|
|
|
err := objAPI.DeleteObject(minioMetaBucket, ncPath)
|
|
|
|
objLock.Unlock()
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-10-21 04:25:17 -04:00
|
|
|
// Remove listener configuration from storage layer. Used when a bucket is deleted.
|
2016-11-19 20:37:57 -05:00
|
|
|
func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
|
2016-10-20 19:09:19 -04:00
|
|
|
// make the path
|
|
|
|
lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
|
2016-12-10 19:15:12 -05:00
|
|
|
|
|
|
|
// Acquire a write lock on notification config before modifying.
|
|
|
|
objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
|
|
|
|
objLock.Lock()
|
|
|
|
err := objAPI.DeleteObject(minioMetaBucket, lcPath)
|
|
|
|
objLock.Unlock()
|
|
|
|
return err
|
2016-11-19 20:37:57 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
// Loads both notification and listener config.
|
|
|
|
func loadNotificationAndListenerConfig(bucketName string, objAPI ObjectLayer) (nCfg *notificationConfig, lCfg []listenerConfig, err error) {
|
|
|
|
// Loads notification config if any.
|
|
|
|
nCfg, err = loadNotificationConfig(bucketName, objAPI)
|
2016-11-23 23:05:04 -05:00
|
|
|
if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
|
2016-11-19 20:37:57 -05:00
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Loads listener config if any.
|
|
|
|
lCfg, err = loadListenerConfig(bucketName, objAPI)
|
2016-11-23 23:05:04 -05:00
|
|
|
if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
|
2016-11-19 20:37:57 -05:00
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
return nCfg, lCfg, nil
|
2016-10-20 19:09:19 -04:00
|
|
|
}
|
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
// loads all bucket notifications if present.
|
2016-10-12 04:03:50 -04:00
|
|
|
func loadAllBucketNotifications(objAPI ObjectLayer) (map[string]*notificationConfig, map[string][]listenerConfig, error) {
|
2016-08-05 01:01:58 -04:00
|
|
|
// List buckets to proceed loading all notification configuration.
|
|
|
|
buckets, err := objAPI.ListBuckets()
|
|
|
|
if err != nil {
|
2016-10-12 04:03:50 -04:00
|
|
|
return nil, nil, err
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
nConfigs := make(map[string]*notificationConfig)
|
|
|
|
lConfigs := make(map[string][]listenerConfig)
|
2016-08-05 01:01:58 -04:00
|
|
|
|
|
|
|
// Loads all bucket notifications.
|
|
|
|
for _, bucket := range buckets {
|
2016-11-19 20:37:57 -05:00
|
|
|
// Load persistent notification and listener configurations
|
|
|
|
// a given bucket name.
|
|
|
|
nConfigs[bucket.Name], lConfigs[bucket.Name], err = loadNotificationAndListenerConfig(bucket.Name, objAPI)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Success.
|
2016-10-12 04:03:50 -04:00
|
|
|
return nConfigs, lConfigs, nil
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
2017-02-24 21:27:52 -05:00
|
|
|
// addQueueTarget - calls newTargetFunc function and adds its returned value to queueTargets
|
|
|
|
func addQueueTarget(queueTargets map[string]*logrus.Logger,
|
|
|
|
accountID, queueType string,
|
|
|
|
newTargetFunc func(string) (*logrus.Logger, error)) (string, error) {
|
|
|
|
|
|
|
|
// Construct the queue ARN for AMQP.
|
|
|
|
queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueType
|
|
|
|
|
|
|
|
// Queue target if already initialized we move to the next ARN.
|
|
|
|
if _, ok := queueTargets[queueARN]; ok {
|
|
|
|
return queueARN, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Using accountID we can now initialize a new AMQP logrus instance.
|
|
|
|
logger, err := newTargetFunc(accountID)
|
|
|
|
if err == nil {
|
|
|
|
queueTargets[queueARN] = logger
|
|
|
|
}
|
|
|
|
|
|
|
|
return queueARN, err
|
|
|
|
}
|
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
// Loads all queue targets, initializes each queueARNs depending on their config.
|
|
|
|
// Each instance of queueARN registers its own logrus to communicate with the
|
|
|
|
// queue service. QueueARN once initialized is not initialized again for the
|
|
|
|
// same queueARN, instead previous connection is used.
|
|
|
|
func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
|
|
|
queueTargets := make(map[string]*logrus.Logger)
|
|
|
|
// Load all amqp targets, initialize their respective loggers.
|
2017-02-09 18:20:54 -05:00
|
|
|
for accountID, amqpN := range serverConfig.Notify.GetAMQP() {
|
2016-08-05 01:01:58 -04:00
|
|
|
if !amqpN.Enable {
|
|
|
|
continue
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
|
|
|
if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeAMQP, newAMQPNotify); err != nil {
|
2016-08-25 02:04:34 -04:00
|
|
|
if _, ok := err.(net.Error); ok {
|
2017-02-24 21:27:52 -05:00
|
|
|
err = &net.OpError{
|
2016-08-25 02:04:34 -04:00
|
|
|
Op: "Connecting to " + queueARN,
|
|
|
|
Net: "tcp",
|
|
|
|
Err: err,
|
|
|
|
}
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
2016-09-30 02:42:10 -04:00
|
|
|
// Load all nats targets, initialize their respective loggers.
|
2017-02-09 18:20:54 -05:00
|
|
|
for accountID, natsN := range serverConfig.Notify.GetNATS() {
|
2016-09-30 02:42:10 -04:00
|
|
|
if !natsN.Enable {
|
|
|
|
continue
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
|
|
|
if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeNATS, newNATSNotify); err != nil {
|
2016-09-30 02:42:10 -04:00
|
|
|
if _, ok := err.(net.Error); ok {
|
2017-02-24 21:27:52 -05:00
|
|
|
err = &net.OpError{
|
2016-09-30 02:42:10 -04:00
|
|
|
Op: "Connecting to " + queueARN,
|
|
|
|
Net: "tcp",
|
|
|
|
Err: err,
|
|
|
|
}
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
2016-09-30 02:42:10 -04:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
// Load redis targets, initialize their respective loggers.
|
2017-02-09 18:20:54 -05:00
|
|
|
for accountID, redisN := range serverConfig.Notify.GetRedis() {
|
2016-08-05 01:01:58 -04:00
|
|
|
if !redisN.Enable {
|
|
|
|
continue
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
|
|
|
if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeRedis, newRedisNotify); err != nil {
|
2016-08-25 02:04:34 -04:00
|
|
|
if _, ok := err.(net.Error); ok {
|
2017-02-24 21:27:52 -05:00
|
|
|
err = &net.OpError{
|
2016-08-25 02:04:34 -04:00
|
|
|
Op: "Connecting to " + queueARN,
|
|
|
|
Net: "tcp",
|
|
|
|
Err: err,
|
|
|
|
}
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
2017-01-09 17:22:10 -05:00
|
|
|
|
|
|
|
// Load Webhook targets, initialize their respective loggers.
|
2017-02-09 18:20:54 -05:00
|
|
|
for accountID, webhookN := range serverConfig.Notify.GetWebhook() {
|
2017-01-09 17:22:10 -05:00
|
|
|
if !webhookN.Enable {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2017-02-24 21:27:52 -05:00
|
|
|
if _, err := addQueueTarget(queueTargets, accountID, queueTypeWebhook, newWebhookNotify); err != nil {
|
2017-01-09 17:22:10 -05:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
// Load elastic targets, initialize their respective loggers.
|
2017-02-09 18:20:54 -05:00
|
|
|
for accountID, elasticN := range serverConfig.Notify.GetElasticSearch() {
|
2016-08-05 01:01:58 -04:00
|
|
|
if !elasticN.Enable {
|
|
|
|
continue
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
|
|
|
if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeElastic, newElasticNotify); err != nil {
|
2016-08-25 02:04:34 -04:00
|
|
|
if _, ok := err.(net.Error); ok {
|
2017-02-24 21:27:52 -05:00
|
|
|
err = &net.OpError{
|
|
|
|
Op: "Connecting to " + queueARN,
|
|
|
|
Net: "tcp",
|
2016-08-25 02:04:34 -04:00
|
|
|
Err: err,
|
|
|
|
}
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
2017-01-09 17:22:10 -05:00
|
|
|
|
2016-10-03 20:29:55 -04:00
|
|
|
// Load PostgreSQL targets, initialize their respective loggers.
|
2017-02-09 18:20:54 -05:00
|
|
|
for accountID, pgN := range serverConfig.Notify.GetPostgreSQL() {
|
2016-10-03 20:29:55 -04:00
|
|
|
if !pgN.Enable {
|
|
|
|
continue
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
|
|
|
if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypePostgreSQL, newPostgreSQLNotify); err != nil {
|
2016-10-03 20:29:55 -04:00
|
|
|
if _, ok := err.(net.Error); ok {
|
2017-02-24 21:27:52 -05:00
|
|
|
err = &net.OpError{
|
|
|
|
Op: "Connecting to " + queueARN,
|
|
|
|
Net: "tcp",
|
2016-10-03 20:29:55 -04:00
|
|
|
Err: err,
|
|
|
|
}
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
2017-03-17 12:29:17 -04:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Load MySQL targets, initialize their respective loggers.
|
|
|
|
for accountID, msqlN := range serverConfig.Notify.GetMySQL() {
|
|
|
|
if !msqlN.Enable {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeMySQL, newMySQLNotify); err != nil {
|
|
|
|
if _, ok := err.(net.Error); ok {
|
|
|
|
err = &net.OpError{
|
|
|
|
Op: "Connecting to " + queueARN,
|
|
|
|
Net: "tcp",
|
|
|
|
Err: err,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-10-03 20:29:55 -04:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
2016-12-15 11:23:48 -05:00
|
|
|
// Load Kafka targets, initialize their respective loggers.
|
2017-02-09 18:20:54 -05:00
|
|
|
for accountID, kafkaN := range serverConfig.Notify.GetKafka() {
|
2016-12-15 11:23:48 -05:00
|
|
|
if !kafkaN.Enable {
|
|
|
|
continue
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
|
|
|
if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeKafka, newKafkaNotify); err != nil {
|
2016-12-15 11:23:48 -05:00
|
|
|
if _, ok := err.(net.Error); ok {
|
2017-02-24 21:27:52 -05:00
|
|
|
err = &net.OpError{
|
|
|
|
Op: "Connecting to " + queueARN,
|
|
|
|
Net: "tcp",
|
2016-12-15 11:23:48 -05:00
|
|
|
Err: err,
|
|
|
|
}
|
|
|
|
}
|
2017-02-24 21:27:52 -05:00
|
|
|
|
2016-12-15 11:23:48 -05:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
2016-10-03 20:29:55 -04:00
|
|
|
|
2016-08-05 01:01:58 -04:00
|
|
|
// Successfully initialized queue targets.
|
|
|
|
return queueTargets, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Global instance of event notification queue.
|
2016-09-10 05:23:28 -04:00
|
|
|
var globalEventNotifier *eventNotifier
|
2016-08-05 01:01:58 -04:00
|
|
|
|
|
|
|
// Initialize event notifier.
|
|
|
|
func initEventNotifier(objAPI ObjectLayer) error {
|
|
|
|
if objAPI == nil {
|
|
|
|
return errInvalidArgument
|
|
|
|
}
|
|
|
|
|
|
|
|
// Read all saved bucket notifications.
|
2016-10-12 04:03:50 -04:00
|
|
|
nConfigs, lConfigs, err := loadAllBucketNotifications(objAPI)
|
2016-08-05 01:01:58 -04:00
|
|
|
if err != nil {
|
2016-10-12 04:03:50 -04:00
|
|
|
errorIf(err, "Error loading bucket notifications - %v", err)
|
2016-08-05 01:01:58 -04:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Initializes all queue targets.
|
|
|
|
queueTargets, err := loadAllQueueTargets()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-10-12 04:03:50 -04:00
|
|
|
// Initialize internal listener targets
|
|
|
|
listenTargets := make(map[string]*listenerLogger)
|
|
|
|
for _, listeners := range lConfigs {
|
|
|
|
for _, listener := range listeners {
|
|
|
|
ln, err := newListenerLogger(
|
|
|
|
listener.TopicConfig.TopicARN,
|
|
|
|
listener.TargetServer,
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
errorIf(err, "Unable to initialize listener target logger.")
|
|
|
|
//TODO: improve error
|
|
|
|
return fmt.Errorf("Error initializing listner target logger - %v", err)
|
|
|
|
}
|
|
|
|
listenTargets[listener.TopicConfig.TopicARN] = ln
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Initialize event notifier queue.
|
2016-09-10 05:23:28 -04:00
|
|
|
globalEventNotifier = &eventNotifier{
|
2016-10-12 04:03:50 -04:00
|
|
|
external: externalNotifier{
|
|
|
|
notificationConfigs: nConfigs,
|
|
|
|
targets: queueTargets,
|
|
|
|
rwMutex: &sync.RWMutex{},
|
|
|
|
},
|
|
|
|
internal: internalNotifier{
|
|
|
|
rwMutex: &sync.RWMutex{},
|
|
|
|
targets: listenTargets,
|
|
|
|
listenerConfigs: lConfigs,
|
|
|
|
connectedListeners: make(map[string]chan []NotificationEvent),
|
|
|
|
},
|
2016-08-05 01:01:58 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|