mirror of
https://github.com/minio/minio.git
synced 2025-11-10 05:59:43 -05:00
feature: added nsq as broker for events (#6740)
This commit is contained in:
Matthias Schneider
committed by
kannappanr
kannappanr
parent
bf414068a3
commit
71c66464c1
116
pkg/event/target/nsq.go
Normal file
116
pkg/event/target/nsq.go
Normal file
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package target
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
|
||||
"github.com/nsqio/go-nsq"
|
||||
|
||||
"github.com/minio/minio/pkg/event"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
// NSQArgs - NSQ target arguments.
|
||||
type NSQArgs struct {
|
||||
Enable bool `json:"enable"`
|
||||
NSQDAddress xnet.Host `json:"nsqdAddress"`
|
||||
Topic string `json:"topic"`
|
||||
TLS struct {
|
||||
Enable bool `json:"enable"`
|
||||
SkipVerify bool `json:"skipVerify"`
|
||||
} `json:"tls"`
|
||||
}
|
||||
|
||||
// Validate NSQArgs fields
|
||||
func (n NSQArgs) Validate() error {
|
||||
if !n.Enable {
|
||||
return nil
|
||||
}
|
||||
|
||||
if n.NSQDAddress.IsEmpty() {
|
||||
return errors.New("empty nsqdAddress")
|
||||
}
|
||||
|
||||
if n.Topic == "" {
|
||||
return errors.New("empty topic")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NSQTarget - NSQ target.
|
||||
type NSQTarget struct {
|
||||
id event.TargetID
|
||||
args NSQArgs
|
||||
producer *nsq.Producer
|
||||
}
|
||||
|
||||
// ID - returns target ID.
|
||||
func (target *NSQTarget) ID() event.TargetID {
|
||||
return target.id
|
||||
}
|
||||
|
||||
// Send - sends event to NSQD.
|
||||
func (target *NSQTarget) Send(eventData event.Event) (err error) {
|
||||
objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := eventData.S3.Bucket.Name + "/" + objectName
|
||||
|
||||
data, err := json.Marshal(event.Log{eventData.EventName, key, []event.Event{eventData}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = target.producer.Publish(target.args.Topic, data)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Close - closes underneath connections to NSQD server.
|
||||
func (target *NSQTarget) Close() (err error) {
|
||||
// this blocks until complete:
|
||||
target.producer.Stop()
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewNSQTarget - creates new NSQ target.
|
||||
func NewNSQTarget(id string, args NSQArgs) (*NSQTarget, error) {
|
||||
config := nsq.NewConfig()
|
||||
if args.TLS.Enable {
|
||||
config.TlsV1 = true
|
||||
config.TlsConfig = &tls.Config{
|
||||
InsecureSkipVerify: args.TLS.SkipVerify,
|
||||
}
|
||||
}
|
||||
producer, err := nsq.NewProducer(args.NSQDAddress.String(), config)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &NSQTarget{
|
||||
id: event.TargetID{id, "nsq"},
|
||||
args: args,
|
||||
producer: producer,
|
||||
}, nil
|
||||
}
|
||||
157
pkg/event/target/nsq_test.go
Normal file
157
pkg/event/target/nsq_test.go
Normal file
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package target
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/net"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
"github.com/nsqio/go-nsq"
|
||||
)
|
||||
|
||||
func TestNewNSQTarget(t *testing.T) {
|
||||
type args struct {
|
||||
id string
|
||||
args NSQArgs
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want *NSQTarget
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "test1",
|
||||
args: args{
|
||||
id: "id",
|
||||
args: NSQArgs{
|
||||
Enable: true,
|
||||
Topic: "",
|
||||
TLS: struct {
|
||||
Enable bool `json:"enable"`
|
||||
SkipVerify bool `json:"skipVerify"`
|
||||
}{true, true},
|
||||
},
|
||||
},
|
||||
want: &NSQTarget{
|
||||
id: event.TargetID{ID: "id", Name: "nsq"},
|
||||
args: NSQArgs{
|
||||
Enable: true,
|
||||
NSQDAddress: net.Host{},
|
||||
Topic: "",
|
||||
TLS: struct {
|
||||
Enable bool `json:"enable"`
|
||||
SkipVerify bool `json:"skipVerify"`
|
||||
}{true, true},
|
||||
},
|
||||
producer: &nsq.Producer{},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := NewNSQTarget(tt.args.id, tt.args.args)
|
||||
// dirty hack, otherwhise cannot compare the pointers:
|
||||
tt.want.producer = got.producer
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("NewNSQTarget() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("NewNSQTarget() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNSQArgs_Validate(t *testing.T) {
|
||||
type fields struct {
|
||||
Enable bool
|
||||
NSQDAddress xnet.Host
|
||||
Topic string
|
||||
TLS struct {
|
||||
Enable bool
|
||||
SkipVerify bool
|
||||
}
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
fields fields
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "test1_missing_topic",
|
||||
fields: fields{
|
||||
Enable: true,
|
||||
NSQDAddress: xnet.Host{
|
||||
Name: "127.0.0.1",
|
||||
Port: 4150,
|
||||
IsPortSet: true,
|
||||
},
|
||||
Topic: "",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "test2_disabled",
|
||||
fields: fields{
|
||||
Enable: false,
|
||||
NSQDAddress: xnet.Host{},
|
||||
Topic: "topic",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "test3_OK",
|
||||
fields: fields{
|
||||
Enable: true,
|
||||
NSQDAddress: xnet.Host{
|
||||
Name: "127.0.0.1",
|
||||
Port: 4150,
|
||||
IsPortSet: true,
|
||||
},
|
||||
Topic: "topic",
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "test4_emptynsqdaddr",
|
||||
fields: fields{
|
||||
Enable: true,
|
||||
NSQDAddress: xnet.Host{},
|
||||
Topic: "topic",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
n := NSQArgs{
|
||||
Enable: tt.fields.Enable,
|
||||
NSQDAddress: tt.fields.NSQDAddress,
|
||||
Topic: tt.fields.Topic,
|
||||
}
|
||||
if err := n.Validate(); (err != nil) != tt.wantErr {
|
||||
t.Errorf("NSQArgs.Validate() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user