MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-09 05:34:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

optimize Listen bucket notification implementation (#9444)

Browse Source 
this commit avoids lots of tiny allocations, repeated
channel creates which are performed when filtering
the incoming events, unescaping a key just for matching.

also remove deprecated code which is not needed
anymore, avoids unexpected data structure transformations
from the map to slice.
This commit is contained in:
Harshavardhana
2020-04-27 06:25:05 -07:00
committed by GitHub
parent f216670814
commit f14bf25cb9
17 changed files with 103 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/event/rules.go
View File

@@ -53,6 +53,16 @@ func (rules Rules) Add(pattern string, targetID TargetID) {
rules[pattern] = NewTargetIDSet(targetID).Union(rules[pattern])
}
// MatchSimple - returns true one of the matching object name in rules.
func (rules Rules) MatchSimple(objectName string) bool {
for pattern := range rules {
if wildcard.MatchSimple(pattern, objectName) {
return true
}
}
return false
}
// Match - returns TargetIDSet matching object name in rules.
func (rules Rules) Match(objectName string) TargetIDSet {
targetIDs := NewTargetIDSet()

5
pkg/event/rulesmap.go
View File

@@ -60,6 +60,11 @@ func (rulesMap RulesMap) Remove(rulesMap2 RulesMap) {
}
}
// MatchSimple - returns true if matching object name and event name in rules map.
func (rulesMap RulesMap) MatchSimple(eventName Name, objectName string) bool {
return rulesMap[eventName].MatchSimple(objectName)
}
// Match - returns TargetIDSet matching object name and event name in rules map.
func (rulesMap RulesMap) Match(eventName Name, objectName string) TargetIDSet {
return rulesMap[eventName].Match(objectName)

17
pkg/event/targetidset.go
View File

@@ -16,23 +16,12 @@
package event
import "fmt"
// TargetIDSet - Set representation of TargetIDs.
type TargetIDSet map[TargetID]struct{}
// ToSlice - returns TargetID slice from TargetIDSet.
func (set TargetIDSet) ToSlice() []TargetID {
keys := make([]TargetID, 0, len(set))
for k := range set {
keys = append(keys, k)
}
return keys
}
// String - returns string representation.
func (set TargetIDSet) String() string {
return fmt.Sprintf("%v", set.ToSlice())
// IsEmpty returns true if the set is empty.
func (set TargetIDSet) IsEmpty() bool {
return len(set) != 0
}
// Clone - returns copy of this set.

50
pkg/event/targetidset_test.go
View File

@@ -21,56 +21,6 @@ import (
"testing"
)
func TestTargetIDSetToSlice(t *testing.T) {
testCases := []struct {
set TargetIDSet
expectedResult []TargetID
}{
{NewTargetIDSet(), []TargetID{}},
{NewTargetIDSet(TargetID{"1", "webhook"}), []TargetID{{"1", "webhook"}}},
{NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"}), []TargetID{{"1", "webhook"}, {"2", "amqp"}}},
}
for i, testCase := range testCases {
result := testCase.set.ToSlice()
if len(result) != len(testCase.expectedResult) {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
for _, targetID1 := range result {
var found bool
for _, targetID2 := range testCase.expectedResult {
if reflect.DeepEqual(targetID1, targetID2) {
found = true
break
}
}
if !found {
t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
}
}
}
func TestTargetIDSetString(t *testing.T) {
testCases := []struct {
set TargetIDSet
expectedResult string
}{
{NewTargetIDSet(), "[]"},
{NewTargetIDSet(TargetID{"1", "webhook"}), "[1:webhook]"},
}
for i, testCase := range testCases {
result := testCase.set.String()
if result != testCase.expectedResult {
t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
}
}
}
func TestTargetIDSetClone(t *testing.T) {
testCases := []struct {
set TargetIDSet

63
pkg/event/targetlist.go
View File

@@ -61,8 +61,9 @@ func (list *TargetList) Exists(id TargetID) bool {
return found
}
// TargetIDErr returns error associated for a targetID
type TargetIDErr struct {
// TargetIDResult returns result of Remove/Send operation, sets err if
// any for the associated TargetID
type TargetIDResult struct {
// ID where the remove or send were initiated.
ID TargetID
// Stores any error while removing a target or while sending an event.
@@ -70,40 +71,17 @@ type TargetIDErr struct {
}
// Remove - closes and removes targets by given target IDs.
func (list *TargetList) Remove(targetids ...TargetID) <-chan TargetIDErr {
errCh := make(chan TargetIDErr)
func (list *TargetList) Remove(targetIDSet TargetIDSet) {
list.Lock()
defer list.Unlock()
go func() {
defer close(errCh)
var wg sync.WaitGroup
for _, id := range targetids {
list.RLock()
target, ok := list.targets[id]
list.RUnlock()
if ok {
wg.Add(1)
go func(id TargetID, target Target) {
defer wg.Done()
if err := target.Close(); err != nil {
errCh <- TargetIDErr{
ID: id,
Err: err,
}
}
}(id, target)
}
}
wg.Wait()
list.Lock()
for _, id := range targetids {
for id := range targetIDSet {
target, ok := list.targets[id]
if ok {
target.Close()
delete(list.targets, id)
}
list.Unlock()
}()
return errCh
}
}
// Targets - list all targets
@@ -140,14 +118,10 @@ func (list *TargetList) TargetMap() map[TargetID]Target {
}
// Send - sends events to targets identified by target IDs.
func (list *TargetList) Send(event Event, targetIDs ...TargetID) <-chan TargetIDErr {
errCh := make(chan TargetIDErr)
func (list *TargetList) Send(event Event, targetIDset TargetIDSet, resCh chan<- TargetIDResult) {
go func() {
defer close(errCh)
var wg sync.WaitGroup
for _, id := range targetIDs {
for id := range targetIDset {
list.RLock()
target, ok := list.targets[id]
list.RUnlock()
@@ -155,19 +129,18 @@ func (list *TargetList) Send(event Event, targetIDs ...TargetID) <-chan TargetID
wg.Add(1)
go func(id TargetID, target Target) {
defer wg.Done()
tgtRes := TargetIDResult{ID: id}
if err := target.Save(event); err != nil {
errCh <- TargetIDErr{
ID: id,
Err: err,
}
tgtRes.Err = err
}
resCh <- tgtRes
}(id, target)
} else {
resCh <- TargetIDResult{ID: id}
}
}
wg.Wait()
}()
return errCh
}
// NewTargetList - creates TargetList.

43
pkg/event/targetlist_test.go
View File

@@ -163,40 +163,6 @@ func TestTargetListExists(t *testing.T) {
}
}
func TestTargetListRemove(t *testing.T) {
targetListCase1 := NewTargetList()
targetListCase2 := NewTargetList()
if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil {
panic(err)
}
targetListCase3 := NewTargetList()
if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, true}); err != nil {
panic(err)
}
testCases := []struct {
targetList *TargetList
targetID TargetID
expectErr bool
}{
{targetListCase1, TargetID{"1", "webhook"}, false},
{targetListCase2, TargetID{"1", "webhook"}, false},
{targetListCase3, TargetID{"3", "testcase"}, true},
}
for i, testCase := range testCases {
errCh := testCase.targetList.Remove(testCase.targetID)
err := <-errCh
expectErr := (err.Err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
}
}
}
func TestTargetListList(t *testing.T) {
targetListCase1 := NewTargetList()
@@ -273,10 +239,13 @@ func TestTargetListSend(t *testing.T) {
{targetListCase4, TargetID{"4", "testcase"}, true},
}
resCh := make(chan TargetIDResult)
for i, testCase := range testCases {
errCh := testCase.targetList.Send(Event{}, testCase.targetID)
err := <-errCh
expectErr := (err.Err != nil)
testCase.targetList.Send(Event{}, map[TargetID]struct{}{
testCase.targetID: {},
}, resCh)
res := <-resCh
expectErr := (res.Err != nil)
if expectErr != testCase.expectErr {
t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)