MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-10 05:59:43 -05:00
Code Issues Packages Projects Releases Wiki Activity

avoid limits on the number of parallel trace/bucket notifications listeners (#14799)

Browse Source 
Simplifies overall limits on the incoming listeners for notifications.

Fixes #14566
This commit is contained in:
Anis Elleuch
2022-06-05 22:29:12 +01:00
committed by GitHub
parent addfa35d93
commit fd02492cb7
8 changed files with 113 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/admin-handlers.go
View File

@@ -1409,9 +1409,15 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
peers, _ := newPeerRestClients(globalEndpoints)
globalTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool {
traceFn := func(entry interface{}) bool {
return mustTrace(entry, traceOpts)
})
}
err = globalTrace.Subscribe(traceCh, ctx.Done(), traceFn)
if err != nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
return
}
for _, peer := range peers {
if peer == nil {
@@ -1483,7 +1489,11 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
peers, _ := newPeerRestClients(globalEndpoints)
globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
err = globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
if err != nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
return
}
for _, peer := range peers {
if peer == nil {

4
cmd/api-router.go
View File

@@ -342,7 +342,7 @@ func registerAPIRouter(router *mux.Router) {
collectAPIStats("getbucketnotification", maxClients(gz(httpTraceAll(api.GetBucketNotificationHandler))))).Queries("notification", "")
// ListenNotification
router.Methods(http.MethodGet).HandlerFunc(
collectAPIStats("listennotification", maxClients(gz(httpTraceAll(api.ListenNotificationHandler))))).Queries("events", "{events:.*}")
collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
// ResetBucketReplicationStatus - MinIO extension API
router.Methods(http.MethodGet).HandlerFunc(
collectAPIStats("resetbucketreplicationstatus", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStatusHandler))))).Queries("replication-reset-status", "")
@@ -474,7 +474,7 @@ func registerAPIRouter(router *mux.Router) {
// ListenNotification
apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
collectAPIStats("listennotification", maxClients(gz(httpTraceAll(api.ListenNotificationHandler))))).Queries("events", "{events:.*}")
collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
// ListBuckets
apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(

9
cmd/consolelogger.go
View File

@@ -45,7 +45,7 @@ type HTTPConsoleLoggerSys struct {
// NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to
// the console logging pub sub system
func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
ps := pubsub.New()
ps := pubsub.New(8)
return &HTTPConsoleLoggerSys{
pubsub: ps,
console: console.New(),
@@ -75,7 +75,7 @@ func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
}
// Subscribe starts console logging for this node.
func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) {
func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) error {
// Enable console logging for remote client.
if !sys.HasLogListeners() {
logger.AddSystemTarget(sys)
@@ -111,11 +111,12 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan
select {
case subCh <- entry:
case <-doneCh:
return
return nil
}
}
}
sys.pubsub.Subscribe(subCh, doneCh, filter)
return sys.pubsub.Subscribe(subCh, doneCh, filter)
}
// Init if HTTPConsoleLoggerSys is valid, always returns nil right now

4
cmd/globals.go
View File

@@ -226,10 +226,10 @@ var (
// global Trace system to send HTTP request/response
// and Storage/OS calls info to registered listeners.
globalTrace = pubsub.New()
globalTrace = pubsub.New(8)
// global Listen system to send S3 API events to registered listeners
globalHTTPListen = pubsub.New()
globalHTTPListen = pubsub.New(0)
// global console system to send console logs to
// registered listeners

10
cmd/listen-notification-handlers.go
View File

@@ -127,7 +127,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
peers, _ := newPeerRestClients(globalEndpoints)
globalHTTPListen.Subscribe(listenCh, ctx.Done(), func(evI interface{}) bool {
listenFn := func(evI interface{}) bool {
ev, ok := evI.(event.Event)
if !ok {
return false
@@ -138,7 +138,13 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
}
}
return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
})
}
err := globalHTTPListen.Subscribe(listenCh, ctx.Done(), listenFn)
if err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
return
}
if bucketName != "" {
values.Set(peerRESTListenBucket, bucketName)

26
cmd/peer-rest-server.go
View File

@@ -933,7 +933,7 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
// Use buffered channel to take care of burst sends or slow w.Write()
ch := make(chan interface{}, 2000)
globalHTTPListen.Subscribe(ch, doneCh, func(evI interface{}) bool {
listenFn := func(evI interface{}) bool {
ev, ok := evI.(event.Event)
if !ok {
return false
@@ -944,7 +944,13 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
}
}
return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
})
}
err := globalHTTPListen.Subscribe(ch, doneCh, listenFn)
if err != nil {
s.writeErrorResponse(w, err)
return
}
keepAliveTicker := time.NewTicker(500 * time.Millisecond)
defer keepAliveTicker.Stop()
@@ -1005,9 +1011,15 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
// Use buffered channel to take care of burst sends or slow w.Write()
ch := make(chan interface{}, 2000)
globalTrace.Subscribe(ch, doneCh, func(entry interface{}) bool {
traceFn := func(entry interface{}) bool {
return mustTrace(entry, traceOpts)
})
}
err = globalTrace.Subscribe(ch, doneCh, traceFn)
if err != nil {
s.writeErrorResponse(w, err)
return
}
keepAliveTicker := time.NewTicker(500 * time.Millisecond)
defer keepAliveTicker.Stop()
@@ -1092,7 +1104,11 @@ func (s *peerRESTServer) ConsoleLogHandler(w http.ResponseWriter, r *http.Reques
defer close(doneCh)
ch := make(chan interface{}, 2000)
globalConsoleSys.Subscribe(ch, doneCh, "", 0, string(logger.All), nil)
err := globalConsoleSys.Subscribe(ch, doneCh, "", 0, string(logger.All), nil)
if err != nil {
s.writeErrorResponse(w, err)
return
}
keepAliveTicker := time.NewTicker(500 * time.Millisecond)
defer keepAliveTicker.Stop()