diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 0e2b93c31..4f2f9f063 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1526,6 +1526,11 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) { return } + // Publish bootstrap events that have already occurred before client could subscribe. + if traceOpts.TraceTypes().Contains(madmin.TraceBootstrap) { + go globalBootstrapTracer.Publish(ctx, globalTrace) + } + for _, peer := range peers { if peer == nil { continue diff --git a/cmd/bootstrap-messages.go b/cmd/bootstrap-messages.go new file mode 100644 index 000000000..55f7a0739 --- /dev/null +++ b/cmd/bootstrap-messages.go @@ -0,0 +1,116 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/minio/madmin-go/v2" + "github.com/minio/minio/internal/pubsub" +) + +const bootstrapMsgsLimit = 4 << 10 + +type bootstrapInfo struct { + msg string + ts time.Time + source string +} +type bootstrapTracer struct { + mu sync.RWMutex + idx int + info [bootstrapMsgsLimit]bootstrapInfo + lastUpdate time.Time +} + +var globalBootstrapTracer = &bootstrapTracer{} + +func (bs *bootstrapTracer) DropEvents() { + bs.mu.Lock() + defer bs.mu.Unlock() + + if time.Now().UTC().Sub(bs.lastUpdate) > 24*time.Hour { + bs.info = [4096]bootstrapInfo{} + bs.idx = 0 + } +} + +func (bs *bootstrapTracer) Empty() bool { + var empty bool + bs.mu.RLock() + empty = bs.info[0].msg == "" + bs.mu.RUnlock() + + return empty +} + +func (bs *bootstrapTracer) Record(msg string) { + source := getSource(2) + bs.mu.Lock() + now := time.Now().UTC() + bs.info[bs.idx] = bootstrapInfo{ + msg: msg, + ts: now, + source: source, + } + bs.lastUpdate = now + bs.idx = (bs.idx + 1) % bootstrapMsgsLimit + bs.mu.Unlock() +} + +func (bs *bootstrapTracer) Events() []madmin.TraceInfo { + traceInfo := make([]madmin.TraceInfo, 0, bootstrapMsgsLimit) + + // Add all messages in order + addAll := func(info []bootstrapInfo) { + for _, msg := range info { + if msg.ts.IsZero() { + continue // skip empty events + } + traceInfo = append(traceInfo, madmin.TraceInfo{ + TraceType: madmin.TraceBootstrap, + Time: msg.ts, + NodeName: globalLocalNodeName, + FuncName: "BOOTSTRAP", + Message: fmt.Sprintf("%s %s", msg.source, msg.msg), + }) + } + } + + bs.mu.RLock() + addAll(bs.info[bs.idx:]) + addAll(bs.info[:bs.idx]) + bs.mu.RUnlock() + return traceInfo +} + +func (bs *bootstrapTracer) Publish(ctx context.Context, trace *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) { + if bs.Empty() { + return + } + for _, bsEvent := range bs.Events() { + select { + case <-ctx.Done(): + default: + trace.Publish(bsEvent) + } + } +} diff --git a/cmd/bootstrap-messages_test.go b/cmd/bootstrap-messages_test.go new file mode 100644 index 000000000..2aa47e756 --- /dev/null +++ b/cmd/bootstrap-messages_test.go @@ -0,0 +1,60 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestBootstrap(t *testing.T) { + // Bootstrap events exceed bootstrap messages limit + bsTracer := &bootstrapTracer{} + for i := 0; i < bootstrapMsgsLimit+10; i++ { + bsTracer.Record(fmt.Sprintf("msg-%d", i)) + } + + traceInfos := bsTracer.Events() + if len(traceInfos) != bootstrapMsgsLimit { + t.Fatalf("Expected length of events %d but got %d", bootstrapMsgsLimit, len(traceInfos)) + } + + // Simulate the case where bootstrap events were updated a day ago + bsTracer.lastUpdate = time.Now().UTC().Add(-25 * time.Hour) + bsTracer.DropEvents() + if !bsTracer.Empty() { + t.Fatalf("Expected all bootstrap events to have been dropped, but found %d events", len(bsTracer.Events())) + } + + // Fewer than 4K bootstrap events + for i := 0; i < 10; i++ { + bsTracer.Record(fmt.Sprintf("msg-%d", i)) + } + events := bsTracer.Events() + if len(events) != 10 { + t.Fatalf("Expected length of events %d but got %d", 10, len(events)) + } + for i, traceInfo := range bsTracer.Events() { + msg := fmt.Sprintf("msg-%d", i) + if !strings.HasSuffix(traceInfo.Message, msg) { + t.Fatalf("Expected %s but got %s", msg, traceInfo.Message) + } + } +} diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index 3af2d399e..c9c7d1add 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -977,6 +977,12 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) { s.writeErrorResponse(w, err) return } + + // Publish bootstrap events that have already occurred before client could subscribe. + if traceOpts.TraceTypes().Contains(madmin.TraceBootstrap) { + go globalBootstrapTracer.Publish(r.Context(), globalTrace) + } + keepAliveTicker := time.NewTicker(500 * time.Millisecond) defer keepAliveTicker.Stop() diff --git a/cmd/server-main.go b/cmd/server-main.go index cdc050d49..46f598f04 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -352,6 +352,8 @@ func configRetriableErrors(err error) bool { } func bootstrapTrace(msg string) { + globalBootstrapTracer.Record(msg) + if globalTrace.NumSubscribers(madmin.TraceBootstrap) == 0 { return }