diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index 7408d8242..e506fc30c 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -732,7 +732,7 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http encoder.Encode(dErrsResp) } -var storageRenameDataHandler = grid.NewSingleHandler[*RenameDataHandlerParams, *RenameDataResp](grid.HandlerRenamedata, func() *RenameDataHandlerParams { +var storageRenameDataHandler = grid.NewSingleHandler[*RenameDataHandlerParams, *RenameDataResp](grid.HandlerRenameData, func() *RenameDataHandlerParams { return &RenameDataHandlerParams{} }, func() *RenameDataResp { return &RenameDataResp{} diff --git a/internal/grid/connection.go b/internal/grid/connection.go index 395a13fc9..f6617e0c7 100644 --- a/internal/grid/connection.go +++ b/internal/grid/connection.go @@ -336,7 +336,7 @@ func (c *Connection) Request(ctx context.Context, h HandlerID, req []byte) ([]by } c.outgoing.Delete(client.MuxID) }() - return client.traceRoundtrip(c.trace, h, req) + return client.traceRoundtrip(ctx, c.trace, h, req) } // Request allows to do a single remote request. @@ -364,7 +364,7 @@ func (c *Subroute) Request(ctx context.Context, h HandlerID, req []byte) ([]byte } c.outgoing.Delete(client.MuxID) }() - return client.traceRoundtrip(c.trace, h, req) + return client.traceRoundtrip(ctx, c.trace, h, req) } // NewStream creates a new stream. diff --git a/internal/grid/handlers.go b/internal/grid/handlers.go index b705ba340..83b57c544 100644 --- a/internal/grid/handlers.go +++ b/internal/grid/handlers.go @@ -55,7 +55,7 @@ const ( HandlerUpdateMetadata HandlerWriteMetadata HandlerCheckParts - HandlerRenamedata + HandlerRenameData // Add more above here ^^^ // If all handlers are used, the type of Handler can be changed. @@ -65,6 +65,34 @@ const ( handlerLast ) +// handlerPrefixes are prefixes for handler IDs used for tracing. +// If a handler is not listed here, it will be traced with "grid" prefix. +var handlerPrefixes = [handlerLast]string{ + HandlerLockLock: lockPrefix, + HandlerLockRLock: lockPrefix, + HandlerLockUnlock: lockPrefix, + HandlerLockRUnlock: lockPrefix, + HandlerLockRefresh: lockPrefix, + HandlerLockForceUnlock: lockPrefix, + HandlerWalkDir: storagePrefix, + HandlerStatVol: storagePrefix, + HandlerDiskInfo: storagePrefix, + HandlerNSScanner: storagePrefix, + HandlerReadXL: storagePrefix, + HandlerReadVersion: storagePrefix, + HandlerDeleteFile: storagePrefix, + HandlerDeleteVersion: storagePrefix, + HandlerUpdateMetadata: storagePrefix, + HandlerWriteMetadata: storagePrefix, + HandlerCheckParts: storagePrefix, + HandlerRenameData: storagePrefix, +} + +const ( + lockPrefix = "lock" + storagePrefix = "storageR" +) + func init() { // Static check if we exceed 255 handler ids. // Extend the type to uint16 when hit. @@ -364,6 +392,7 @@ func (h *SingleHandler[Req, Resp]) Call(ctx context.Context, c Requester, req Re if err != nil { return resp, err } + ctx = context.WithValue(ctx, TraceParamsKey{}, req) res, err := c.Request(ctx, h.id, payload) PutByteBuffer(payload) if err != nil { diff --git a/internal/grid/handlers_string.go b/internal/grid/handlers_string.go index 160920aea..8c20fee5f 100644 --- a/internal/grid/handlers_string.go +++ b/internal/grid/handlers_string.go @@ -26,13 +26,13 @@ func _() { _ = x[HandlerUpdateMetadata-15] _ = x[HandlerWriteMetadata-16] _ = x[HandlerCheckParts-17] - _ = x[HandlerRenamedata-18] + _ = x[HandlerRenameData-18] _ = x[handlerTest-19] _ = x[handlerTest2-20] _ = x[handlerLast-21] } -const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenamedatahandlerTesthandlerTest2handlerLast" +const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDatahandlerTesthandlerTest2handlerLast" var _HandlerID_index = [...]uint8{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 207, 219, 230} diff --git a/internal/grid/trace.go b/internal/grid/trace.go index 58c7ef116..cb8ad0938 100644 --- a/internal/grid/trace.go +++ b/internal/grid/trace.go @@ -18,19 +18,27 @@ package grid import ( + "context" + "fmt" "net/http" + "net/url" "time" "github.com/minio/madmin-go/v3" "github.com/minio/minio/internal/pubsub" ) +// TraceParamsKey allows to pass trace parameters to the request via context. +// This is only needed when un-typed requests are used. +// MSS, map[string]string types are preferred, but any struct with exported fields will work. +type TraceParamsKey struct{} + // traceRequests adds request tracing to the connection. func (c *Connection) traceRequests(p *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) { c.trace = &tracer{ Publisher: p, TraceType: madmin.TraceInternal, - Prefix: "grid.", + Prefix: "grid", Local: c.Local, Remote: c.Remote, Subroute: "", @@ -56,7 +64,7 @@ type tracer struct { Subroute string } -func (c *muxClient) traceRoundtrip(t *tracer, h HandlerID, req []byte) ([]byte, error) { +func (c *muxClient) traceRoundtrip(ctx context.Context, t *tracer, h HandlerID, req []byte) ([]byte, error) { if t == nil || t.Publisher.NumSubscribers(t.TraceType) == 0 { return c.roundtrip(h, req) } @@ -74,9 +82,14 @@ func (c *muxClient) traceRoundtrip(t *tracer, h HandlerID, req []byte) ([]byte, status = http.StatusBadRequest } } + + prefix := t.Prefix + if p := handlerPrefixes[h]; p != "" { + prefix = p + } trace := madmin.TraceInfo{ TraceType: t.TraceType, - FuncName: t.Prefix + h.String(), + FuncName: prefix + "." + h.String(), NodeName: t.Local, Time: start, Duration: end.Sub(start), @@ -105,6 +118,20 @@ func (c *muxClient) traceRoundtrip(t *tracer, h HandlerID, req []byte) ([]byte, }, }, } + // If the context contains a TraceParamsKey, add it to the trace path. + v := ctx.Value(TraceParamsKey{}) + if p, ok := v.(*MSS); ok && p != nil { + trace.Path += p.ToQuery() + trace.HTTP.ReqInfo.Path = trace.Path + } else if p, ok := v.(map[string]string); ok { + m := MSS(p) + trace.Path += m.ToQuery() + trace.HTTP.ReqInfo.Path = trace.Path + } else if v != nil { + // Print exported fields as single request to path. + trace.Path = fmt.Sprintf("%s?req=%s", trace.Path, url.QueryEscape(fmt.Sprintf("%+v", v))) + trace.HTTP.ReqInfo.Path = trace.Path + } t.Publisher.Publish(trace) return resp, err } diff --git a/internal/grid/types.go b/internal/grid/types.go index c9dee4abe..35851f286 100644 --- a/internal/grid/types.go +++ b/internal/grid/types.go @@ -19,6 +19,9 @@ package grid import ( "errors" + "net/url" + "sort" + "strings" "github.com/tinylib/msgp/msgp" ) @@ -117,6 +120,31 @@ func NewMSSWith(m map[string]string) *MSS { return &m2 } +// ToQuery constructs a URL query string from the MSS, including "?" if there are any keys. +func (m MSS) ToQuery() string { + if len(m) == 0 { + return "" + } + var buf strings.Builder + buf.WriteByte('?') + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := m[k] + keyEscaped := url.QueryEscape(k) + if buf.Len() > 1 { + buf.WriteByte('&') + } + buf.WriteString(keyEscaped) + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(v)) + } + return buf.String() +} + // NewBytes returns a new Bytes. func NewBytes() *Bytes { b := Bytes(GetByteBuffer()[:0])