separate lock from common grid to avoid epoll contention (#20180)

epoll contention on TCP causes latency build-up when
we have high volume ingress. This PR is an attempt to
relieve this pressure.

upstream issue https://github.com/golang/go/issues/65064
It seems to be a deeper problem; haven't yet tried the fix
provide in this issue, but however this change without
changing the compiler helps. 

Of course, this is a workaround for now, hoping for a
more comprehensive fix from Go runtime.

internal/grid/connection.go

@@ -1319,7 +1319,11 @@ func (c *Connection) handleConnectMux(ctx context.Context, m message, subID *sub
 			handler = c.handlers.subStateless[*subID]
 		}
 		if handler == nil {
-			gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: "Invalid Handler for type"}))
+			msg := fmt.Sprintf("Invalid Handler for type: %v", m.Handler)
+			if subID != nil {
+				msg = fmt.Sprintf("Invalid Handler for type: %v", *subID)
+			}
+			gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: msg}))
 			return
 		}
 		_, _ = c.inStream.LoadOrCompute(m.MuxID, func() *muxServer {
@@ -1338,7 +1342,11 @@ func (c *Connection) handleConnectMux(ctx context.Context, m message, subID *sub
 			handler = c.handlers.subStreams[*subID]
 		}
 		if handler == nil {
-			gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: "Invalid Handler for type"}))
+			msg := fmt.Sprintf("Invalid Handler for type: %v", m.Handler)
+			if subID != nil {
+				msg = fmt.Sprintf("Invalid Handler for type: %v", *subID)
+			}
+			gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: msg}))
 			return
 		}
 
@@ -1392,7 +1400,11 @@ func (c *Connection) handleRequest(ctx context.Context, m message, subID *subHan
 		handler = c.handlers.subSingle[*subID]
 	}
 	if handler == nil {
-		gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: "Invalid Handler for type"}))
+		msg := fmt.Sprintf("Invalid Handler for type: %v", m.Handler)
+		if subID != nil {
+			msg = fmt.Sprintf("Invalid Handler for type: %v", *subID)
+		}
+		gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: msg}))
 		return
 	}
 

internal/grid/debug.go

@@ -26,7 +26,6 @@ import (
 	"sync"
 	"time"
 
-	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/mux"
 )
 
@@ -90,6 +89,7 @@ func SetupTestGrid(n int) (*TestGrid, error) {
 			AuthFn:       dummyNewToken,
 			AuthToken:    dummyTokenValidate,
 			BlockConnect: ready,
+			RoutePath:    RoutePath,
 		})
 		if err != nil {
 			return nil, err
@@ -101,7 +101,7 @@ func SetupTestGrid(n int) (*TestGrid, error) {
 		res.Listeners = append(res.Listeners, listeners[i])
 		res.Mux = append(res.Mux, m)
 	}
-	xioutil.SafeClose(ready)
+	close(ready)
 	for _, m := range res.Managers {
 		for _, remote := range m.Targets() {
 			if err := m.Connection(remote).WaitForConnect(ctx); err != nil {

internal/grid/grid.go

@@ -202,13 +202,12 @@ func bytesOrLength(b []byte) string {
 // The net.Conn must support all features as described by the net.Conn interface.
 type ConnDialer func(ctx context.Context, address string) (net.Conn, error)
 
-// ConnectWS returns a function that dials a websocket connection to the given address.
-// Route and auth are added to the connection.
-func ConnectWS(dial ContextDialer, auth AuthFn, tls *tls.Config) func(ctx context.Context, remote string) (net.Conn, error) {
+// ConnectWSWithRoutePath is like ConnectWS but with a custom grid route path.
+func ConnectWSWithRoutePath(dial ContextDialer, auth AuthFn, tls *tls.Config, routePath string) func(ctx context.Context, remote string) (net.Conn, error) {
 	return func(ctx context.Context, remote string) (net.Conn, error) {
 		toDial := strings.Replace(remote, "http://", "ws://", 1)
 		toDial = strings.Replace(toDial, "https://", "wss://", 1)
-		toDial += RoutePath
+		toDial += routePath
 
 		dialer := ws.DefaultDialer
 		dialer.ReadBufferSize = readBufferSize
@@ -234,5 +233,11 @@ func ConnectWS(dial ContextDialer, auth AuthFn, tls *tls.Config) func(ctx contex
 	}
 }
 
+// ConnectWS returns a function that dials a websocket connection to the given address.
+// Route and auth are added to the connection.
+func ConnectWS(dial ContextDialer, auth AuthFn, tls *tls.Config) func(ctx context.Context, remote string) (net.Conn, error) {
+	return ConnectWSWithRoutePath(dial, auth, tls, RoutePath)
+}
+
 // ValidateTokenFn must validate the token and return an error if it is invalid.
 type ValidateTokenFn func(token string) error

internal/grid/grid_test.go

@@ -166,7 +166,7 @@ func TestSingleRoundtripNotReady(t *testing.T) {
 		const testPayload = "Hello Grid World!"
 		// Single requests should have remote errors.
 		_, err := remoteConn.Request(context.Background(), handlerTest, []byte(testPayload))
-		if v, ok := err.(*RemoteErr); !ok || v.Error() != "Invalid Handler for type" {
+		if _, ok := err.(*RemoteErr); !ok {
 			t.Fatalf("Unexpected error: %v, %T", err, err)
 		}
 		// Streams should not be able to set up until registered.

internal/grid/manager.go

@@ -45,6 +45,9 @@ const (
 
 	// RoutePath is the remote path to connect to.
 	RoutePath = "/minio/grid/" + apiVersion
+
+	// RouteLockPath is the remote lock path to connect to.
+	RouteLockPath = "/minio/grid/lock/" + apiVersion
 )
 
 // Manager will contain all the connections to the grid.
@@ -65,6 +68,9 @@ type Manager struct {
 
 	// authToken is a function that will validate a token.
 	authToken ValidateTokenFn
+
+	// routePath indicates the dial route path
+	routePath string
 }
 
 // ManagerOptions are options for creating a new grid manager.
@@ -74,6 +80,7 @@ type ManagerOptions struct {
 	Incoming     func(n int64) // Record incoming bytes.
 	Outgoing     func(n int64) // Record outgoing bytes.
 	BlockConnect chan struct{} // If set, incoming and outgoing connections will be blocked until closed.
+	RoutePath    string
 	TraceTo      *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]
 	Dialer       ConnDialer
 	// Sign a token for the given audience.
@@ -99,6 +106,7 @@ func NewManager(ctx context.Context, o ManagerOptions) (*Manager, error) {
 		targets:   make(map[string]*Connection, len(o.Hosts)),
 		local:     o.Local,
 		authToken: o.AuthToken,
+		routePath: o.RoutePath,
 	}
 	m.handlers.init()
 	if ctx == nil {
@@ -137,7 +145,7 @@ func NewManager(ctx context.Context, o ManagerOptions) (*Manager, error) {
 
 // AddToMux will add the grid manager to the given mux.
 func (m *Manager) AddToMux(router *mux.Router, authReq func(r *http.Request) error) {
-	router.Handle(RoutePath, m.Handler(authReq))
+	router.Handle(m.routePath, m.Handler(authReq))
 }
 
 // Handler returns a handler that can be used to serve grid requests.