Fix Mux Connect Error (#18567)

`OpMuxConnectError` was not handled correctly.

Remove local checks for single request handlers so they can 
run before being registered locally.

Bonus: Only log IAM bootstrap on startup.

internal/grid/connection.go

@@ -321,10 +321,7 @@ func (c *Connection) Request(ctx context.Context, h HandlerID, req []byte) ([]by
 	if c.State() != StateConnected {
 		return nil, ErrDisconnected
 	}
-	handler := c.handlers.single[h]
-	if handler == nil {
-		return nil, ErrUnknownHandler
-	}
+	// Create mux client and call.
 	client, err := c.newMuxClient(ctx)
 	if err != nil {
 		return nil, err
@@ -349,10 +346,7 @@ func (c *Subroute) Request(ctx context.Context, h HandlerID, req []byte) ([]byte
 	if c.State() != StateConnected {
 		return nil, ErrDisconnected
 	}
-	handler := c.handlers.subSingle[makeZeroSubHandlerID(h)]
-	if handler == nil {
-		return nil, ErrUnknownHandler
-	}
+	// Create mux client and call.
 	client, err := c.newMuxClient(ctx)
 	if err != nil {
 		return nil, err
@@ -1159,6 +1153,8 @@ func (c *Connection) handleMsg(ctx context.Context, m message, subID *subHandler
 		c.handleAckMux(ctx, m)
 	case OpConnectMux:
 		c.handleConnectMux(ctx, m, subID)
+	case OpMuxConnectError:
+		c.handleConnectMuxError(ctx, m)
 	default:
 		logger.LogIf(ctx, fmt.Errorf("unknown message type: %v", m.Op))
 	}
@@ -1210,6 +1206,18 @@ func (c *Connection) handleConnectMux(ctx context.Context, m message, subID *sub
 	}
 }
 
+// handleConnectMuxError when mux connect was rejected.
+func (c *Connection) handleConnectMuxError(ctx context.Context, m message) {
+	if v, ok := c.outgoing.Load(m.MuxID); ok {
+		var cErr muxConnectError
+		_, err := cErr.UnmarshalMsg(m.Payload)
+		logger.LogIf(ctx, err)
+		v.error(RemoteErr(cErr.Error))
+		return
+	}
+	PutByteBuffer(m.Payload)
+}
+
 func (c *Connection) handleAckMux(ctx context.Context, m message) {
 	PutByteBuffer(m.Payload)
 	v, ok := c.outgoing.Load(m.MuxID)

internal/grid/grid_test.go

@@ -129,6 +129,54 @@ func TestSingleRoundtrip(t *testing.T) {
 	})
 }
 
+func TestSingleRoundtripNotReady(t *testing.T) {
+	defer testlogger.T.SetLogTB(t)()
+	errFatal := func(t testing.TB, err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	grid, err := SetupTestGrid(2)
+	errFatal(t, err)
+	remoteHost := grid.Hosts[1]
+	local := grid.Managers[0]
+
+	// 1: Echo
+	errFatal(t, local.RegisterSingleHandler(handlerTest, func(payload []byte) ([]byte, *RemoteErr) {
+		t.Log("1: server payload: ", len(payload), "bytes.")
+		return append([]byte{}, payload...), nil
+	}))
+	// 2: Return as error
+	errFatal(t, local.RegisterSingleHandler(handlerTest2, func(payload []byte) ([]byte, *RemoteErr) {
+		t.Log("2: server payload: ", len(payload), "bytes.")
+		err := RemoteErr(payload)
+		return nil, &err
+	}))
+
+	// Do not register remote handlers
+
+	// local to remote
+	remoteConn := local.Connection(remoteHost)
+	remoteConn.WaitForConnect(context.Background())
+	defer testlogger.T.SetErrorTB(t)()
+
+	t.Run("localToRemote", func(t *testing.T) {
+		const testPayload = "Hello Grid World!"
+		// Single requests should have remote errors.
+		_, err := remoteConn.Request(context.Background(), handlerTest, []byte(testPayload))
+		if v, ok := err.(*RemoteErr); !ok || v.Error() != "Invalid Handler for type" {
+			t.Fatalf("Unexpected error: %v, %T", err, err)
+		}
+		// Streams should not be able to set up until registered.
+		// Thus, the error is a local error.
+		_, err = remoteConn.NewStream(context.Background(), handlerTest, []byte(testPayload))
+		if !errors.Is(err, ErrUnknownHandler) {
+			t.Fatalf("Unexpected error: %v, %T", err, err)
+		}
+	})
+}
+
 func TestSingleRoundtripGenerics(t *testing.T) {
 	defer testlogger.T.SetLogTB(t)()
 	errFatal := func(err error) {

internal/grid/handlers.go

@@ -89,7 +89,7 @@ var handlerPrefixes = [handlerLast]string{
 }
 
 const (
-	lockPrefix    = "lock"
+	lockPrefix    = "lockR"
 	storagePrefix = "storageR"
 )