Mux: Send async mux ack and fix stream error responses (#19149)

Streams can return errors if the cancelation is picked up before the response 
stream close is picked up. Under extreme load, this could lead to missing 
responses.

Send server mux ack async so a blocked send cannot block newMuxStream 
call. Stream will not progress until mux has been acked.

internal/grid/muxclient.go

@@ -323,7 +323,10 @@ func (m *muxClient) handleOneWayStream(respHandler chan<- Response, respServer <
 			if debugPrint {
 				fmt.Println("Client sending disconnect to mux", m.MuxID)
 			}
-			m.addErrorNonBlockingClose(respHandler, context.Cause(m.ctx))
+			err := context.Cause(m.ctx)
+			if !errors.Is(err, errStreamEOF) {
+				m.addErrorNonBlockingClose(respHandler, err)
+			}
 			return
 		case resp, ok := <-respServer:
 			if !ok {
@@ -463,6 +466,7 @@ func (m *muxClient) response(seq uint32, r Response) {
 			fmt.Println(m.MuxID, m.parent.String(), "CHECKSEQ FAIL", m.RecvSeq, seq)
 		}
 		PutByteBuffer(r.Msg)
+		r.Msg = nil
 		r.Err = ErrIncorrectSequence
 		m.addResponse(r)
 		return
@@ -474,6 +478,8 @@ func (m *muxClient) response(seq uint32, r Response) {
 	}
 }
 
+var errStreamEOF = errors.New("stream EOF")
+
 // error is a message from the server to disconnect.
 func (m *muxClient) error(err RemoteErr) {
 	if debugPrint {