Release v0.3.0

vendor/github.com/btcsuite/btcwallet/chain/bitcoind_conn.go

@@ -16,6 +16,28 @@ import (
 	"github.com/lightninglabs/gozmq"
 )
 
+const (
+	// rawBlockZMQCommand is the command used to receive raw block
+	// notifications from bitcoind through ZMQ.
+	rawBlockZMQCommand = "rawblock"
+
+	// rawTxZMQCommand is the command used to receive raw transaction
+	// notifications from bitcoind through ZMQ.
+	rawTxZMQCommand = "rawtx"
+
+	// maxRawBlockSize is the maximum size in bytes for a raw block received
+	// from bitcoind through ZMQ.
+	maxRawBlockSize = 4e6
+
+	// maxRawTxSize is the maximum size in bytes for a raw transaction
+	// received from bitcoind through ZMQ.
+	maxRawTxSize = maxRawBlockSize
+
+	// seqNumLen is the length of the sequence number of a message sent from
+	// bitcoind through ZMQ.
+	seqNumLen = 4
+)
+
 // BitcoindConn represents a persistent client connection to a bitcoind node
 // that listens for events read from a ZMQ connection.
 type BitcoindConn struct {
@@ -79,7 +101,7 @@ func NewBitcoindConn(chainParams *chaincfg.Params,
 	// concern to ensure one type of event isn't dropped from the connection
 	// queue due to another type of event filling it up.
 	zmqBlockConn, err := gozmq.Subscribe(
-		zmqBlockHost, []string{"rawblock"}, zmqPollInterval,
+		zmqBlockHost, []string{rawBlockZMQCommand}, zmqPollInterval,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("unable to subscribe for zmq block "+
@@ -87,7 +109,7 @@ func NewBitcoindConn(chainParams *chaincfg.Params,
 	}
 
 	zmqTxConn, err := gozmq.Subscribe(
-		zmqTxHost, []string{"rawtx"}, zmqPollInterval,
+		zmqTxHost, []string{rawTxZMQCommand}, zmqPollInterval,
 	)
 	if err != nil {
 		zmqBlockConn.Close()
@@ -164,6 +186,20 @@ func (c *BitcoindConn) blockEventHandler() {
 	log.Info("Started listening for bitcoind block notifications via ZMQ "+
 		"on", c.zmqBlockConn.RemoteAddr())
 
+	// Set up the buffers we expect our messages to consume. ZMQ
+	// messages from bitcoind include three parts: the command, the
+	// data, and the sequence number.
+	//
+	// We'll allocate a fixed data slice that we'll reuse when reading
+	// blocks from bitcoind through ZMQ. There's no need to recycle this
+	// slice (zero out) after using it, as further reads will overwrite the
+	// slice and we'll only be deserializing the bytes needed.
+	var (
+		command [len(rawBlockZMQCommand)]byte
+		seqNum  [seqNumLen]byte
+		data    = make([]byte, maxRawBlockSize)
+	)
+
 	for {
 		// Before attempting to read from the ZMQ socket, we'll make
 		// sure to check if we've been requested to shut down.
@@ -174,7 +210,11 @@ func (c *BitcoindConn) blockEventHandler() {
 		}
 
 		// Poll an event from the ZMQ socket.
-		msgBytes, err := c.zmqBlockConn.Receive()
+		var (
+			bufs = [][]byte{command[:], data, seqNum[:]}
+			err  error
+		)
+		bufs, err = c.zmqBlockConn.Receive(bufs)
 		if err != nil {
 			// EOF should only be returned if the connection was
 			// explicitly closed, so we can exit at this point.
@@ -187,22 +227,24 @@ func (c *BitcoindConn) blockEventHandler() {
 			// error to prevent spamming the logs.
 			netErr, ok := err.(net.Error)
 			if ok && netErr.Timeout() {
+				log.Trace("Re-establishing timed out ZMQ " +
+					"block connection")
 				continue
 			}
 
-			log.Errorf("Unable to receive ZMQ rawblock message: %v",
-				err)
+			log.Errorf("Unable to receive ZMQ %v message: %v",
+				rawBlockZMQCommand, err)
 			continue
 		}
 
 		// We have an event! We'll now ensure it is a block event,
 		// deserialize it, and report it to the different rescan
 		// clients.
-		eventType := string(msgBytes[0])
+		eventType := string(bufs[0])
 		switch eventType {
-		case "rawblock":
+		case rawBlockZMQCommand:
 			block := &wire.MsgBlock{}
-			r := bytes.NewReader(msgBytes[1])
+			r := bytes.NewReader(bufs[1])
 			if err := block.Deserialize(r); err != nil {
 				log.Errorf("Unable to deserialize block: %v",
 					err)
@@ -229,8 +271,9 @@ func (c *BitcoindConn) blockEventHandler() {
 				continue
 			}
 
-			log.Warnf("Received unexpected event type from "+
-				"rawblock subscription: %v", eventType)
+			log.Warnf("Received unexpected event type from %v "+
+				"subscription: %v", rawBlockZMQCommand,
+				eventType)
 		}
 	}
 }
@@ -245,6 +288,20 @@ func (c *BitcoindConn) txEventHandler() {
 	log.Info("Started listening for bitcoind transaction notifications "+
 		"via ZMQ on", c.zmqTxConn.RemoteAddr())
 
+	// Set up the buffers we expect our messages to consume. ZMQ
+	// messages from bitcoind include three parts: the command, the
+	// data, and the sequence number.
+	//
+	// We'll allocate a fixed data slice that we'll reuse when reading
+	// transactions from bitcoind through ZMQ. There's no need to recycle
+	// this slice (zero out) after using it, as further reads will overwrite
+	// the slice and we'll only be deserializing the bytes needed.
+	var (
+		command [len(rawTxZMQCommand)]byte
+		seqNum  [seqNumLen]byte
+		data    = make([]byte, maxRawTxSize)
+	)
+
 	for {
 		// Before attempting to read from the ZMQ socket, we'll make
 		// sure to check if we've been requested to shut down.
@@ -255,7 +312,11 @@ func (c *BitcoindConn) txEventHandler() {
 		}
 
 		// Poll an event from the ZMQ socket.
-		msgBytes, err := c.zmqTxConn.Receive()
+		var (
+			bufs = [][]byte{command[:], data, seqNum[:]}
+			err  error
+		)
+		bufs, err = c.zmqTxConn.Receive(bufs)
 		if err != nil {
 			// EOF should only be returned if the connection was
 			// explicitly closed, so we can exit at this point.
@@ -268,22 +329,24 @@ func (c *BitcoindConn) txEventHandler() {
 			// error to prevent spamming the logs.
 			netErr, ok := err.(net.Error)
 			if ok && netErr.Timeout() {
+				log.Trace("Re-establishing timed out ZMQ " +
+					"transaction connection")
 				continue
 			}
 
-			log.Errorf("Unable to receive ZMQ rawtx message: %v",
-				err)
+			log.Errorf("Unable to receive ZMQ %v message: %v",
+				rawTxZMQCommand, err)
 			continue
 		}
 
 		// We have an event! We'll now ensure it is a transaction event,
 		// deserialize it, and report it to the different rescan
 		// clients.
-		eventType := string(msgBytes[0])
+		eventType := string(bufs[0])
 		switch eventType {
-		case "rawtx":
+		case rawTxZMQCommand:
 			tx := &wire.MsgTx{}
-			r := bytes.NewReader(msgBytes[1])
+			r := bytes.NewReader(bufs[1])
 			if err := tx.Deserialize(r); err != nil {
 				log.Errorf("Unable to deserialize "+
 					"transaction: %v", err)
@@ -310,8 +373,8 @@ func (c *BitcoindConn) txEventHandler() {
 				continue
 			}
 
-			log.Warnf("Received unexpected event type from rawtx "+
-				"subscription: %v", eventType)
+			log.Warnf("Received unexpected event type from %v "+
+				"subscription: %v", rawTxZMQCommand, eventType)
 		}
 	}
 }

vendor/github.com/btcsuite/btcwallet/chain/neutrino.go

@@ -199,6 +199,11 @@ func (s *NeutrinoClient) FilterBlocks(
 	// the filter returns a positive match, the full block is then requested
 	// and scanned for addresses using the block filterer.
 	for i, blk := range req.Blocks {
+		// TODO(wilmer): Investigate why polling it still necessary
+		// here. While testing, I ran into a few instances where the
+		// filter was not retrieved, leading to a panic. This should not
+		// happen in most cases thanks to the query logic revamp within
+		// Neutrino, but it seems there's still an uncovered edge case.
 		filter, err := s.pollCFilter(&blk.Hash)
 		if err != nil {
 			return nil, err
@@ -312,7 +317,9 @@ func (s *NeutrinoClient) pollCFilter(hash *chainhash.Hash) (*gcs.Filter, error)
 			time.Sleep(100 * time.Millisecond)
 		}
 
-		filter, err = s.CS.GetCFilter(*hash, wire.GCSFilterRegular)
+		filter, err = s.CS.GetCFilter(
+			*hash, wire.GCSFilterRegular, neutrino.OptimisticBatch(),
+		)
 		if err != nil {
 			count++
 			continue