Merge pull request #6345 from ellemouton/bitcoind-rpc-polling

multi: bitcoind rpc polling option

.github/workflows/main.yml

@@ -259,6 +259,8 @@ jobs:
             args: backend=bitcoind
           - name: bitcoind-notxindex
             args: backend="bitcoind notxindex"
+          - name: bitcoind-rpcpolling
+            args: backend="bitcoind rpcpolling"
           - name: bitcoind-etcd
             args: backend=bitcoind dbbackend=etcd
           - name: bitcoind-postgres

chainntnfs/bitcoindnotify/bitcoind.go

@@ -19,9 +19,14 @@ import (
 )
 
 const (
-	// notifierType uniquely identifies this concrete implementation of the
-	// ChainNotifier interface.
-	notifierType = "bitcoind"
+	// notifierType uniquely identifies a concrete implementation of the
+	// ChainNotifier interface that makes use of the bitcoind ZMQ interface.
+	notifierTypeZMQ = "bitcoind"
+
+	// notifierTypeRPCPolling uniquely identifies a concrete implementation
+	// of the ChainNotifier interface that makes use of the bitcoind RPC
+	// interface.
+	notifierTypeRPCPolling = "bitcoind-rpc-polling"
 )
 
 // TODO(roasbeef): generalize struct below:

chainntnfs/bitcoindnotify/bitcoind_test.go

@@ -108,13 +108,18 @@ func syncNotifierWithMiner(t *testing.T, notifier *BitcoindNotifier,
 // TestHistoricalConfDetailsTxIndex ensures that we correctly retrieve
 // historical confirmation details using the backend node's txindex.
 func TestHistoricalConfDetailsTxIndex(t *testing.T) {
+	testHistoricalConfDetailsTxIndex(t, true)
+	testHistoricalConfDetailsTxIndex(t, false)
+}
+
+func testHistoricalConfDetailsTxIndex(t *testing.T, rpcPolling bool) {
 	miner, tearDown := chainntnfs.NewMiner(
 		t, []string{"--txindex"}, true, 25,
 	)
 	defer tearDown()
 
 	bitcoindConn, cleanUp := chainntnfs.NewBitcoindBackend(
-		t, miner.P2PAddress(), true,
+		t, miner.P2PAddress(), true, rpcPolling,
 	)
 	defer cleanUp()
 
@@ -206,11 +211,16 @@ func TestHistoricalConfDetailsTxIndex(t *testing.T) {
 // historical confirmation details using the set of fallback methods when the
 // backend node's txindex is disabled.
 func TestHistoricalConfDetailsNoTxIndex(t *testing.T) {
+	testHistoricalConfDetailsNoTxIndex(t, true)
+	testHistoricalConfDetailsNoTxIndex(t, false)
+}
+
+func testHistoricalConfDetailsNoTxIndex(t *testing.T, rpcpolling bool) {
 	miner, tearDown := chainntnfs.NewMiner(t, nil, true, 25)
 	defer tearDown()
 
 	bitcoindConn, cleanUp := chainntnfs.NewBitcoindBackend(
-		t, miner.P2PAddress(), false,
+		t, miner.P2PAddress(), false, rpcpolling,
 	)
 	defer cleanUp()
 

chainntnfs/bitcoindnotify/driver.go

@@ -56,13 +56,21 @@ func createNewNotifier(args ...interface{}) (chainntnfs.ChainNotifier, error) {
 // chainntnfs.ChainNotifier interface.
 func init() {
 	// Register the driver.
-	notifier := &chainntnfs.NotifierDriver{
-		NotifierType: notifierType,
+	notifierZMQ := &chainntnfs.NotifierDriver{
+		NotifierType: notifierTypeZMQ,
 		New:          createNewNotifier,
 	}
-
-	if err := chainntnfs.RegisterNotifier(notifier); err != nil {
+	if err := chainntnfs.RegisterNotifier(notifierZMQ); err != nil {
 		panic(fmt.Sprintf("failed to register notifier driver '%s': %v",
-			notifierType, err))
+			notifierTypeZMQ, err))
+	}
+
+	notifierRPC := &chainntnfs.NotifierDriver{
+		NotifierType: notifierTypeRPCPolling,
+		New:          createNewNotifier,
+	}
+	if err := chainntnfs.RegisterNotifier(notifierRPC); err != nil {
+		panic(fmt.Sprintf("failed to register notifier driver '%s': %v",
+			notifierTypeRPCPolling, err))
 	}
 }

chainntnfs/test/bitcoind/bitcoind_test.go

@@ -13,4 +13,5 @@ import (
 // powered chain notifier.
 func TestInterfaces(t *testing.T) {
 	chainntnfstest.TestInterfaces(t, "bitcoind")
+	chainntnfstest.TestInterfaces(t, "bitcoind-rpc-polling")
 }

chainntnfs/test/test_interface.go

@@ -1966,7 +1966,19 @@ func TestInterfaces(t *testing.T, targetBackEnd string) {
 		case "bitcoind":
 			var bitcoindConn *chain.BitcoindConn
 			bitcoindConn, cleanUp = chainntnfs.NewBitcoindBackend(
-				t, p2pAddr, true,
+				t, p2pAddr, true, false,
+			)
+			newNotifier = func() (chainntnfs.TestChainNotifier, error) {
+				return bitcoindnotify.New(
+					bitcoindConn, chainntnfs.NetParams,
+					hintCache, hintCache, blockCache,
+				), nil
+			}
+
+		case "bitcoind-rpc-polling":
+			var bitcoindConn *chain.BitcoindConn
+			bitcoindConn, cleanUp = chainntnfs.NewBitcoindBackend(
+				t, p2pAddr, true, true,
 			)
 			newNotifier = func() (chainntnfs.TestChainNotifier, error) {
 				return bitcoindnotify.New(

chainntnfs/test_utils.go

@@ -26,6 +26,7 @@ import (
 	"github.com/btcsuite/btcwallet/walletdb"
 	"github.com/lightninglabs/neutrino"
 	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/lntest/wait"
 )
 
 var (
@@ -193,10 +194,12 @@ func NewMiner(t *testing.T, extraArgs []string, createChain bool,
 
 // NewBitcoindBackend spawns a new bitcoind node that connects to a miner at the
 // specified address. The txindex boolean can be set to determine whether the
-// backend node should maintain a transaction index. A connection to the newly
-// spawned bitcoind node is returned.
-func NewBitcoindBackend(t *testing.T, minerAddr string,
-	txindex bool) (*chain.BitcoindConn, func()) {
+// backend node should maintain a transaction index. The rpcpolling boolean
+// can be set to determine whether bitcoind's RPC polling interface should be
+// used for block and tx notifications or if its ZMQ interface should be used.
+// A connection to the newly spawned bitcoind node is returned.
+func NewBitcoindBackend(t *testing.T, minerAddr string, txindex,
+	rpcpolling bool) (*chain.BitcoindConn, func()) {
 
 	t.Helper()
 
@@ -231,29 +234,41 @@ func NewBitcoindBackend(t *testing.T, minerAddr string,
 	}
 
 	// Wait for the bitcoind instance to start up.
-	time.Sleep(time.Second)
-
 	host := fmt.Sprintf("127.0.0.1:%d", rpcPort)
-	conn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{
-		ChainParams:     NetParams,
-		Host:            host,
-		User:            "weks",
-		Pass:            "weks",
-		ZMQBlockHost:    zmqBlockHost,
-		ZMQTxHost:       zmqTxHost,
-		ZMQReadDeadline: 5 * time.Second,
+	cfg := &chain.BitcoindConfig{
+		ChainParams: NetParams,
+		Host:        host,
+		User:        "weks",
+		Pass:        "weks",
 		// Fields only required for pruned nodes, not needed for these
 		// tests.
 		Dialer:             nil,
 		PrunedModeMaxPeers: 0,
-	})
-	if err != nil {
-		bitcoind.Process.Kill()
-		bitcoind.Wait()
-		os.RemoveAll(tempBitcoindDir)
-		t.Fatalf("unable to establish connection to bitcoind: %v", err)
 	}
-	if err := conn.Start(); err != nil {
+
+	if rpcpolling {
+		cfg.ZMQConfig = &chain.ZMQConfig{
+			ZMQBlockHost:    zmqBlockHost,
+			ZMQTxHost:       zmqTxHost,
+			ZMQReadDeadline: 5 * time.Second,
+		}
+	} else {
+		cfg.PollingConfig = &chain.PollingConfig{
+			BlockPollingInterval: time.Millisecond * 20,
+			TxPollingInterval:    time.Millisecond * 20,
+		}
+	}
+
+	var conn *chain.BitcoindConn
+	err = wait.NoError(func() error {
+		conn, err = chain.NewBitcoindConn(cfg)
+		if err != nil {
+			return err
+		}
+
+		return conn.Start()
+	}, 10*time.Second)
+	if err != nil {
 		bitcoind.Process.Kill()
 		bitcoind.Wait()
 		os.RemoveAll(tempBitcoindDir)

chainreg/chainregistry.go

@@ -399,19 +399,31 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) {
 			}
 		}
 
-		// Establish the connection to bitcoind and create the clients
-		// required for our relevant subsystems.
-		bitcoindConn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{
+		bitcoindCfg := &chain.BitcoindConfig{
 			ChainParams:        cfg.ActiveNetParams.Params,
 			Host:               bitcoindHost,
 			User:               bitcoindMode.RPCUser,
 			Pass:               bitcoindMode.RPCPass,
-			ZMQBlockHost:       bitcoindMode.ZMQPubRawBlock,
-			ZMQTxHost:          bitcoindMode.ZMQPubRawTx,
-			ZMQReadDeadline:    5 * time.Second,
 			Dialer:             cfg.Dialer,
 			PrunedModeMaxPeers: bitcoindMode.PrunedNodeMaxPeers,
-		})
+		}
+
+		if bitcoindMode.RPCPolling {
+			bitcoindCfg.PollingConfig = &chain.PollingConfig{
+				BlockPollingInterval: bitcoindMode.BlockPollingInterval,
+				TxPollingInterval:    bitcoindMode.TxPollingInterval,
+			}
+		} else {
+			bitcoindCfg.ZMQConfig = &chain.ZMQConfig{
+				ZMQBlockHost:    bitcoindMode.ZMQPubRawBlock,
+				ZMQTxHost:       bitcoindMode.ZMQPubRawTx,
+				ZMQReadDeadline: bitcoindMode.ZMQReadDeadline,
+			}
+		}
+
+		// Establish the connection to bitcoind and create the clients
+		// required for our relevant subsystems.
+		bitcoindConn, err := chain.NewBitcoindConn(bitcoindCfg)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -495,7 +507,7 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) {
 		// version 0.17.0) we make sure lnd subscribes to the correct
 		// zmq events. We do this to avoid a situation in which we are
 		// not notified of new transactions or blocks.
-		if ver >= 170000 {
+		if ver >= 170000 && !bitcoindMode.RPCPolling {
 			zmqPubRawBlockURL, err := url.Parse(bitcoindMode.ZMQPubRawBlock)
 			if err != nil {
 				return nil, nil, err

config.go

@@ -83,6 +83,10 @@ const (
 	defaultTorV2PrivateKeyFilename = "v2_onion_private_key"
 	defaultTorV3PrivateKeyFilename = "v3_onion_private_key"
 
+	// defaultZMQReadDeadline is the default read deadline to be used for
+	// both the block and tx ZMQ subscriptions.
+	defaultZMQReadDeadline = 5 * time.Second
+
 	// DefaultAutogenValidity is the default validity of a self-signed
 	// certificate. The value corresponds to 14 months
 	// (14 months * 30 days * 24 hours).
@@ -483,6 +487,7 @@ func DefaultConfig() Config {
 			RPCHost:            defaultRPCHost,
 			EstimateMode:       defaultBitcoindEstimateMode,
 			PrunedNodeMaxPeers: defaultPrunedNodeMaxPeers,
+			ZMQReadDeadline:    defaultZMQReadDeadline,
 		},
 		Litecoin: &lncfg.Chain{
 			MinHTLCIn:     chainreg.DefaultLitecoinMinHTLCInMSat,
@@ -1772,11 +1777,19 @@ func parseRPCParams(cConfig *lncfg.Chain, nodeConfig interface{},
 			}
 		}
 
-		// If all of RPCUser, RPCPass, ZMQBlockHost, and ZMQTxHost are
-		// set, we assume those parameters are good to use.
-		if conf.RPCUser != "" && conf.RPCPass != "" &&
-			conf.ZMQPubRawBlock != "" && conf.ZMQPubRawTx != "" {
-			return nil
+		if conf.RPCUser != "" && conf.RPCPass != "" {
+			// If all of RPCUser, RPCPass, ZMQBlockHost, and
+			// ZMQTxHost are set, we assume those parameters are
+			// good to use.
+			if conf.ZMQPubRawBlock != "" && conf.ZMQPubRawTx != "" {
+				return nil
+			}
+
+			// If RPCUser and RPCPass are set and RPCPolling is
+			// enabled, we assume the parameters are good to use.
+			if conf.RPCPolling {
+				return nil
+			}
 		}
 
 		// Get the daemon name for displaying proper errors.

docs/release-notes/release-notes-0.15.0.md

@@ -64,6 +64,12 @@ to Bitcoin nodes that advertise a Tor v3 onion service address.
 capable of status checks, adding, disconnecting and listing peers, fetching
 compact filters and block/block headers.
 
+## Btcwallet
+
+* [Add option to configure the block and transaction subscription 
+  notifications from bitcoind to be obtained through polling of the RPC 
+  interface instead of using ZMQ](https://github.com/lightningnetwork/lnd/pull/6345)
+
 ## Bug Fixes
 
 * [Pipelining an UpdateFulfillHTLC message now only happens when the related UpdateAddHTLC is locked-in.](https://github.com/lightningnetwork/lnd/pull/6246)

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/btcsuite/btcd/btcutil/psbt v1.1.3
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
-	github.com/btcsuite/btcwallet v0.14.1-0.20220412233800-3a6d5d0702b7
+	github.com/btcsuite/btcwallet v0.15.0
 	github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3
 	github.com/btcsuite/btcwallet/wallet/txrules v1.2.0
 	github.com/btcsuite/btcwallet/walletdb v1.4.0

go.sum

@@ -95,8 +95,8 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtyd
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/btcwallet v0.14.1-0.20220412233800-3a6d5d0702b7 h1:K7omr0FIjwwwQXTMgp2d+DTRH71RlkjRKj/Ir6Zkqb0=
-github.com/btcsuite/btcwallet v0.14.1-0.20220412233800-3a6d5d0702b7/go.mod h1:EE9BactCCWhCFoVfxCJrSFINrYqLx/Tq6quxRlPTpzM=
+github.com/btcsuite/btcwallet v0.15.0 h1:FdgC7JySVQJIcU+3W+kswDPv8rtzArGNQLOn2g3TiLg=
+github.com/btcsuite/btcwallet v0.15.0/go.mod h1:EE9BactCCWhCFoVfxCJrSFINrYqLx/Tq6quxRlPTpzM=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.2.1/go.mod h1:/74bubxX5Js48d76nf/TsNabpYp/gndUuJw4chzCmhU=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3 h1:M2yr5UlULvpqtxUqpMxTME/pA92Z9cpqeyvAFk9lAg0=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3/go.mod h1:T2xSiKGpUkSLCh68aF+FMXmKK9mFqNdHl9VaqOr+JjU=

lncfg/bitcoind.go

@@ -1,16 +1,22 @@
 package lncfg
 
+import "time"
+
 // Bitcoind holds the configuration options for the daemon's connection to
 // bitcoind.
 type Bitcoind struct {
-	Dir                string `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."`
-	ConfigPath         string `long:"config" description:"Configuration filepath. If not set, will default to the default filename under 'dir'."`
-	RPCCookie          string `long:"rpccookie" description:"Authentication cookie file for RPC connections. If not set, will default to .cookie under 'dir'."`
-	RPCHost            string `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."`
-	RPCUser            string `long:"rpcuser" description:"Username for RPC connections"`
-	RPCPass            string `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`
-	ZMQPubRawBlock     string `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"`
-	ZMQPubRawTx        string `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"`
-	EstimateMode       string `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."`
-	PrunedNodeMaxPeers int    `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."`
+	Dir                  string        `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."`
+	ConfigPath           string        `long:"config" description:"Configuration filepath. If not set, will default to the default filename under 'dir'."`
+	RPCCookie            string        `long:"rpccookie" description:"Authentication cookie file for RPC connections. If not set, will default to .cookie under 'dir'."`
+	RPCHost              string        `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."`
+	RPCUser              string        `long:"rpcuser" description:"Username for RPC connections"`
+	RPCPass              string        `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`
+	ZMQPubRawBlock       string        `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"`
+	ZMQPubRawTx          string        `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"`
+	ZMQReadDeadline      time.Duration `long:"zmqreaddeadline" description:"The read deadline for reading ZMQ messages from both the block and tx subscriptions"`
+	EstimateMode         string        `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."`
+	PrunedNodeMaxPeers   int           `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."`
+	RPCPolling           bool          `long:"rpcpolling" description:"Poll the bitcoind RPC interface for block and transaction notifications instead of using the ZMQ interface"`
+	BlockPollingInterval time.Duration `long:"blockpollinginterval" description:"The interval that will be used to poll bitcoind for new blocks. Only used if rpcpolling is true."`
+	TxPollingInterval    time.Duration `long:"txpollinginterval" description:"The interval that will be used to poll bitcoind for new tx. Only used if rpcpolling is true."`
 }

lntest/bitcoind.go

@@ -1,5 +1,5 @@
-//go:build bitcoind && !notxindex
-// +build bitcoind,!notxindex
+//go:build bitcoind && !notxindex && !rpcpolling
+// +build bitcoind,!notxindex,!rpcpolling
 
 package lntest
 
@@ -19,5 +19,5 @@ func NewBackend(miner string, netParams *chaincfg.Params) (
 		"-disablewallet",
 	}
 
-	return newBackend(miner, netParams, extraArgs)
+	return newBackend(miner, netParams, extraArgs, false)
 }

lntest/bitcoind_common.go

@@ -29,6 +29,7 @@ type BitcoindBackendConfig struct {
 	zmqTxPath    string
 	p2pPort      int
 	rpcClient    *rpcclient.Client
+	rpcPolling   bool
 
 	// minerAddr is the p2p address of the miner to connect to.
 	minerAddr string
@@ -46,10 +47,19 @@ func (b BitcoindBackendConfig) GenArgs() []string {
 	args = append(args, fmt.Sprintf("--bitcoind.rpchost=%v", b.rpcHost))
 	args = append(args, fmt.Sprintf("--bitcoind.rpcuser=%v", b.rpcUser))
 	args = append(args, fmt.Sprintf("--bitcoind.rpcpass=%v", b.rpcPass))
-	args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawblock=%v",
-		b.zmqBlockPath))
-	args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawtx=%v",
-		b.zmqTxPath))
+
+	if b.rpcPolling {
+		args = append(args, fmt.Sprintf("--bitcoind.rpcpolling"))
+		args = append(args,
+			fmt.Sprintf("--bitcoind.blockpollinginterval=10ms"))
+		args = append(args,
+			fmt.Sprintf("--bitcoind.txpollinginterval=10ms"))
+	} else {
+		args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawblock=%v",
+			b.zmqBlockPath))
+		args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawtx=%v",
+			b.zmqTxPath))
+	}
 
 	return args
 }
@@ -76,8 +86,8 @@ func (b BitcoindBackendConfig) Name() string {
 
 // newBackend starts a bitcoind node with the given extra parameters and returns
 // a BitcoindBackendConfig for that node.
-func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
-	*BitcoindBackendConfig, func() error, error) {
+func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string,
+	rpcPolling bool) (*BitcoindBackendConfig, func() error, error) {
 
 	baseLogDir := fmt.Sprintf(logDirPattern, GetLogDir())
 	if netParams != &chaincfg.RegressionNetParams {
@@ -192,6 +202,7 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
 		p2pPort:      p2pPort,
 		rpcClient:    client,
 		minerAddr:    miner,
+		rpcPolling:   rpcPolling,
 	}
 
 	return &bd, cleanUp, nil

lntest/bitcoind_notxindex.go

@@ -1,5 +1,5 @@
-//go:build bitcoind && notxindex
-// +build bitcoind,notxindex
+//go:build bitcoind && notxindex && !rpcpolling
+// +build bitcoind,notxindex,!rpcpolling
 
 package lntest
 
@@ -18,5 +18,5 @@ func NewBackend(miner string, netParams *chaincfg.Params) (
 		"-disablewallet",
 	}
 
-	return newBackend(miner, netParams, extraArgs)
+	return newBackend(miner, netParams, extraArgs, false)
 }

lntest/bitcoind_rpcpolling.go

@@ -0,0 +1,23 @@
+//go:build bitcoind && rpcpolling
+// +build bitcoind,rpcpolling
+
+package lntest
+
+import (
+	"github.com/btcsuite/btcd/chaincfg"
+)
+
+// NewBackend starts a bitcoind node without the txindex enabled and returns a
+// BitoindBackendConfig for that node.
+func NewBackend(miner string, netParams *chaincfg.Params) (
+	*BitcoindBackendConfig, func() error, error) {
+
+	extraArgs := []string{
+		"-debug",
+		"-regtest",
+		"-txindex",
+		"-disablewallet",
+	}
+
+	return newBackend(miner, netParams, extraArgs, true)
+}

lnwallet/btcwallet/btcwallet.go

@@ -932,7 +932,17 @@ func (b *BtcWallet) LeaseOutput(id wtxmgr.LockID, op wire.OutPoint,
 
 // ListLeasedOutputs returns a list of all currently locked outputs.
 func (b *BtcWallet) ListLeasedOutputs() ([]*wtxmgr.LockedOutput, error) {
-	return b.wallet.ListLeasedOutputs()
+	leasedOutputs, err := b.wallet.ListLeasedOutputs()
+	if err != nil {
+		return nil, err
+	}
+
+	lockedOutputs := make([]*wtxmgr.LockedOutput, len(leasedOutputs))
+	for i, output := range leasedOutputs {
+		lockedOutputs[i] = output.LockedOutput
+	}
+
+	return lockedOutputs, nil
 }
 
 // ReleaseOutput unlocks an output, allowing it to be available for coin

lnwallet/test/bitcoind/bitcoind_test.go

@@ -6,8 +6,14 @@ import (
 	lnwallettest "github.com/lightningnetwork/lnd/lnwallet/test"
 )
 
-// TestLightningWallet tests LightningWallet powered by bitcoind against our
-// suite of interface tests.
-func TestLightningWallet(t *testing.T) {
+// TestLightningWalletBitcoindZMQ tests LightningWallet powered by bitcoind,
+// using its ZMQ interface, against our suite of interface tests.
+func TestLightningWalletBitcoindZMQ(t *testing.T) {
 	lnwallettest.TestLightningWallet(t, "bitcoind")
 }
+
+// TestLightningWalletBitcoindRPCPolling tests LightningWallet powered by
+// bitcoind, using its RPC interface, against our suite of interface tests.
+func TestLightningWalletBitcoindRPCPolling(t *testing.T) {
+	lnwallettest.TestLightningWallet(t, "bitcoind-rpc-polling")
+}

lnwallet/test/test_interface.go

@@ -6,7 +6,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io/ioutil"
-	"math/rand"
 	"net"
 	"os"
 	"os/exec"
@@ -14,6 +13,7 @@ import (
 	"reflect"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -96,6 +96,35 @@ var (
 	defaultMaxLocalCsvDelay uint16 = 10000
 )
 
+var (
+	// lastPort is the last port determined to be free for use by a new
+	// bitcoind server. It should be used atomically.
+	lastPort uint32 = 1024
+)
+
+// getFreePort returns the first port that is available for listening by a new
+// embedded etcd server. It panics if no port is found and the maximum available
+// TCP port is reached.
+func getFreePort() int {
+	port := atomic.AddUint32(&lastPort, 1)
+	for port < 65535 {
+		// If there are no errors while attempting to listen on this
+		// port, close the socket and return it as available.
+		addr := fmt.Sprintf("127.0.0.1:%d", port)
+		l, err := net.Listen("tcp4", addr)
+		if err == nil {
+			err := l.Close()
+			if err == nil {
+				return int(port)
+			}
+		}
+		port = atomic.AddUint32(&lastPort, 1)
+	}
+
+	// No ports available? Must be a mistake.
+	panic("no ports available for listening")
+}
+
 // assertProperBalance asserts than the total value of the unspent outputs
 // within the wallet are *exactly* amount. If unable to retrieve the current
 // balance, or the assertion fails, the test will halt with a fatal error.
@@ -3377,7 +3406,7 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 			zmqBlockHost := "ipc:///" + tempBitcoindDir + "/blocks.socket"
 			zmqTxHost := "ipc:///" + tempBitcoindDir + "/tx.socket"
 			defer os.RemoveAll(tempBitcoindDir)
-			rpcPort := rand.Int()%(65536-1024) + 1024
+			rpcPort := getFreePort()
 			bitcoind := exec.Command(
 				"bitcoind",
 				"-datadir="+tempBitcoindDir,
@@ -3396,8 +3425,18 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 			if err != nil {
 				t.Fatalf("couldn't start bitcoind: %v", err)
 			}
-			defer bitcoind.Wait()
-			defer bitcoind.Process.Kill()
+
+			// Sanity check to ensure that the process did in fact
+			// start.
+			if bitcoind.Process == nil {
+				t.Fatalf("bitcoind cmd Process is not set " +
+					"after Start")
+			}
+
+			defer func() {
+				_ = bitcoind.Process.Kill()
+				_ = bitcoind.Wait()
+			}()
 
 			// Wait for the bitcoind instance to start up.
 
@@ -3405,13 +3444,15 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 			var chainConn *chain.BitcoindConn
 			err = wait.NoError(func() error {
 				chainConn, err = chain.NewBitcoindConn(&chain.BitcoindConfig{
-					ChainParams:     netParams,
-					Host:            host,
-					User:            "weks",
-					Pass:            "weks",
-					ZMQBlockHost:    zmqBlockHost,
-					ZMQTxHost:       zmqTxHost,
-					ZMQReadDeadline: 5 * time.Second,
+					ChainParams: netParams,
+					Host:        host,
+					User:        "weks",
+					Pass:        "weks",
+					ZMQConfig: &chain.ZMQConfig{
+						ZMQBlockHost:    zmqBlockHost,
+						ZMQTxHost:       zmqTxHost,
+						ZMQReadDeadline: 5 * time.Second,
+					},
 					// Fields only required for pruned nodes, not
 					// needed for these tests.
 					Dialer:             nil,
@@ -3433,6 +3474,78 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 			// Bob.
 			aliceClient = chainConn.NewBitcoindClient()
 			bobClient = chainConn.NewBitcoindClient()
+
+		case "bitcoind-rpc-polling":
+			// Start a bitcoind instance.
+			tempBitcoindDir, err := ioutil.TempDir("", "bitcoind")
+			if err != nil {
+				t.Fatalf("unable to create temp directory: %v", err)
+			}
+			defer os.RemoveAll(tempBitcoindDir)
+			rpcPort := getFreePort()
+			bitcoind := exec.Command(
+				"bitcoind",
+				"-datadir="+tempBitcoindDir,
+				"-regtest",
+				"-connect="+miningNode.P2PAddress(),
+				"-txindex",
+				"-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f"+
+					"d$507c670e800a95284294edb5773b05544b"+
+					"220110063096c221be9933c82d38e1",
+				fmt.Sprintf("-rpcport=%d", rpcPort),
+				"-disablewallet",
+			)
+			err = bitcoind.Start()
+			if err != nil {
+				t.Fatalf("couldn't start bitcoind: %v", err)
+			}
+			defer func() {
+				_ = bitcoind.Process.Kill()
+				_ = bitcoind.Wait()
+			}()
+
+			// Sanity check to ensure that the process did in fact
+			// start.
+			if bitcoind.Process == nil {
+				t.Fatalf("bitcoind cmd Process is not set " +
+					"after Start")
+			}
+
+			// Wait for the bitcoind instance to start up.
+			host := fmt.Sprintf("127.0.0.1:%d", rpcPort)
+			var chainConn *chain.BitcoindConn
+			err = wait.NoError(func() error {
+				chainConn, err = chain.NewBitcoindConn(&chain.BitcoindConfig{
+					ChainParams: netParams,
+					Host:        host,
+					User:        "weks",
+					Pass:        "weks",
+					PollingConfig: &chain.PollingConfig{
+						BlockPollingInterval: time.Millisecond * 20,
+						TxPollingInterval:    time.Millisecond * 20,
+					},
+					// Fields only required for pruned nodes, not
+					// needed for these tests.
+					Dialer:             nil,
+					PrunedModeMaxPeers: 0,
+				})
+				if err != nil {
+					return err
+				}
+
+				return chainConn.Start()
+			}, 10*time.Second)
+			if err != nil {
+				t.Fatalf("unable to establish connection to "+
+					"bitcoind: %v", err)
+			}
+			defer chainConn.Stop()
+
+			// Create a btcwallet bitcoind client for both Alice and
+			// Bob.
+			aliceClient = chainConn.NewBitcoindClient()
+			bobClient = chainConn.NewBitcoindClient()
+
 		default:
 			t.Fatalf("unknown chain driver: %v", backEnd)
 		}

routing/chainview/interface_test.go

@@ -4,11 +4,12 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
-	"math/rand"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -28,6 +29,7 @@ import (
 	"github.com/lightningnetwork/lnd/blockcache"
 	"github.com/lightningnetwork/lnd/channeldb"
 	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/lntest/wait"
 )
 
 var (
@@ -48,6 +50,35 @@ var (
 	testScript, _ = txscript.PayToAddrScript(testAddr)
 )
 
+var (
+	// lastPort is the last port determined to be free for use by a new
+	// bitcoind server. It should be used atomically.
+	lastPort uint32 = 1024
+)
+
+// getFreePort returns the first port that is available for listening by a new
+// embedded etcd server. It panics if no port is found and the maximum available
+// TCP port is reached.
+func getFreePort() int {
+	port := atomic.AddUint32(&lastPort, 1)
+	for port < 65535 {
+		// If there are no errors while attempting to listen on this
+		// port, close the socket and return it as available.
+		addr := fmt.Sprintf("127.0.0.1:%d", port)
+		l, err := net.Listen("tcp4", addr)
+		if err == nil {
+			err := l.Close()
+			if err == nil {
+				return int(port)
+			}
+		}
+		port = atomic.AddUint32(&lastPort, 1)
+	}
+
+	// No ports available? Must be a mistake.
+	panic("no ports available for listening")
+}
+
 func waitForMempoolTx(r *rpctest.Harness, txid *chainhash.Hash) error {
 	var found bool
 	var tx *btcutil.Tx
@@ -560,9 +591,14 @@ func testFilterBlockDisconnected(node *rpctest.Harness,
 		t.Fatalf("unable to set up mining node: %v", err)
 	}
 
+	_, bestHeight, err := reorgNode.Client.GetBestBlock()
+	if err != nil {
+		t.Fatalf("error getting best block: %v", err)
+	}
+
 	// Init a chain view that has this node as its block source.
 	cleanUpFunc, reorgView, err := chainViewInit(
-		reorgNode.RPCConfig(), reorgNode.P2PAddress(),
+		reorgNode.RPCConfig(), reorgNode.P2PAddress(), bestHeight,
 	)
 	if err != nil {
 		t.Fatalf("unable to create chain view: %v", err)
@@ -751,7 +787,7 @@ func testFilterBlockDisconnected(node *rpctest.Harness,
 }
 
 type chainViewInitFunc func(rpcInfo rpcclient.ConnConfig,
-	p2pAddr string) (func(), FilteredChainView, error)
+	p2pAddr string, bestHeight int32) (func(), FilteredChainView, error)
 
 type testCase struct {
 	name string
@@ -785,7 +821,8 @@ var interfaceImpls = []struct {
 	{
 		name: "bitcoind_zmq",
 		chainViewInit: func(_ rpcclient.ConnConfig,
-			p2pAddr string) (func(), FilteredChainView, error) {
+			p2pAddr string, bestHeight int32) (func(),
+			FilteredChainView, error) {
 
 			// Start a bitcoind instance.
 			tempBitcoindDir, err := ioutil.TempDir("", "bitcoind")
@@ -797,7 +834,7 @@ var interfaceImpls = []struct {
 			cleanUp1 := func() {
 				os.RemoveAll(tempBitcoindDir)
 			}
-			rpcPort := rand.Int()%(65536-1024) + 1024
+			rpcPort := getFreePort()
 			bitcoind := exec.Command(
 				"bitcoind",
 				"-datadir="+tempBitcoindDir,
@@ -817,35 +854,170 @@ var interfaceImpls = []struct {
 				cleanUp1()
 				return nil, nil, err
 			}
+
+			// Sanity check to ensure that the process did in fact
+			// start.
+			if bitcoind.Process == nil {
+				cleanUp1()
+				return nil, nil, fmt.Errorf("bitcoind cmd " +
+					"Process is not set after Start")
+			}
+
 			cleanUp2 := func() {
-				bitcoind.Process.Kill()
-				bitcoind.Wait()
+				_ = bitcoind.Process.Kill()
+				_ = bitcoind.Wait()
 				cleanUp1()
 			}
 
-			// Wait for the bitcoind instance to start up.
-			time.Sleep(time.Second)
-
 			host := fmt.Sprintf("127.0.0.1:%d", rpcPort)
-			chainConn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{
-				ChainParams:     &chaincfg.RegressionNetParams,
-				Host:            host,
-				User:            "weks",
-				Pass:            "weks",
-				ZMQBlockHost:    zmqBlockHost,
-				ZMQTxHost:       zmqTxHost,
-				ZMQReadDeadline: 5 * time.Second,
+			cfg := &chain.BitcoindConfig{
+				ChainParams: &chaincfg.RegressionNetParams,
+				Host:        host,
+				User:        "weks",
+				Pass:        "weks",
+				ZMQConfig: &chain.ZMQConfig{
+					ZMQBlockHost:    zmqBlockHost,
+					ZMQTxHost:       zmqTxHost,
+					ZMQReadDeadline: 5 * time.Second,
+				},
 				// Fields only required for pruned nodes, not
 				// needed for these tests.
 				Dialer:             nil,
 				PrunedModeMaxPeers: 0,
-			})
+			}
+
+			var chainConn *chain.BitcoindConn
+			err = wait.NoError(func() error {
+				chainConn, err = chain.NewBitcoindConn(cfg)
+				if err != nil {
+					return err
+				}
+
+				err = chainConn.Start()
+				if err != nil {
+					return err
+				}
+
+				client := chainConn.NewBitcoindClient()
+				_, currentHeight, err := client.GetBestBlock()
+				if err != nil {
+					return err
+				}
+
+				if currentHeight < bestHeight {
+					return fmt.Errorf("not synced yet")
+				}
+
+				return nil
+			}, 10*time.Second)
 			if err != nil {
 				return cleanUp2, nil, fmt.Errorf("unable to "+
 					"establish connection to bitcoind: %v",
 					err)
 			}
-			if err := chainConn.Start(); err != nil {
+			cleanUp3 := func() {
+				chainConn.Stop()
+				cleanUp2()
+			}
+
+			blockCache := blockcache.NewBlockCache(10000)
+
+			chainView := NewBitcoindFilteredChainView(
+				chainConn, blockCache,
+			)
+
+			return cleanUp3, chainView, nil
+		},
+	},
+	{
+		name: "bitcoind_polling",
+		chainViewInit: func(_ rpcclient.ConnConfig,
+			p2pAddr string, bestHeight int32) (func(),
+			FilteredChainView, error) {
+
+			// Start a bitcoind instance.
+			tempBitcoindDir, err := ioutil.TempDir("", "bitcoind")
+			if err != nil {
+				return nil, nil, err
+			}
+			cleanUp1 := func() {
+				os.RemoveAll(tempBitcoindDir)
+			}
+			rpcPort := getFreePort()
+			bitcoind := exec.Command(
+				"bitcoind",
+				"-datadir="+tempBitcoindDir,
+				"-regtest",
+				"-connect="+p2pAddr,
+				"-txindex",
+				"-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f"+
+					"d$507c670e800a95284294edb5773b05544b"+
+					"220110063096c221be9933c82d38e1",
+				fmt.Sprintf("-rpcport=%d", rpcPort),
+				"-disablewallet",
+			)
+			err = bitcoind.Start()
+			if err != nil {
+				cleanUp1()
+				return nil, nil, err
+			}
+
+			// Sanity check to ensure that the process did in fact
+			// start.
+			if bitcoind.Process == nil {
+				cleanUp1()
+				return nil, nil, fmt.Errorf("bitcoind cmd " +
+					"Process is not set after Start")
+			}
+
+			cleanUp2 := func() {
+				_ = bitcoind.Process.Kill()
+				_ = bitcoind.Wait()
+				cleanUp1()
+			}
+
+			host := fmt.Sprintf("127.0.0.1:%d", rpcPort)
+			cfg := &chain.BitcoindConfig{
+				ChainParams: &chaincfg.RegressionNetParams,
+				Host:        host,
+				User:        "weks",
+				Pass:        "weks",
+				PollingConfig: &chain.PollingConfig{
+					BlockPollingInterval: time.Millisecond * 100,
+					TxPollingInterval:    time.Millisecond * 100,
+				},
+				// Fields only required for pruned nodes, not
+				// needed for these tests.
+				Dialer:             nil,
+				PrunedModeMaxPeers: 0,
+			}
+
+			// Wait for the bitcoind instance to start up.
+			var chainConn *chain.BitcoindConn
+			err = wait.NoError(func() error {
+				chainConn, err = chain.NewBitcoindConn(cfg)
+				if err != nil {
+					return err
+				}
+
+				err = chainConn.Start()
+				if err != nil {
+					return err
+				}
+
+				client := chainConn.NewBitcoindClient()
+				_, currentHeight, err := client.GetBestBlock()
+				if err != nil {
+					return err
+				}
+
+				if currentHeight < bestHeight {
+					return fmt.Errorf("not synced yet")
+				}
+
+				return nil
+			}, 10*time.Second)
+			if err != nil {
 				return cleanUp2, nil, fmt.Errorf("unable to "+
 					"establish connection to bitcoind: %v",
 					err)
@@ -867,7 +1039,8 @@ var interfaceImpls = []struct {
 	{
 		name: "p2p_neutrino",
 		chainViewInit: func(_ rpcclient.ConnConfig,
-			p2pAddr string) (func(), FilteredChainView, error) {
+			p2pAddr string, bestHeight int32) (func(),
+			FilteredChainView, error) {
 
 			spvDir, err := ioutil.TempDir("", "neutrino")
 			if err != nil {
@@ -893,12 +1066,30 @@ var interfaceImpls = []struct {
 			if err != nil {
 				return nil, nil, err
 			}
-			spvNode.Start()
 
 			// Wait until the node has fully synced up to the local
 			// btcd node.
-			for !spvNode.IsCurrent() {
-				time.Sleep(time.Millisecond * 100)
+			err = wait.NoError(func() error {
+				err := spvNode.Start()
+				if err != nil {
+					return err
+				}
+
+				bestBlock, err := spvNode.BestBlock()
+				if err != nil {
+					return err
+				}
+
+				if bestBlock.Height < bestHeight {
+					return fmt.Errorf("not synced yet")
+				}
+
+				return nil
+			}, 10*time.Second)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unable to "+
+					"establish connection to bitcoind: %v",
+					err)
 			}
 
 			cleanUp := func() {
@@ -922,7 +1113,8 @@ var interfaceImpls = []struct {
 	{
 		name: "btcd_websockets",
 		chainViewInit: func(config rpcclient.ConnConfig,
-			_ string) (func(), FilteredChainView, error) {
+			p2pAddr string, bestHeight int32) (func(),
+			FilteredChainView, error) {
 
 			blockCache := blockcache.NewBlockCache(10000)
 			chainView, err := NewBtcdFilteredChainView(
@@ -958,13 +1150,17 @@ func TestFilteredChainView(t *testing.T) {
 		t.Logf("Testing '%v' implementation of FilteredChainView",
 			chainViewImpl.name)
 
+		_, bestHeight, err := miner.Client.GetBestBlock()
+		if err != nil {
+			t.Fatalf("error getting best block: %v", err)
+		}
+
 		cleanUpFunc, chainView, err := chainViewImpl.chainViewInit(
-			rpcConfig, p2pAddr,
+			rpcConfig, p2pAddr, bestHeight,
 		)
 		if err != nil {
 			t.Fatalf("unable to make chain view: %v", err)
 		}
-
 		if err := chainView.Start(); err != nil {
 			t.Fatalf("unable to start chain view: %v", err)
 		}

sample-lnd.conf

@@ -604,6 +604,15 @@ bitcoin.node=btcd
 ; likely won't need to be set (other than for a remote bitcoind instance).
 ; bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332
 ; bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333
+; bitcoind.zmqreaddeadline=10s
+
+; Use bitcoind's rpc interface to get block and transaction notifications
+; instead of using the zmq interface. Only the rpcpolling option needs to
+; be set in order to enable this, the rest of the options can be used to
+; change the default values used for this configuration.
+; bitcoind.rpcpolling
+; bitcoind.blockpollinginterval=1m
+; bitcoind.txpollinginterval=30s
 
 ; Fee estimate mode for bitcoind. It must be either "ECONOMICAL" or "CONSERVATIVE".
 ; If unset, the default value is "CONSERVATIVE".
@@ -811,6 +820,15 @@ litecoin.node=ltcd
 ; likely won't need to be set (other than for a remote litecoind instance).
 ; litecoind.zmqpubrawblock=tcp://127.0.0.1:28332
 ; litecoind.zmqpubrawtx=tcp://127.0.0.1:28333
+; litecoind.zmqreaddeadline=10s
+
+; Use litecoind's rpc interface to get block and transaction notifications
+; instead of using the zmq interface. Only the rpcpolling option needs to
+; be set in order to enable this, the rest of the options can be used to
+; change the default values used for this configuration.
+; litecoind.rpcpolling
+; litecoind.blockpollinginterval=1m
+; litecoind.txpollinginterval=30s
 
 ; Fee estimate mode for litecoind. It must be either "ECONOMICAL" or "CONSERVATIVE".
 ; If unset, the default value is "CONSERVATIVE".