From 35f35441114f3c872161ce5bfbd1c40605630eea Mon Sep 17 00:00:00 2001 From: "Johan T. Halseth" Date: Thu, 11 Jul 2019 13:51:22 +0200 Subject: [PATCH 1/7] lnd_test: use RegTest instead of SimNet during integration tests --- lntest/btcd.go | 5 +++-- lntest/itest/lnd_test.go | 8 +++++--- lntest/neutrino.go | 10 ++++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lntest/btcd.go b/lntest/btcd.go index 7187befd3..603bb3af2 100644 --- a/lntest/btcd.go +++ b/lntest/btcd.go @@ -67,7 +67,9 @@ func (b BtcdBackendConfig) Name() string { // NewBackend starts a new rpctest.Harness and returns a BtcdBackendConfig for // that node. miner should be set to the P2P address of the miner to connect // to. -func NewBackend(miner string) (*BtcdBackendConfig, func(), error) { +func NewBackend(miner string, netParams *chaincfg.Params) ( + *BtcdBackendConfig, func(), error) { + args := []string{ "--rejectnonstd", "--txindex", @@ -76,7 +78,6 @@ func NewBackend(miner string) (*BtcdBackendConfig, func(), error) { "--logdir=" + logDir, "--connect=" + miner, } - netParams := &chaincfg.SimNetParams chainBackend, err := rpctest.New(netParams, nil, args) if err != nil { return nil, nil, fmt.Errorf("unable to create btcd node: %v", err) diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index 4ee67480a..90a8523b7 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -45,7 +45,7 @@ import ( ) var ( - harnessNetParams = &chaincfg.SimNetParams + harnessNetParams = &chaincfg.RegressionNetParams ) const ( @@ -14415,7 +14415,9 @@ func TestLightningNetworkDaemon(t *testing.T) { }() // Start a chain backend. - chainBackend, cleanUp, err := lntest.NewBackend(miner.P2PAddress()) + chainBackend, cleanUp, err := lntest.NewBackend( + miner.P2PAddress(), harnessNetParams, + ) if err != nil { ht.Fatalf("unable to start backend: %v", err) } @@ -14454,7 +14456,7 @@ func TestLightningNetworkDaemon(t *testing.T) { // Next mine enough blocks in order for segwit and the CSV package // soft-fork to activate on SimNet. - numBlocks := chaincfg.SimNetParams.MinerConfirmationWindow * 2 + numBlocks := harnessNetParams.MinerConfirmationWindow * 2 if _, err := miner.Node.Generate(numBlocks); err != nil { ht.Fatalf("unable to generate blocks: %v", err) } diff --git a/lntest/neutrino.go b/lntest/neutrino.go index d127c0b50..a8275fc62 100644 --- a/lntest/neutrino.go +++ b/lntest/neutrino.go @@ -2,7 +2,11 @@ package lntest -import "fmt" +import ( + "fmt" + + "github.com/btcsuite/btcd/chaincfg" +) // NeutrinoBackendConfig is an implementation of the BackendConfig interface // backed by a neutrino node. @@ -35,7 +39,9 @@ func (b NeutrinoBackendConfig) Name() string { } // NewBackend starts and returns a NeutrinoBackendConfig for the node. -func NewBackend(miner string) (*NeutrinoBackendConfig, func(), error) { +func NewBackend(miner string, _ *chaincfg.Params) ( + *NeutrinoBackendConfig, func(), error) { + bd := &NeutrinoBackendConfig{ minerAddr: miner, } From 45d41dce17a5f3c7c091dd254be4e46dc0314dce Mon Sep 17 00:00:00 2001 From: "Johan T. Halseth" Date: Thu, 11 Jul 2019 13:51:22 +0200 Subject: [PATCH 2/7] lntest: add BitcoindBackendCfg --- lntest/bitcoind.go | 184 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 lntest/bitcoind.go diff --git a/lntest/bitcoind.go b/lntest/bitcoind.go new file mode 100644 index 000000000..dd0cc5e96 --- /dev/null +++ b/lntest/bitcoind.go @@ -0,0 +1,184 @@ +// +build bitcoind + +package lntest + +import ( + "fmt" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/rpcclient" +) + +// logDir is the name of the temporary log directory. +const logDir = "./.backendlogs" + +// BitcoindBackendConfig is an implementation of the BackendConfig interface +// backed by a Bitcoind node. +type BitcoindBackendConfig struct { + rpcHost string + rpcUser string + rpcPass string + zmqBlockPath string + zmqTxPath string + p2pPort int + rpcClient *rpcclient.Client + + // minerAddr is the p2p address of the miner to connect to. + minerAddr string +} + +// A compile time assertion to ensure BitcoindBackendConfig meets the +// BackendConfig interface. +var _ BackendConfig = (*BitcoindBackendConfig)(nil) + +// GenArgs returns the arguments needed to be passed to LND at startup for +// using this node as a chain backend. +func (b BitcoindBackendConfig) GenArgs() []string { + var args []string + args = append(args, "--bitcoin.node=bitcoind") + args = append(args, fmt.Sprintf("--bitcoind.rpchost=%v", b.rpcHost)) + args = append(args, fmt.Sprintf("--bitcoind.rpcuser=%v", b.rpcUser)) + args = append(args, fmt.Sprintf("--bitcoind.rpcpass=%v", b.rpcPass)) + args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawblock=%v", + b.zmqBlockPath)) + args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawtx=%v", + b.zmqTxPath)) + + return args +} + +// ConnectMiner is called to establish a connection to the test miner. +func (b BitcoindBackendConfig) ConnectMiner() error { + return b.rpcClient.AddNode(b.minerAddr, rpcclient.ANAdd) +} + +// DisconnectMiner is called to disconnect the miner. +func (b BitcoindBackendConfig) DisconnectMiner() error { + return b.rpcClient.AddNode(b.minerAddr, rpcclient.ANRemove) +} + +// Name returns the name of the backend type. +func (b BitcoindBackendConfig) Name() string { + return "bitcoind" +} + +// NewBackend starts a bitcoind node and returns a BitoindBackendConfig for +// that node. +func NewBackend(miner string, netParams *chaincfg.Params) ( + *BitcoindBackendConfig, func(), error) { + + if netParams != &chaincfg.RegressionNetParams { + return nil, nil, fmt.Errorf("only regtest supported") + } + + if err := os.MkdirAll(logDir, 0700); err != nil { + return nil, nil, err + } + + logFile, err := filepath.Abs(logDir + "/bitcoind.log") + if err != nil { + return nil, nil, err + } + + tempBitcoindDir, err := ioutil.TempDir("", "bitcoind") + if err != nil { + return nil, nil, + fmt.Errorf("unable to create temp directory: %v", err) + } + + zmqBlockPath := "ipc:///" + tempBitcoindDir + "/blocks.socket" + zmqTxPath := "ipc:///" + tempBitcoindDir + "/txs.socket" + rpcPort := rand.Int()%(65536-1024) + 1024 + p2pPort := rand.Int()%(65536-1024) + 1024 + + bitcoind := exec.Command( + "bitcoind", + "-datadir="+tempBitcoindDir, + "-regtest", + "-txindex", + "-whitelist=127.0.0.1", // whitelist localhost to speed up relay + "-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f"+ + "d$507c670e800a95284294edb5773b05544b"+ + "220110063096c221be9933c82d38e1", + fmt.Sprintf("-rpcport=%d", rpcPort), + fmt.Sprintf("-port=%d", p2pPort), + "-disablewallet", + "-zmqpubrawblock="+zmqBlockPath, + "-zmqpubrawtx="+zmqTxPath, + "-debuglogfile="+logFile, + ) + + err = bitcoind.Start() + if err != nil { + os.RemoveAll(tempBitcoindDir) + return nil, nil, fmt.Errorf("couldn't start bitcoind: %v", err) + } + + cleanUp := func() { + bitcoind.Process.Kill() + bitcoind.Wait() + + // After shutting down the chain backend, we'll make a copy of + // the log file before deleting the temporary log dir. + err := CopyFile("./output_bitcoind_chainbackend.log", logFile) + if err != nil { + fmt.Printf("unable to copy file: %v\n", err) + } + if err = os.RemoveAll(logDir); err != nil { + fmt.Printf("Cannot remove dir %s: %v\n", logDir, err) + } + + os.RemoveAll(tempBitcoindDir) + } + + // Allow process to start. + time.Sleep(1 * time.Second) + + rpcHost := fmt.Sprintf("127.0.0.1:%d", rpcPort) + rpcUser := "weks" + rpcPass := "weks" + + rpcCfg := rpcclient.ConnConfig{ + Host: rpcHost, + User: rpcUser, + Pass: rpcPass, + DisableConnectOnNew: true, + DisableAutoReconnect: false, + DisableTLS: true, + HTTPPostMode: true, + } + + client, err := rpcclient.New(&rpcCfg, nil) + if err != nil { + cleanUp() + return nil, nil, fmt.Errorf("unable to create rpc client: %v", + err) + } + + // We start by adding the miner to the bitcoind addnode list. We do + // this instead of connecting using command line flags, because it will + // allow us to disconnect the miner using the AddNode command later. + if err := client.AddNode(miner, rpcclient.ANAdd); err != nil { + cleanUp() + return nil, nil, fmt.Errorf("unable to add node: %v", err) + } + + bd := BitcoindBackendConfig{ + rpcHost: rpcHost, + rpcUser: rpcUser, + rpcPass: rpcPass, + zmqBlockPath: zmqBlockPath, + zmqTxPath: zmqTxPath, + p2pPort: p2pPort, + rpcClient: client, + minerAddr: miner, + } + + return &bd, cleanUp, nil +} From a57b3de7f91b5c4816aa5ac3f21e830806dbdf18 Mon Sep 17 00:00:00 2001 From: "Johan T. Halseth" Date: Thu, 11 Jul 2019 13:51:22 +0200 Subject: [PATCH 3/7] lntest: compile time check btcd and neutrino BackendCfg interface --- lntest/btcd.go | 4 ++++ lntest/neutrino.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/lntest/btcd.go b/lntest/btcd.go index 603bb3af2..3c50e551e 100644 --- a/lntest/btcd.go +++ b/lntest/btcd.go @@ -35,6 +35,10 @@ type BtcdBackendConfig struct { minerAddr string } +// A compile time assertion to ensure BtcdBackendConfig meets the BackendConfig +// interface. +var _ BackendConfig = (*BtcdBackendConfig)(nil) + // GenArgs returns the arguments needed to be passed to LND at startup for // using this node as a chain backend. func (b BtcdBackendConfig) GenArgs() []string { diff --git a/lntest/neutrino.go b/lntest/neutrino.go index a8275fc62..1a5497b95 100644 --- a/lntest/neutrino.go +++ b/lntest/neutrino.go @@ -14,6 +14,10 @@ type NeutrinoBackendConfig struct { minerAddr string } +// A compile time assertion to ensure NeutrinoBackendConfig meets the +// BackendConfig interface. +var _ BackendConfig = (*NeutrinoBackendConfig)(nil) + // GenArgs returns the arguments needed to be passed to LND at startup for // using this node as a chain backend. func (b NeutrinoBackendConfig) GenArgs() []string { From 857a2c3e658d171229a6765a3327baf49efec558 Mon Sep 17 00:00:00 2001 From: "Johan T. Halseth" Date: Thu, 11 Jul 2019 13:51:22 +0200 Subject: [PATCH 4/7] make+travis: enable bitcoind itests on travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index b5aad649e..a26e4458b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ env: - RACE=true - ITEST=true - NEUTRINO_ITEST=true + - BITCOIND_ITEST=true - COVER=true sudo: required @@ -35,6 +36,9 @@ script: # Run neutrino integration tests. - 'if [ "$NEUTRINO_ITEST" = true ]; then make travis-itest backend=neutrino; fi' + # Run bitcoind integration tests. + - 'if [ "$BITCOIND_ITEST" = true ]; then make travis-itest backend=bitcoind; fi' + # Run unit tests and generate coverage report. - 'if [ "$COVER" = true ]; then make travis-cover; fi' From 168fc4e1abec7b91900c5a17830d819e60f1cdf9 Mon Sep 17 00:00:00 2001 From: "Johan T. Halseth" Date: Wed, 7 Aug 2019 13:12:32 +0200 Subject: [PATCH 5/7] lntest/itest: cleanup force closed channel in testSendUpdateDisableChannel Otherwise following tests would be flaky because of unexpected sweep transaction in the mempool. --- lntest/itest/lnd_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index 90a8523b7..096c6b7f3 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -12760,6 +12760,10 @@ func testSendUpdateDisableChannel(net *lntest.NetworkHarness, t *harnessTest) { }, ) mineBlocks(t, net, 1, 1) + + // And finally, clean up the force closed channel by mining the + // sweeping transaction. + cleanupForceClose(t, net, net.Alice, chanPointAliceCarol) } // testAbandonChannel abandones a channel and asserts that it is no From c3480c6066c463b795c77e324d131a6eda04b388 Mon Sep 17 00:00:00 2001 From: "Johan T. Halseth" Date: Wed, 28 Aug 2019 15:41:59 +0200 Subject: [PATCH 6/7] lntest/bitcoind: set debug loglevel Co-authored-by: Wilmer Paulino --- lntest/bitcoind.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lntest/bitcoind.go b/lntest/bitcoind.go index dd0cc5e96..0a73c86e8 100644 --- a/lntest/bitcoind.go +++ b/lntest/bitcoind.go @@ -100,6 +100,7 @@ func NewBackend(miner string, netParams *chaincfg.Params) ( bitcoind := exec.Command( "bitcoind", "-datadir="+tempBitcoindDir, + "-debug", "-regtest", "-txindex", "-whitelist=127.0.0.1", // whitelist localhost to speed up relay From d7f0372fa2cdd2d871025b3503621ea882691ed3 Mon Sep 17 00:00:00 2001 From: "Johan T. Halseth" Date: Mon, 2 Sep 2019 12:05:51 +0200 Subject: [PATCH 7/7] lntest/itest: use timeout constants in rejectHTLC test The local timeout could be too short for certain backends. --- lntest/itest/lnd_test.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index 096c6b7f3..5579ee8d5 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -8853,7 +8853,6 @@ func testRejectHTLC(net *lntest.NetworkHarness, t *harnessTest) { // const chanAmt = btcutil.Amount(1000000) ctxb := context.Background() - timeout := time.Duration(time.Second * 5) // Create Carol with reject htlc flag. carol, err := net.NewNode("Carol", []string{"--rejecthtlc"}) @@ -8885,7 +8884,7 @@ func testRejectHTLC(net *lntest.NetworkHarness, t *harnessTest) { } // Open a channel between Alice and Carol. - ctxt, _ := context.WithTimeout(ctxb, timeout) + ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) chanPointAlice := openChannelAndAssert( ctxt, t, net, net.Alice, carol, lntest.OpenChannelParams{ @@ -8894,7 +8893,7 @@ func testRejectHTLC(net *lntest.NetworkHarness, t *harnessTest) { ) // Open a channel between Carol and Bob. - ctxt, _ = context.WithTimeout(ctxb, timeout) + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) chanPointCarol := openChannelAndAssert( ctxt, t, net, carol, net.Bob, lntest.OpenChannelParams{ @@ -8934,7 +8933,7 @@ func testRejectHTLC(net *lntest.NetworkHarness, t *harnessTest) { } // Alice pays Carols invoice. - ctxt, _ = context.WithTimeout(ctxb, timeout) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, net.Alice, []string{resp.PaymentRequest}, true, ) @@ -8959,7 +8958,7 @@ func testRejectHTLC(net *lntest.NetworkHarness, t *harnessTest) { } // Carol pays Bobs invoice. - ctxt, _ = context.WithTimeout(ctxb, timeout) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, carol, []string{resp.PaymentRequest}, true, ) @@ -8987,7 +8986,7 @@ func testRejectHTLC(net *lntest.NetworkHarness, t *harnessTest) { // Alice attempts to pay Bobs invoice. This payment should be rejected since // we are using Carol as an intermediary hop, Carol is running lnd with // --rejecthtlc. - ctxt, _ = context.WithTimeout(ctxb, timeout) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, net.Alice, []string{resp.PaymentRequest}, true, ) @@ -9001,9 +9000,9 @@ func testRejectHTLC(net *lntest.NetworkHarness, t *harnessTest) { } // Close all channels. - ctxt, _ = context.WithTimeout(ctxb, timeout) + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) - ctxt, _ = context.WithTimeout(ctxb, timeout) + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) }