test: add ability to networkHarness to cause an arbitrary node to restart

This commit adds a new feature to the network harness: test writers are
now able to select arbitrary nodes, causing them to restart.

This functionality will be useful in the future in order to test
scenarios like persisting data across restarts, re-syncing after
re-connections, reacting to the counter party broadcasting revoked
states, etc.

networktest.go

@@ -91,6 +91,10 @@ type lightningNode struct {
 	cmd     *exec.Cmd
 	pidFile string
 
+	// processExit is a channel that's closed once it's detected that the
+	// process this instance of lightningNode is bound to has exited.
+	processExit chan struct{}
+
 	extraArgs []string
 
 	lnrpc.LightningClient
@@ -121,13 +125,13 @@ func newLightningNode(rpcConfig *btcrpcclient.ConnConfig, lndArgs []string) (*li
 
 	numActiveNodes++
 
-	return &lightningNode{
-		cfg:       cfg,
-		p2pAddr:   net.JoinHostPort("127.0.0.1", strconv.Itoa(cfg.PeerPort)),
-		rpcAddr:   net.JoinHostPort("127.0.0.1", strconv.Itoa(cfg.RPCPort)),
-		rpcCert:   rpcConfig.Certificates,
-		nodeId:    nodeNum,
-		extraArgs: lndArgs,
+	return &lightningNode{cfg: cfg,
+		p2pAddr:     net.JoinHostPort("127.0.0.1", strconv.Itoa(cfg.PeerPort)),
+		rpcAddr:     net.JoinHostPort("127.0.0.1", strconv.Itoa(cfg.RPCPort)),
+		rpcCert:     rpcConfig.Certificates,
+		nodeId:      nodeNum,
+		processExit: make(chan struct{}),
+		extraArgs:   lndArgs,
 	}, nil
 }
 
@@ -176,6 +180,9 @@ func (l *lightningNode) start(lndError chan error) error {
 		if err := l.cmd.Wait(); err != nil {
 			lndError <- errors.New(errb.String())
 		}
+
+		// Signal any onlookers that this process has exited.
+		close(l.processExit)
 	}()
 
 	pid, err := os.Create(filepath.Join(l.cfg.DataDir,
@@ -255,6 +262,22 @@ func (l *lightningNode) stop() error {
 	return l.cmd.Process.Signal(os.Interrupt)
 }
 
+// restart attempts to restart a lightning node by shutting it down cleanly,
+// then restarting the process. This function is fully blocking. Upon restart,
+// the RPC connection to the node will be re-attempted, continuing iff the
+// connection attempt is successful.
+func (l *lightningNode) restart(errChan chan error) error {
+	if err := l.stop(); err != nil {
+		return nil
+	}
+
+	<-l.processExit
+
+	l.processExit = make(chan struct{})
+
+	return l.start(errChan)
+}
+
 // shutdown stops the active lnd process and clean up any temporary directories
 // created along the way.
 func (l *lightningNode) shutdown() error {
@@ -513,6 +536,18 @@ func (n *networkHarness) ConnectNodes(ctx context.Context, a, b *lightningNode)
 	return nil
 }
 
+// RestartNode  attempts to restart a lightning node by shutting it down
+// cleanly, then restarting the process. This function is fully blocking. Upon
+// restart, the RPC connection to the node will be re-attempted, continuing iff
+// the connection attempt is successful.
+//
+// This method can be useful when testing edge cases such as a node broadcast
+// and invalidated prior state, or persistent state recovery, simulating node
+// crashes, etc.
+func (n *networkHarness) RestartNode(node *lightningNode) error {
+	return node.restart(n.lndErrorChan)
+}
+
 // TODO(roasbeef): add a WithChannel higher-order function?
 //  * python-like context manager w.r.t using a channel within a test
 //  * possibly  adds more funds to the target wallet if the funds are not