htlcswitch+itest: catch link quit signal when processing hodlQueue

This commit makes sure when processing resolutions, e.g, settling
invoices, when the link is already broken, the process would exit with
an error. This fixes the issue we found in the itest, where an
unexpected empty remote pending commitment was created although the
remote peer is already offline.

contractcourt/channel_arbitrator.go

@@ -2155,6 +2155,20 @@ func (c *ChannelArbitrator) checkRemoteDanglingActions(
 			continue
 		}
 
+		preimageAvailable, err := c.isPreimageAvailable(htlc.RHash)
+		if err != nil {
+			log.Errorf("ChannelArbitrator(%v): failed to query "+
+				"preimage for dangling htlc=%x from remote "+
+				"commitments diff", c.cfg.ChanPoint,
+				htlc.RHash[:])
+
+			continue
+		}
+
+		if preimageAvailable {
+			continue
+		}
+
 		// Dust htlcs can be canceled back even before the commitment
 		// transaction confirms. Dust htlcs are not enforceable onchain.
 		// If another version of the commit tx would confirm we either

htlcswitch/link.go

@@ -1582,6 +1582,11 @@ loop:
 		select {
 		case item := <-l.hodlQueue.ChanOut():
 			htlcResolution = item.(invoices.HtlcResolution)
+
+		// No need to process it if the link is broken.
+		case <-l.cg.Done():
+			return ErrLinkShuttingDown
+
 		default:
 			break loop
 		}

itest/lnd_multi-hop_force_close_test.go

@@ -744,6 +744,7 @@ func runMultiHopReceiverPreimageClaim(ht *lntest.HarnessTest,
 
 	// Stop Bob so he won't be able to settle the incoming htlc.
 	restartBob := ht.SuspendNode(bob)
+	ht.AssertPeerNotConnected(carol, bob)
 
 	// Settle invoice. This will just mark the invoice as settled, as there
 	// is no link anymore to remove the htlc from the commitment tx. For
@@ -1690,6 +1691,7 @@ func runLocalClaimIncomingHTLC(ht *lntest.HarnessTest,
 
 	// Suspend Bob to force Carol to go to chain.
 	restartBob := ht.SuspendNode(bob)
+	ht.AssertPeerNotConnected(carol, bob)
 
 	// Settle invoice. This will just mark the invoice as settled, as there
 	// is no link anymore to remove the htlc from the commitment tx. For
@@ -1967,6 +1969,7 @@ func runLocalClaimIncomingHTLCLeased(ht *lntest.HarnessTest,
 
 	// Suspend Bob to force Carol to go to chain.
 	restartBob := ht.SuspendNode(bob)
+	ht.AssertPeerNotConnected(carol, bob)
 
 	// Settle invoice. This will just mark the invoice as settled, as there
 	// is no link anymore to remove the htlc from the commitment tx. For
@@ -2307,6 +2310,7 @@ func runLocalPreimageClaim(ht *lntest.HarnessTest,
 
 	// Suspend bob, so Carol is forced to go on chain.
 	restartBob := ht.SuspendNode(bob)
+	ht.AssertPeerNotConnected(carol, bob)
 
 	// Settle invoice. This will just mark the invoice as settled, as there
 	// is no link anymore to remove the htlc from the commitment tx. For
@@ -2547,6 +2551,7 @@ func runLocalPreimageClaimLeased(ht *lntest.HarnessTest,
 
 	// Suspend bob, so Carol is forced to go on chain.
 	restartBob := ht.SuspendNode(bob)
+	ht.AssertPeerNotConnected(carol, bob)
 
 	// Settle invoice. This will just mark the invoice as settled, as there
 	// is no link anymore to remove the htlc from the commitment tx. For
@@ -2964,6 +2969,7 @@ func runHtlcAggregation(ht *lntest.HarnessTest,
 	// close. However, Carol will cancel her invoices to prevent force
 	// closes, so we shut her down for now.
 	restartCarol := ht.SuspendNode(carol)
+	ht.AssertPeerNotConnected(bob, carol)
 
 	// We'll now mine enough blocks to trigger Bob's broadcast of his
 	// commitment transaction due to the fact that the Carol's HTLCs are