rpcserver+lntemp: add more error logs to PendingChannels

This commit adds error logs to `PendingChannels` and changes how we
assert the num of pending force close channels, see the docs for more
info.

lntemp/harness_assertion.go

@@ -435,7 +435,24 @@ func (h *HarnessTest) AssertNumPendingForceClose(hn *node.HarnessNode,
 	oldForce := hn.State.CloseChannel.PendingForceClose
 
 	err := wait.NoError(func() error {
-		resp := hn.RPC.PendingChannels()
+		// TODO(yy): we should be able to use `hn.RPC.PendingChannels`
+		// here to avoid checking the RPC error. However, we may get a
+		// `unable to find arbitrator` error from the rpc point, due to
+		// a timing issue in rpcserver,
+		// 1. `r.server.chanStateDB.FetchClosedChannels` fetches
+		//    the pending force close channel.
+		// 2. `r.arbitratorPopulateForceCloseResp` relies on the
+		//    channel arbitrator to get the report, and,
+		// 3. the arbitrator may be deleted due to the force close
+		//    channel being resolved.
+		// Somewhere along the line is missing a lock to keep the data
+		// consistent.
+		req := &lnrpc.PendingChannelsRequest{}
+		resp, err := hn.RPC.LN.PendingChannels(h.runCtx, req)
+		if err != nil {
+			return fmt.Errorf("PendingChannels got: %w", err)
+		}
+
 		channels = resp.PendingForceClosingChannels
 		total := len(channels)
 

rpcserver.go

@@ -3479,6 +3479,9 @@ func (r *rpcServer) fetchPendingForceCloseChannels() (pendingForceClose,
 				&chanPoint, currentHeight, forceClose,
 			)
 			if err != nil {
+				rpcsLog.Errorf("unable to populate nursery "+
+					"force close resp:%s, %v",
+					chanPoint, err)
 				return nil, 0, err
 			}
 
@@ -3486,6 +3489,9 @@ func (r *rpcServer) fetchPendingForceCloseChannels() (pendingForceClose,
 				&chanPoint, currentHeight, forceClose,
 			)
 			if err != nil {
+				rpcsLog.Errorf("unable to populate arbitrator "+
+					"force close resp:%s, %v",
+					chanPoint, err)
 				return nil, 0, err
 			}