invoices: replay awareness

Previously the invoice registry wasn't aware of replayed htlcs. This was
dealt with by keeping the invoice accept/settle logic idempotent, so
that a replay wouldn't have an effect.

This mechanism has two limitations:

1. No accurate tracking of the total amount paid to an invoice. The total
amount couldn't just be increased with every htlc received, because it
could be a replay which would lead to counting the htlc amount multiple
times. Therefore the total amount was set to the amount of the first
htlc that was received, even though there may have been multiple htlcs
paying to the invoice.

2. Impossible to check htlc expiry consistently for hodl invoices. When
an htlc is new, its expiry needs to be checked against the invoice cltv
delta. But for a replay, that check must be skipped. The htlc was
accepted in time, the invoice was moved to the accepted state and a
replay some blocks later shouldn't lead to that htlc being cancelled.
Because the invoice registry couldn't recognize replays, it stopped
checking htlc expiry heights when the invoice reached the accepted
state. This prevents hold htlcs from being cancelled after a restart.
But unfortunately this also caused additional htlcs to be accepted on an
already accepted invoice without their expiry being checked.

In this commit, the invoice registry starts to persistently track htlcs
so that replays can be recognized. For replays, an htlc resolution
action is returned early. This fixes both limitations mentioned above.

invoices/invoiceregistry.go

@@ -27,6 +27,10 @@ var (
 
 	// errNoUpdate is returned when no invoice updated is required.
 	errNoUpdate = errors.New("no update needed")
+
+	// errReplayedHtlc is returned if the htlc is already recorded on the
+	// invoice.
+	errReplayedHtlc = errors.New("replayed htlc")
 )
 
 // HodlEvent describes how an htlc should be resolved. If HodlEvent.Preimage is
@@ -435,20 +439,34 @@ func (i *InvoiceRegistry) NotifyExitHopHtlc(rHash lntypes.Hash,
 			rHash[:], s, amtPaid, expiry, circuitKey)
 	}
 
-	const (
-		actionCancel = iota
-		actionSettle
-		actionHold
-	)
-
-	// If no action is set, cancel the htlc.
-	action := actionCancel
+	// Default is to not update subscribers after the invoice update.
+	updateSubscribers := false
 
 	updateInvoice := func(inv *channeldb.Invoice) (
 		*channeldb.InvoiceUpdateDesc, error) {
 
-		// If the invoice is already canceled, there is no
-		// further checking to do.
+		// Don't update the invoice when this is a replayed htlc.
+		htlc, ok := inv.Htlcs[circuitKey]
+		if ok {
+			switch htlc.State {
+			case channeldb.HtlcStateCancelled:
+				debugLog("replayed htlc to canceled invoice")
+
+			case channeldb.HtlcStateAccepted:
+				debugLog("replayed htlc to accepted invoice")
+
+			case channeldb.HtlcStateSettled:
+				debugLog("replayed htlc to settled invoice")
+
+			default:
+				return nil, errors.New("unexpected htlc state")
+			}
+
+			return nil, errNoUpdate
+		}
+
+		// If the invoice is already canceled, there is no further
+		// checking to do.
 		if inv.Terms.State == channeldb.ContractCanceled {
 			debugLog("invoice already canceled")
 			return nil, errNoUpdate
@@ -462,36 +480,6 @@ func (i *InvoiceRegistry) NotifyExitHopHtlc(rHash lntypes.Hash,
 			return nil, errNoUpdate
 		}
 
-		// Return early in case the invoice was already accepted or
-		// settled. We don't want to check the expiry again, because it
-		// may be that we are just restarting.
-		//
-		// NOTE: Though our recovery and forwarding logic is
-		// predominately batched, settling invoices happens iteratively.
-		// We may reject one of two payments for the same rhash at
-		// first, but then restart and reject both after seeing that the
-		// invoice has been settled. Without any record of which one
-		// settles first, it is ambiguous as to which one actually
-		// settled the invoice. Thus, by accepting all payments, we
-		// eliminate the race condition that can lead to this
-		// inconsistency.
-		//
-		// TODO(conner): track ownership of settlements to properly
-		// recover from failures? or add batch invoice settlement
-		switch inv.Terms.State {
-		case channeldb.ContractAccepted:
-			debugLog("accepting duplicate payment to accepted invoice")
-			action = actionHold
-			return nil, errNoUpdate
-
-		// If invoice is already settled, settle htlc. This means we
-		// accept more payments to the same invoice hash.
-		case channeldb.ContractSettled:
-			debugLog("accepting duplicate payment to settled invoice")
-			action = actionSettle
-			return nil, errNoUpdate
-		}
-
 		// The invoice is still open. Check the expiry.
 		if expiry < uint32(currentHeight+i.finalCltvRejectDelta) {
 			debugLog("expiry too soon")
@@ -503,8 +491,31 @@ func (i *InvoiceRegistry) NotifyExitHopHtlc(rHash lntypes.Hash,
 			return nil, errNoUpdate
 		}
 
+		// Record HTLC in the invoice database.
+		newHtlcs := map[channeldb.CircuitKey]*channeldb.HtlcAcceptDesc{
+			circuitKey: {
+				Amt:          amtPaid,
+				Expiry:       expiry,
+				AcceptHeight: currentHeight,
+			},
+		}
+
 		update := channeldb.InvoiceUpdateDesc{
-			AmtPaid: amtPaid,
+			Htlcs: newHtlcs,
+		}
+
+		// Don't update invoice state if we are accepting a duplicate
+		// payment. We do accept or settle the HTLC.
+		switch inv.Terms.State {
+		case channeldb.ContractAccepted:
+			debugLog("accepting duplicate payment to accepted invoice")
+			update.State = channeldb.ContractAccepted
+			return &update, nil
+
+		case channeldb.ContractSettled:
+			debugLog("accepting duplicate payment to settled invoice")
+			update.State = channeldb.ContractSettled
+			return &update, nil
 		}
 
 		// Check to see if we can settle or this is an hold invoice and
@@ -512,14 +523,15 @@ func (i *InvoiceRegistry) NotifyExitHopHtlc(rHash lntypes.Hash,
 		holdInvoice := inv.Terms.PaymentPreimage == channeldb.UnknownPreimage
 		if holdInvoice {
 			debugLog("accepted")
-			action = actionHold
 			update.State = channeldb.ContractAccepted
 		} else {
 			debugLog("settled")
-			action = actionSettle
 			update.Preimage = inv.Terms.PaymentPreimage
 			update.State = channeldb.ContractSettled
 		}
+
+		updateSubscribers = true
+
 		return &update, nil
 	}
 
@@ -527,36 +539,39 @@ func (i *InvoiceRegistry) NotifyExitHopHtlc(rHash lntypes.Hash,
 	// one exists). The callback will set the resolution action that is
 	// returned to the link or contract resolver.
 	invoice, err := i.cdb.UpdateInvoice(rHash, updateInvoice)
-	switch err {
-
-	// Invoice was updated, notify clients.
-	case nil:
-		i.notifyClients(rHash, invoice, invoice.Terms.State)
-
-	// No invoice update in the database was performed, no action required.
-	case errNoUpdate:
-
-	// Log and return other unexpected errors.
-	default:
+	if err != nil && err != errNoUpdate {
 		debugLog(err.Error())
 
 		return nil, err
 	}
 
-	switch action {
+	if updateSubscribers {
+		i.notifyClients(rHash, invoice, invoice.Terms.State)
+	}
 
-	case actionSettle:
+	// Inspect latest htlc state on the invoice.
+	invoiceHtlc, ok := invoice.Htlcs[circuitKey]
+
+	// If it isn't recorded, cancel htlc.
+	if !ok {
+		return &HodlEvent{
+			Hash: rHash,
+		}, nil
+	}
+
+	switch invoiceHtlc.State {
+	case channeldb.HtlcStateCancelled:
+		return &HodlEvent{
+			Hash: rHash,
+		}, nil
+
+	case channeldb.HtlcStateSettled:
 		return &HodlEvent{
 			Hash:     rHash,
 			Preimage: &invoice.Terms.PaymentPreimage,
 		}, nil
 
-	case actionCancel:
-		return &HodlEvent{
-			Hash: rHash,
-		}, nil
-
-	case actionHold:
+	case channeldb.HtlcStateAccepted:
 		i.hodlSubscribe(hodlChan, rHash)
 		return nil, nil
 
@@ -583,7 +598,6 @@ func (i *InvoiceRegistry) SettleHodlInvoice(preimage lntypes.Preimage) error {
 		}
 
 		return &channeldb.InvoiceUpdateDesc{
-			AmtPaid:  invoice.AmtPaid,
 			State:    channeldb.ContractSettled,
 			Preimage: preimage,
 		}, nil
@@ -626,10 +640,18 @@ func (i *InvoiceRegistry) CancelInvoice(payHash lntypes.Hash) error {
 			return nil, channeldb.ErrInvoiceAlreadyCanceled
 		}
 
+		// Mark individual held htlcs as cancelled.
+		canceledHtlcs := make(
+			map[channeldb.CircuitKey]*channeldb.HtlcAcceptDesc,
+		)
+		for key := range invoice.Htlcs {
+			canceledHtlcs[key] = nil
+		}
+
 		// Move invoice to the canceled state.
 		return &channeldb.InvoiceUpdateDesc{
-			AmtPaid: 0,
-			State:   channeldb.ContractCanceled,
+			Htlcs: canceledHtlcs,
+			State: channeldb.ContractCanceled,
 		}, nil
 	}
 

invoices/invoiceregistry_test.go

@@ -199,13 +199,14 @@ func TestSettleInvoice(t *testing.T) {
 		t.Fatal("expected cancel event")
 	}
 
-	// Check that settled amount remains unchanged.
+	// Check that settled amount is equal to the sum of values of the htlcs
+	// 0 and 1.
 	inv, err := registry.LookupInvoice(hash)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if inv.AmtPaid != amtPaid {
-		t.Fatal("expected amount to be unchanged")
+	if inv.AmtPaid != amtPaid+amtPaid+600 {
+		t.Fatal("amount incorrect")
 	}
 
 	// Try to cancel.
@@ -426,8 +427,7 @@ func TestHoldInvoice(t *testing.T) {
 	}
 
 	// Test a new htlc coming in that doesn't meet the final cltv delta
-	// requirement. It should be rejected, but because invoice registry
-	// doesn't track individual htlcs it is accepted.
+	// requirement. It should be rejected.
 	event, err = registry.NotifyExitHopHtlc(
 		hash, amtPaid, 1, testCurrentHeight,
 		getCircuitKey(1), hodlChan, nil,
@@ -435,8 +435,8 @@ func TestHoldInvoice(t *testing.T) {
 	if err != nil {
 		t.Fatalf("expected settle to succeed but got %v", err)
 	}
-	if event != nil {
-		t.Fatalf("expected htlc to be held")
+	if event == nil || event.Preimage != nil {
+		t.Fatalf("expected htlc to be cancelled")
 	}
 
 	// We expect the accepted state to be sent to the single invoice