channeldb+routing: store all payment htlcs

This commit converts the database structure of a payment so that it can
not just store the last htlc attempt, but all attempts that have been
made. This is a preparation for mpp sending.

In addition to that, we now also persist the fail time of an htlc. In a
later commit, the full failure reason will be added as well.

A key change is made to the control tower interface. Previously the
control tower wasn't aware of individual htlc outcomes. The payment
remained in-flight with the latest attempt recorded, but an outcome was
only set when the payment finished. With this commit, the outcome of
every htlc is expected by the control tower and recorded in the
database.

Co-authored-by: Johan T. Halseth <johanth@gmail.com>

routing/control_tower.go

@@ -14,7 +14,6 @@ import (
 // restarts. Payments are transitioned through various payment states, and the
 // ControlTower interface provides access to driving the state transitions.
 type ControlTower interface {
-	// InitPayment atomically moves the payment into the InFlight state.
 	// This method checks that no suceeded payment exist for this payment
 	// hash.
 	InitPayment(lntypes.Hash, *channeldb.PaymentCreationInfo) error
@@ -22,17 +21,25 @@ type ControlTower interface {
 	// RegisterAttempt atomically records the provided HTLCAttemptInfo.
 	RegisterAttempt(lntypes.Hash, *channeldb.HTLCAttemptInfo) error
 
-	// Success transitions a payment into the Succeeded state. After
-	// invoking this method, InitPayment should always return an error to
-	// prevent us from making duplicate payments to the same payment hash.
-	// The provided preimage is atomically saved to the DB for record
-	// keeping.
-	Success(lntypes.Hash, lntypes.Preimage) error
+	// SettleAttempt marks the given attempt settled with the preimage. If
+	// this is a multi shard payment, this might implicitly mean the the
+	// full payment succeeded.
+	//
+	// After invoking this method, InitPayment should always return an
+	// error to prevent us from making duplicate payments to the same
+	// payment hash. The provided preimage is atomically saved to the DB
+	// for record keeping.
+	SettleAttempt(lntypes.Hash, uint64, *channeldb.HTLCSettleInfo) error
+
+	// FailAttempt marks the given payment attempt failed.
+	FailAttempt(lntypes.Hash, uint64, *channeldb.HTLCFailInfo) error
 
 	// Fail transitions a payment into the Failed state, and records the
-	// reason the payment failed. After invoking this method, InitPayment
-	// should return nil on its next call for this payment hash, allowing
-	// the switch to make a subsequent payment.
+	// ultimate reason the payment failed. Note that this should only be
+	// called when all active active attempts are already failed. After
+	// invoking this method, InitPayment should return nil on its next call
+	// for this payment hash, allowing the user to make a subsequent
+	// payment.
 	Fail(lntypes.Hash, channeldb.FailureReason) error
 
 	// FetchInFlightPayments returns all payments with status InFlight.
@@ -99,14 +106,13 @@ func (p *controlTower) RegisterAttempt(paymentHash lntypes.Hash,
 	return p.db.RegisterAttempt(paymentHash, attempt)
 }
 
-// Success transitions a payment into the Succeeded state. After invoking this
-// method, InitPayment should always return an error to prevent us from making
-// duplicate payments to the same payment hash. The provided preimage is
-// atomically saved to the DB for record keeping.
-func (p *controlTower) Success(paymentHash lntypes.Hash,
-	preimage lntypes.Preimage) error {
+// SettleAttempt marks the given attempt settled with the preimage. If
+// this is a multi shard payment, this might implicitly mean the the
+// full payment succeeded.
+func (p *controlTower) SettleAttempt(paymentHash lntypes.Hash,
+	attemptID uint64, settleInfo *channeldb.HTLCSettleInfo) error {
 
-	payment, err := p.db.Success(paymentHash, preimage)
+	payment, err := p.db.SettleAttempt(paymentHash, attemptID, settleInfo)
 	if err != nil {
 		return err
 	}
@@ -119,6 +125,13 @@ func (p *controlTower) Success(paymentHash lntypes.Hash,
 	return nil
 }
 
+// FailAttempt marks the given payment attempt failed.
+func (p *controlTower) FailAttempt(paymentHash lntypes.Hash,
+	attemptID uint64, failInfo *channeldb.HTLCFailInfo) error {
+
+	return p.db.FailAttempt(paymentHash, attemptID, failInfo)
+}
+
 // createSuccessResult creates a success result to send to subscribers.
 func createSuccessResult(htlcs []channeldb.HTLCAttempt) *PaymentResult {
 	// Extract any preimage from the list of HTLCs.

routing/control_tower_test.go

@@ -13,9 +13,8 @@ import (
 	"github.com/btcsuite/btcd/btcec"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/lightningnetwork/lnd/channeldb"
-	"github.com/lightningnetwork/lnd/routing/route"
-
 	"github.com/lightningnetwork/lnd/lntypes"
+	"github.com/lightningnetwork/lnd/routing/route"
 )
 
 var (
@@ -111,7 +110,13 @@ func TestControlTowerSubscribeSuccess(t *testing.T) {
 	}
 
 	// Mark the payment as successful.
-	if err := pControl.Success(info.PaymentHash, preimg); err != nil {
+	err = pControl.SettleAttempt(
+		info.PaymentHash, attempt.AttemptID,
+		&channeldb.HTLCSettleInfo{
+			Preimage: preimg,
+		},
+	)
+	if err != nil {
 		t.Fatal(err)
 	}
 
@@ -213,6 +218,15 @@ func testPaymentControlSubscribeFail(t *testing.T, registerAttempt bool) {
 		if err != nil {
 			t.Fatal(err)
 		}
+
+		// Fail the payment attempt.
+		err := pControl.FailAttempt(
+			info.PaymentHash, attempt.AttemptID,
+			&channeldb.HTLCFailInfo{},
+		)
+		if err != nil {
+			t.Fatalf("unable to fail htlc: %v", err)
+		}
 	}
 
 	// Mark the payment as failed.

routing/mock_test.go

@@ -231,7 +231,8 @@ func (m *mockControlTower) InitPayment(phash lntypes.Hash,
 	}
 
 	m.inflights[phash] = channeldb.InFlightPayment{
-		Info: c,
+		Info:     c,
+		Attempts: make([]channeldb.HTLCAttemptInfo, 0),
 	}
 
 	return nil
@@ -252,20 +253,20 @@ func (m *mockControlTower) RegisterAttempt(phash lntypes.Hash,
 		return fmt.Errorf("not in flight")
 	}
 
-	p.Attempt = a
+	p.Attempts = append(p.Attempts, *a)
 	m.inflights[phash] = p
 
 	return nil
 }
 
-func (m *mockControlTower) Success(phash lntypes.Hash,
-	preimg lntypes.Preimage) error {
+func (m *mockControlTower) SettleAttempt(phash lntypes.Hash,
+	pid uint64, settleInfo *channeldb.HTLCSettleInfo) error {
 
 	m.Lock()
 	defer m.Unlock()
 
 	if m.success != nil {
-		m.success <- successArgs{preimg}
+		m.success <- successArgs{settleInfo.Preimage}
 	}
 
 	delete(m.inflights, phash)
@@ -310,3 +311,9 @@ func (m *mockControlTower) SubscribePayment(paymentHash lntypes.Hash) (
 
 	return false, nil, errors.New("not implemented")
 }
+
+func (m *mockControlTower) FailAttempt(hash lntypes.Hash, pid uint64,
+	failInfo *channeldb.HTLCFailInfo) error {
+
+	return nil
+}

routing/payment_lifecycle.go

@@ -58,6 +58,13 @@ func (p *paymentLifecycle) resumePayment() ([32]byte, *route.Route, error) {
 			// the DB, we send it.
 			sendErr := p.sendPaymentAttempt(firstHop, htlcAdd)
 			if sendErr != nil {
+				// TODO(joostjager): Distinguish unexpected
+				// internal errors from real send errors.
+				err = p.failAttempt()
+				if err != nil {
+					return [32]byte{}, nil, err
+				}
+
 				// We must inspect the error to know whether it
 				// was critical or not, to decide whether we
 				// should continue trying.
@@ -110,6 +117,11 @@ func (p *paymentLifecycle) resumePayment() ([32]byte, *route.Route, error) {
 				"the Switch, retrying.", p.attempt.AttemptID,
 				p.payment.PaymentHash)
 
+			err = p.failAttempt()
+			if err != nil {
+				return [32]byte{}, nil, err
+			}
+
 			// Reset the attempt to indicate we want to make a new
 			// attempt.
 			p.attempt = nil
@@ -146,6 +158,11 @@ func (p *paymentLifecycle) resumePayment() ([32]byte, *route.Route, error) {
 			log.Errorf("Attempt to send payment %x failed: %v",
 				p.payment.PaymentHash, result.Error)
 
+			err = p.failAttempt()
+			if err != nil {
+				return [32]byte{}, nil, err
+			}
+
 			// We must inspect the error to know whether it was
 			// critical or not, to decide whether we should
 			// continue trying.
@@ -174,7 +191,13 @@ func (p *paymentLifecycle) resumePayment() ([32]byte, *route.Route, error) {
 
 		// In case of success we atomically store the db payment and
 		// move the payment to the success state.
-		err = p.router.cfg.Control.Success(p.payment.PaymentHash, result.Preimage)
+		err = p.router.cfg.Control.SettleAttempt(
+			p.payment.PaymentHash, p.attempt.AttemptID,
+			&channeldb.HTLCSettleInfo{
+				Preimage:   result.Preimage,
+				SettleTime: p.router.cfg.Clock.Now(),
+			},
+		)
 		if err != nil {
 			log.Errorf("Unable to succeed payment "+
 				"attempt: %v", err)
@@ -339,9 +362,10 @@ func (p *paymentLifecycle) createNewPaymentAttempt() (lnwire.ShortChannelID,
 	// We now have all the information needed to populate
 	// the current attempt information.
 	p.attempt = &channeldb.HTLCAttemptInfo{
-		AttemptID:  attemptID,
-		SessionKey: sessionKey,
-		Route:      *rt,
+		AttemptID:   attemptID,
+		AttemptTime: p.router.cfg.Clock.Now(),
+		SessionKey:  sessionKey,
+		Route:       *rt,
 	}
 
 	// Before sending this HTLC to the switch, we checkpoint the
@@ -421,3 +445,15 @@ func (p *paymentLifecycle) handleSendError(sendErr error) error {
 	// Terminal state, return the error we encountered.
 	return sendErr
 }
+
+// failAttempt calls control tower to fail the current payment attempt.
+func (p *paymentLifecycle) failAttempt() error {
+	failInfo := &channeldb.HTLCFailInfo{
+		FailTime: p.router.cfg.Clock.Now(),
+	}
+
+	return p.router.cfg.Control.FailAttempt(
+		p.payment.PaymentHash, p.attempt.AttemptID,
+		failInfo,
+	)
+}

routing/router.go

@@ -541,7 +541,14 @@ func (r *ChannelRouter) Start() error {
 				PaymentHash: payment.Info.PaymentHash,
 			}
 
-			_, _, err := r.sendPayment(payment.Attempt, lPayment, paySession)
+			// TODO(joostjager): For mpp, possibly relaunch multiple
+			// in-flight htlcs here.
+			var attempt *channeldb.HTLCAttemptInfo
+			if len(payment.Attempts) > 0 {
+				attempt = &payment.Attempts[0]
+			}
+
+			_, _, err := r.sendPayment(attempt, lPayment, paySession)
 			if err != nil {
 				log.Errorf("Resuming payment with hash %v "+
 					"failed: %v.", payment.Info.PaymentHash, err)