Merge pull request #3970 from halseth/amp-router-mvp-2020

MPP: Enable MultiPathPayments for payment lifecycle

channeldb/mp_payment.go

@@ -131,6 +131,52 @@ type MPPayment struct {
 	Status PaymentStatus
 }
 
+// TerminalInfo returns any HTLC settle info recorded. If no settle info is
+// recorded, any payment level failure will be returned. If neither a settle
+// nor a failure is recorded, both return values will be nil.
+func (m *MPPayment) TerminalInfo() (*HTLCSettleInfo, *FailureReason) {
+	for _, h := range m.HTLCs {
+		if h.Settle != nil {
+			return h.Settle, nil
+		}
+	}
+
+	return nil, m.FailureReason
+}
+
+// SentAmt returns the sum of sent amount and fees for HTLCs that are either
+// settled or still in flight.
+func (m *MPPayment) SentAmt() (lnwire.MilliSatoshi, lnwire.MilliSatoshi) {
+	var sent, fees lnwire.MilliSatoshi
+	for _, h := range m.HTLCs {
+		if h.Failure != nil {
+			continue
+		}
+
+		// The attempt was not failed, meaning the amount was
+		// potentially sent to the receiver.
+		sent += h.Route.ReceiverAmt()
+		fees += h.Route.TotalFees()
+	}
+
+	return sent, fees
+}
+
+// InFlightHTLCs returns the HTLCs that are still in-flight, meaning they have
+// not been settled or failed.
+func (m *MPPayment) InFlightHTLCs() []HTLCAttempt {
+	var inflights []HTLCAttempt
+	for _, h := range m.HTLCs {
+		if h.Settle != nil || h.Failure != nil {
+			continue
+		}
+
+		inflights = append(inflights, h)
+	}
+
+	return inflights
+}
+
 // serializeHTLCSettleInfo serializes the details of a settled htlc.
 func serializeHTLCSettleInfo(w io.Writer, s *HTLCSettleInfo) error {
 	if _, err := w.Write(s.Preimage[:]); err != nil {

channeldb/payment_control.go

@@ -18,22 +18,59 @@ var (
 	// already "in flight" on the network.
 	ErrPaymentInFlight = errors.New("payment is in transition")
 
-	// ErrPaymentNotInitiated is returned  if payment wasn't initiated in
+	// ErrPaymentNotInitiated is returned if the payment wasn't initiated.
-	// switch.
 	ErrPaymentNotInitiated = errors.New("payment isn't initiated")
 
 	// ErrPaymentAlreadySucceeded is returned in the event we attempt to
 	// change the status of a payment already succeeded.
 	ErrPaymentAlreadySucceeded = errors.New("payment is already succeeded")
 
-	// ErrPaymentAlreadyFailed is returned in the event we attempt to
+	// ErrPaymentAlreadyFailed is returned in the event we attempt to alter
-	// re-fail a failed payment.
+	// a failed payment.
 	ErrPaymentAlreadyFailed = errors.New("payment has already failed")
 
 	// ErrUnknownPaymentStatus is returned when we do not recognize the
 	// existing state of a payment.
 	ErrUnknownPaymentStatus = errors.New("unknown payment status")
 
+	// ErrPaymentTerminal is returned if we attempt to alter a payment that
+	// already has reached a terminal condition.
+	ErrPaymentTerminal = errors.New("payment has reached terminal condition")
+
+	// ErrAttemptAlreadySettled is returned if we try to alter an already
+	// settled HTLC attempt.
+	ErrAttemptAlreadySettled = errors.New("attempt already settled")
+
+	// ErrAttemptAlreadyFailed is returned if we try to alter an already
+	// failed HTLC attempt.
+	ErrAttemptAlreadyFailed = errors.New("attempt already failed")
+
+	// ErrValueMismatch is returned if we try to register a non-MPP attempt
+	// with an amount that doesn't match the payment amount.
+	ErrValueMismatch = errors.New("attempted value doesn't match payment" +
+		"amount")
+
+	// ErrValueExceedsAmt is returned if we try to register an attempt that
+	// would take the total sent amount above the payment amount.
+	ErrValueExceedsAmt = errors.New("attempted value exceeds payment" +
+		"amount")
+
+	// ErrNonMPPayment is returned if we try to register an MPP attempt for
+	// a payment that already has a non-MPP attempt regitered.
+	ErrNonMPPayment = errors.New("payment has non-MPP attempts")
+
+	// ErrMPPayment is returned if we try to register a non-MPP attempt for
+	// a payment that already has an MPP attempt regitered.
+	ErrMPPayment = errors.New("payment has MPP attempts")
+
+	// ErrMPPPaymentAddrMismatch is returned if we try to register an MPP
+	// shard where the payment address doesn't match existing shards.
+	ErrMPPPaymentAddrMismatch = errors.New("payment address mismatch")
+
+	// ErrMPPTotalAmountMismatch is returned if we try to register an MPP
+	// shard where the total amount doesn't match existing shards.
+	ErrMPPTotalAmountMismatch = errors.New("mp payment total amount mismatch")
+
 	// errNoAttemptInfo is returned when no attempt info is stored yet.
 	errNoAttemptInfo = errors.New("unable to find attempt info for " +
 		"inflight payment")
@@ -52,7 +89,7 @@ func NewPaymentControl(db *DB) *PaymentControl {
 }
 
 // InitPayment checks or records the given PaymentCreationInfo with the DB,
-// making sure it does not already exist as an in-flight payment. Then this
+// making sure it does not already exist as an in-flight payment. When this
 // method returns successfully, the payment is guranteeed to be in the InFlight
 // state.
 func (p *PaymentControl) InitPayment(paymentHash lntypes.Hash,
@@ -168,12 +205,69 @@ func (p *PaymentControl) RegisterAttempt(paymentHash lntypes.Hash,
 			return err
 		}
 
-		// We can only register attempts for payments that are
+		payment, err := fetchPayment(bucket)
-		// in-flight.
+		if err != nil {
-		if err := ensureInFlight(bucket); err != nil {
 			return err
 		}
 
+		// Ensure the payment is in-flight.
+		if err := ensureInFlight(payment); err != nil {
+			return err
+		}
+
+		// We cannot register a new attempt if the payment already has
+		// reached a terminal condition:
+		settle, fail := payment.TerminalInfo()
+		if settle != nil || fail != nil {
+			return ErrPaymentTerminal
+		}
+
+		// Make sure any existing shards match the new one with regards
+		// to MPP options.
+		mpp := attempt.Route.FinalHop().MPP
+		for _, h := range payment.InFlightHTLCs() {
+			hMpp := h.Route.FinalHop().MPP
+
+			switch {
+
+			// We tried to register a non-MPP attempt for a MPP
+			// payment.
+			case mpp == nil && hMpp != nil:
+				return ErrMPPayment
+
+			// We tried to register a MPP shard for a non-MPP
+			// payment.
+			case mpp != nil && hMpp == nil:
+				return ErrNonMPPayment
+
+			// Non-MPP payment, nothing more to validate.
+			case mpp == nil:
+				continue
+			}
+
+			// Check that MPP options match.
+			if mpp.PaymentAddr() != hMpp.PaymentAddr() {
+				return ErrMPPPaymentAddrMismatch
+			}
+
+			if mpp.TotalMsat() != hMpp.TotalMsat() {
+				return ErrMPPTotalAmountMismatch
+			}
+		}
+
+		// If this is a non-MPP attempt, it must match the total amount
+		// exactly.
+		amt := attempt.Route.ReceiverAmt()
+		if mpp == nil && amt != payment.Info.Value {
+			return ErrValueMismatch
+		}
+
+		// Ensure we aren't sending more than the total payment amount.
+		sentAmt, _ := payment.SentAmt()
+		if sentAmt+amt > payment.Info.Value {
+			return ErrValueExceedsAmt
+		}
+
 		htlcsBucket, err := bucket.CreateBucketIfNotExists(
 			paymentHtlcsBucket,
 		)
@@ -241,8 +335,15 @@ func (p *PaymentControl) updateHtlcKey(paymentHash lntypes.Hash,
 			return err
 		}
 
-		// We can only update keys of in-flight payments.
+		p, err := fetchPayment(bucket)
-		if err := ensureInFlight(bucket); err != nil {
+		if err != nil {
+			return err
+		}
+
+		// We can only update keys of in-flight payments. We allow
+		// updating keys even if the payment has reached a terminal
+		// condition, since the HTLC outcomes must still be updated.
+		if err := ensureInFlight(p); err != nil {
 			return err
 		}
 
@@ -257,6 +358,15 @@ func (p *PaymentControl) updateHtlcKey(paymentHash lntypes.Hash,
 				attemptID)
 		}
 
+		// Make sure the shard is not already failed or settled.
+		if htlcBucket.Get(htlcFailInfoKey) != nil {
+			return ErrAttemptAlreadyFailed
+		}
+
+		if htlcBucket.Get(htlcSettleInfoKey) != nil {
+			return ErrAttemptAlreadySettled
+		}
+
 		// Add or update the key for this htlc.
 		err = htlcBucket.Put(key, value)
 		if err != nil {
@@ -299,9 +409,17 @@ func (p *PaymentControl) Fail(paymentHash lntypes.Hash,
 			return err
 		}
 
-		// We can only mark in-flight payments as failed.
+		// We mark the payent as failed as long as it is known. This
-		if err := ensureInFlight(bucket); err != nil {
+		// lets the last attempt to fail with a terminal write its
-			updateErr = err
+		// failure to the PaymentControl without synchronizing with
+		// other attempts.
+		paymentStatus, err := fetchPaymentStatus(bucket)
+		if err != nil {
+			return err
+		}
+
+		if paymentStatus == StatusUnknown {
+			updateErr = ErrPaymentNotInitiated
 			return nil
 		}
 
@@ -318,14 +436,6 @@ func (p *PaymentControl) Fail(paymentHash lntypes.Hash,
 			return err
 		}
 
-		// Final sanity check to see if there are no in-flight htlcs.
-		for _, htlc := range payment.HTLCs {
-			if htlc.Settle == nil && htlc.Failure == nil {
-				return errors.New("payment failed with " +
-					"in-flight htlc(s)")
-			}
-		}
-
 		return nil
 	})
 	if err != nil {
@@ -428,45 +538,29 @@ func nextPaymentSequence(tx kvdb.RwTx) ([]byte, error) {
 // fetchPaymentStatus fetches the payment status of the payment. If the payment
 // isn't found, it will default to "StatusUnknown".
 func fetchPaymentStatus(bucket kvdb.ReadBucket) (PaymentStatus, error) {
-	htlcsBucket := bucket.NestedReadBucket(paymentHtlcsBucket)
+	// Creation info should be set for all payments, regardless of state.
-	if htlcsBucket != nil {
+	// If not, it is unknown.
-		htlcs, err := fetchHtlcAttempts(htlcsBucket)
+	if bucket.Get(paymentCreationInfoKey) == nil {
-		if err != nil {
+		return StatusUnknown, nil
-			return 0, err
-		}
-
-		// Go through all HTLCs, and return StatusSucceeded if any of
-		// them did succeed.
-		for _, h := range htlcs {
-			if h.Settle != nil {
-				return StatusSucceeded, nil
-			}
-		}
 	}
 
-	if bucket.Get(paymentFailInfoKey) != nil {
+	payment, err := fetchPayment(bucket)
-		return StatusFailed, nil
+	if err != nil {
+		return 0, err
 	}
 
-	if bucket.Get(paymentCreationInfoKey) != nil {
+	return payment.Status, nil
-		return StatusInFlight, nil
-	}
-
-	return StatusUnknown, nil
 }
 
 // ensureInFlight checks whether the payment found in the given bucket has
 // status InFlight, and returns an error otherwise. This should be used to
 // ensure we only mark in-flight payments as succeeded or failed.
-func ensureInFlight(bucket kvdb.ReadBucket) error {
+func ensureInFlight(payment *MPPayment) error {
-	paymentStatus, err := fetchPaymentStatus(bucket)
+	paymentStatus := payment.Status
-	if err != nil {
-		return err
-	}
 
 	switch {
 
-	// The payment was indeed InFlight, return.
+	// The payment was indeed InFlight.
 	case paymentStatus == StatusInFlight:
 		return nil
 
@@ -528,14 +622,7 @@ func (p *PaymentControl) FetchInFlightPayments() ([]*InFlightPayment, error) {
 			inFlight := &InFlightPayment{}
 
 			// Get the CreationInfo.
-			b := bucket.Get(paymentCreationInfoKey)
+			inFlight.Info, err = fetchCreationInfo(bucket)
-			if b == nil {
-				return fmt.Errorf("unable to find creation " +
-					"info for inflight payment")
-			}
-
-			r := bytes.NewReader(b)
-			inFlight.Info, err = deserializePaymentCreationInfo(r)
 			if err != nil {
 				return err
 			}

channeldb/payment_control_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/btcsuite/fastsha256"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/lightningnetwork/lnd/lntypes"
+	"github.com/lightningnetwork/lnd/record"
 )
 
 func initDB() (*DB, error) {
@@ -48,14 +49,14 @@ func genInfo() (*PaymentCreationInfo, *HTLCAttemptInfo,
 	rhash := fastsha256.Sum256(preimage[:])
 	return &PaymentCreationInfo{
 			PaymentHash:    rhash,
-			Value:          1,
+			Value:          testRoute.ReceiverAmt(),
 			CreationTime:   time.Unix(time.Now().Unix(), 0),
 			PaymentRequest: []byte("hola"),
 		},
 		&HTLCAttemptInfo{
-			AttemptID:  1,
+			AttemptID:  0,
 			SessionKey: priv,
-			Route:      testRoute,
+			Route:      *testRoute.Copy(),
 		}, preimage, nil
 }
 
@@ -85,8 +86,7 @@ func TestPaymentControlSwitchFail(t *testing.T) {
 
 	assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
 	assertPaymentInfo(
-		t, pControl, info.PaymentHash, info, 0, nil, lntypes.Preimage{},
+		t, pControl, info.PaymentHash, info, nil, nil,
-		nil,
 	)
 
 	// Fail the payment, which should moved it to Failed.
@@ -99,8 +99,7 @@ func TestPaymentControlSwitchFail(t *testing.T) {
 	// Verify the status is indeed Failed.
 	assertPaymentStatus(t, pControl, info.PaymentHash, StatusFailed)
 	assertPaymentInfo(
-		t, pControl, info.PaymentHash, info, 0, nil, lntypes.Preimage{},
+		t, pControl, info.PaymentHash, info, &failReason, nil,
-		&failReason,
 	)
 
 	// Sends the htlc again, which should succeed since the prior payment
@@ -112,44 +111,57 @@ func TestPaymentControlSwitchFail(t *testing.T) {
 
 	assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
 	assertPaymentInfo(
-		t, pControl, info.PaymentHash, info, 0, nil, lntypes.Preimage{},
+		t, pControl, info.PaymentHash, info, nil, nil,
-		nil,
 	)
 
 	// Record a new attempt. In this test scenario, the attempt fails.
 	// However, this is not communicated to control tower in the current
 	// implementation. It only registers the initiation of the attempt.
-	attempt.AttemptID = 2
 	err = pControl.RegisterAttempt(info.PaymentHash, attempt)
 	if err != nil {
 		t.Fatalf("unable to register attempt: %v", err)
 	}
 
+	htlcReason := HTLCFailUnreadable
 	err = pControl.FailAttempt(
-		info.PaymentHash, 2, &HTLCFailInfo{
+		info.PaymentHash, attempt.AttemptID,
-			Reason: HTLCFailUnreadable,
+		&HTLCFailInfo{
+			Reason: htlcReason,
 		},
 	)
 	if err != nil {
 		t.Fatal(err)
 	}
+	assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
+
+	htlc := &htlcStatus{
+		HTLCAttemptInfo: attempt,
+		failure:         &htlcReason,
+	}
+
+	assertPaymentInfo(t, pControl, info.PaymentHash, info, nil, htlc)
 
 	// Record another attempt.
-	attempt.AttemptID = 3
+	attempt.AttemptID = 1
 	err = pControl.RegisterAttempt(info.PaymentHash, attempt)
 	if err != nil {
 		t.Fatalf("unable to send htlc message: %v", err)
 	}
 	assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
+
+	htlc = &htlcStatus{
+		HTLCAttemptInfo: attempt,
+	}
+
 	assertPaymentInfo(
-		t, pControl, info.PaymentHash, info, 0, attempt, lntypes.Preimage{},
+		t, pControl, info.PaymentHash, info, nil, htlc,
-		nil,
 	)
 
-	// Settle the attempt and verify that status was changed to StatusSucceeded.
+	// Settle the attempt and verify that status was changed to
+	// StatusSucceeded.
 	var payment *MPPayment
 	payment, err = pControl.SettleAttempt(
-		info.PaymentHash, 3,
+		info.PaymentHash, attempt.AttemptID,
 		&HTLCSettleInfo{
 			Preimage: preimg,
 		},
@@ -171,7 +183,11 @@ func TestPaymentControlSwitchFail(t *testing.T) {
 	}
 
 	assertPaymentStatus(t, pControl, info.PaymentHash, StatusSucceeded)
-	assertPaymentInfo(t, pControl, info.PaymentHash, info, 1, attempt, preimg, nil)
+
+	htlc.settle = &preimg
+	assertPaymentInfo(
+		t, pControl, info.PaymentHash, info, nil, htlc,
+	)
 
 	// Attempt a final payment, which should now fail since the prior
 	// payment succeed.
@@ -207,8 +223,7 @@ func TestPaymentControlSwitchDoubleSend(t *testing.T) {
 
 	assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
 	assertPaymentInfo(
-		t, pControl, info.PaymentHash, info, 0, nil, lntypes.Preimage{},
+		t, pControl, info.PaymentHash, info, nil, nil,
-		nil,
 	)
 
 	// Try to initiate double sending of htlc message with the same
@@ -226,9 +241,12 @@ func TestPaymentControlSwitchDoubleSend(t *testing.T) {
 		t.Fatalf("unable to send htlc message: %v", err)
 	}
 	assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
+
+	htlc := &htlcStatus{
+		HTLCAttemptInfo: attempt,
+	}
 	assertPaymentInfo(
-		t, pControl, info.PaymentHash, info, 0, attempt, lntypes.Preimage{},
+		t, pControl, info.PaymentHash, info, nil, htlc,
-		nil,
 	)
 
 	// Sends base htlc message which initiate StatusInFlight.
@@ -240,7 +258,7 @@ func TestPaymentControlSwitchDoubleSend(t *testing.T) {
 
 	// After settling, the error should be ErrAlreadyPaid.
 	_, err = pControl.SettleAttempt(
-		info.PaymentHash, 1,
+		info.PaymentHash, attempt.AttemptID,
 		&HTLCSettleInfo{
 			Preimage: preimg,
 		},
@@ -249,7 +267,9 @@ func TestPaymentControlSwitchDoubleSend(t *testing.T) {
 		t.Fatalf("error shouldn't have been received, got: %v", err)
 	}
 	assertPaymentStatus(t, pControl, info.PaymentHash, StatusSucceeded)
-	assertPaymentInfo(t, pControl, info.PaymentHash, info, 0, attempt, preimg, nil)
+
+	htlc.settle = &preimg
+	assertPaymentInfo(t, pControl, info.PaymentHash, info, nil, htlc)
 
 	err = pControl.InitPayment(info.PaymentHash, info)
 	if err != ErrAlreadyPaid {
@@ -360,12 +380,17 @@ func TestPaymentControlDeleteNonInFligt(t *testing.T) {
 			t.Fatalf("unable to send htlc message: %v", err)
 		}
 
+		htlc := &htlcStatus{
+			HTLCAttemptInfo: attempt,
+		}
+
 		if p.failed {
 			// Fail the payment attempt.
+			htlcFailure := HTLCFailUnreadable
 			err := pControl.FailAttempt(
 				info.PaymentHash, attempt.AttemptID,
 				&HTLCFailInfo{
-					Reason: HTLCFailUnreadable,
+					Reason: htlcFailure,
 				},
 			)
 			if err != nil {
@@ -381,14 +406,16 @@ func TestPaymentControlDeleteNonInFligt(t *testing.T) {
 
 			// Verify the status is indeed Failed.
 			assertPaymentStatus(t, pControl, info.PaymentHash, StatusFailed)
+
+			htlc.failure = &htlcFailure
 			assertPaymentInfo(
-				t, pControl, info.PaymentHash, info, 0, attempt,
+				t, pControl, info.PaymentHash, info,
-				lntypes.Preimage{}, &failReason,
+				&failReason, htlc,
 			)
 		} else if p.success {
 			// Verifies that status was changed to StatusSucceeded.
 			_, err := pControl.SettleAttempt(
-				info.PaymentHash, 1,
+				info.PaymentHash, attempt.AttemptID,
 				&HTLCSettleInfo{
 					Preimage: preimg,
 				},
@@ -398,14 +425,15 @@ func TestPaymentControlDeleteNonInFligt(t *testing.T) {
 			}
 
 			assertPaymentStatus(t, pControl, info.PaymentHash, StatusSucceeded)
+
+			htlc.settle = &preimg
 			assertPaymentInfo(
-				t, pControl, info.PaymentHash, info, 0, attempt, preimg, nil,
+				t, pControl, info.PaymentHash, info, nil, htlc,
 			)
 		} else {
 			assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
 			assertPaymentInfo(
-				t, pControl, info.PaymentHash, info, 0, attempt,
+				t, pControl, info.PaymentHash, info, nil, htlc,
-				lntypes.Preimage{}, nil,
 			)
 		}
 	}
@@ -431,6 +459,366 @@ func TestPaymentControlDeleteNonInFligt(t *testing.T) {
 	}
 }
 
+// TestPaymentControlMultiShard checks the ability of payment control to
+// have multiple in-flight HTLCs for a single payment.
+func TestPaymentControlMultiShard(t *testing.T) {
+	t.Parallel()
+
+	// We will register three HTLC attempts, and always fail the second
+	// one. We'll generate all combinations of settling/failing the first
+	// and third HTLC, and assert that the payment status end up as we
+	// expect.
+	type testCase struct {
+		settleFirst bool
+		settleLast  bool
+	}
+
+	var tests []testCase
+	for _, f := range []bool{true, false} {
+		for _, l := range []bool{true, false} {
+			tests = append(tests, testCase{f, l})
+		}
+	}
+
+	runSubTest := func(t *testing.T, test testCase) {
+		db, err := initDB()
+		if err != nil {
+			t.Fatalf("unable to init db: %v", err)
+		}
+
+		pControl := NewPaymentControl(db)
+
+		info, attempt, preimg, err := genInfo()
+		if err != nil {
+			t.Fatalf("unable to generate htlc message: %v", err)
+		}
+
+		// Init the payment, moving it to the StatusInFlight state.
+		err = pControl.InitPayment(info.PaymentHash, info)
+		if err != nil {
+			t.Fatalf("unable to send htlc message: %v", err)
+		}
+
+		assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
+		assertPaymentInfo(
+			t, pControl, info.PaymentHash, info, nil, nil,
+		)
+
+		// Create three unique attempts we'll use for the test, and
+		// register them with the payment control. We set each
+		// attempts's value to one third of the payment amount, and
+		// populate the MPP options.
+		shardAmt := info.Value / 3
+		attempt.Route.FinalHop().AmtToForward = shardAmt
+		attempt.Route.FinalHop().MPP = record.NewMPP(
+			info.Value, [32]byte{1},
+		)
+
+		var attempts []*HTLCAttemptInfo
+		for i := uint64(0); i < 3; i++ {
+			a := *attempt
+			a.AttemptID = i
+			attempts = append(attempts, &a)
+
+			err = pControl.RegisterAttempt(info.PaymentHash, &a)
+			if err != nil {
+				t.Fatalf("unable to send htlc message: %v", err)
+			}
+			assertPaymentStatus(
+				t, pControl, info.PaymentHash, StatusInFlight,
+			)
+
+			htlc := &htlcStatus{
+				HTLCAttemptInfo: &a,
+			}
+			assertPaymentInfo(
+				t, pControl, info.PaymentHash, info, nil, htlc,
+			)
+		}
+
+		// For a fourth attempt, check that attempting to
+		// register it will fail since the total sent amount
+		// will be too large.
+		b := *attempt
+		b.AttemptID = 3
+		err = pControl.RegisterAttempt(info.PaymentHash, &b)
+		if err != ErrValueExceedsAmt {
+			t.Fatalf("expected ErrValueExceedsAmt, got: %v",
+				err)
+		}
+
+		// Fail the second attempt.
+		a := attempts[1]
+		htlcFail := HTLCFailUnreadable
+		err = pControl.FailAttempt(
+			info.PaymentHash, a.AttemptID,
+			&HTLCFailInfo{
+				Reason: htlcFail,
+			},
+		)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		htlc := &htlcStatus{
+			HTLCAttemptInfo: a,
+			failure:         &htlcFail,
+		}
+		assertPaymentInfo(
+			t, pControl, info.PaymentHash, info, nil, htlc,
+		)
+
+		// Payment should still be in-flight.
+		assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
+
+		// Depending on the test case, settle or fail the first attempt.
+		a = attempts[0]
+		htlc = &htlcStatus{
+			HTLCAttemptInfo: a,
+		}
+
+		var firstFailReason *FailureReason
+		if test.settleFirst {
+			_, err := pControl.SettleAttempt(
+				info.PaymentHash, a.AttemptID,
+				&HTLCSettleInfo{
+					Preimage: preimg,
+				},
+			)
+			if err != nil {
+				t.Fatalf("error shouldn't have been "+
+					"received, got: %v", err)
+			}
+
+			// Assert that the HTLC has had the preimage recorded.
+			htlc.settle = &preimg
+			assertPaymentInfo(
+				t, pControl, info.PaymentHash, info, nil, htlc,
+			)
+		} else {
+			err := pControl.FailAttempt(
+				info.PaymentHash, a.AttemptID,
+				&HTLCFailInfo{
+					Reason: htlcFail,
+				},
+			)
+			if err != nil {
+				t.Fatalf("error shouldn't have been "+
+					"received, got: %v", err)
+			}
+
+			// Assert the failure was recorded.
+			htlc.failure = &htlcFail
+			assertPaymentInfo(
+				t, pControl, info.PaymentHash, info, nil, htlc,
+			)
+
+			// We also record a payment level fail, to move it into
+			// a terminal state.
+			failReason := FailureReasonNoRoute
+			_, err = pControl.Fail(info.PaymentHash, failReason)
+			if err != nil {
+				t.Fatalf("unable to fail payment hash: %v", err)
+			}
+
+			// Record the reason we failed the payment, such that
+			// we can assert this later in the test.
+			firstFailReason = &failReason
+		}
+
+		// The payment should still be considered in-flight, since there
+		// is still an active HTLC.
+		assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
+
+		// Try to register yet another attempt. This should fail now
+		// that the payment has reached a terminal condition.
+		b = *attempt
+		b.AttemptID = 3
+		err = pControl.RegisterAttempt(info.PaymentHash, &b)
+		if err != ErrPaymentTerminal {
+			t.Fatalf("expected ErrPaymentTerminal, got: %v", err)
+		}
+
+		assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight)
+
+		// Settle or fail the remaining attempt based on the testcase.
+		a = attempts[2]
+		htlc = &htlcStatus{
+			HTLCAttemptInfo: a,
+		}
+		if test.settleLast {
+			// Settle the last outstanding attempt.
+			_, err = pControl.SettleAttempt(
+				info.PaymentHash, a.AttemptID,
+				&HTLCSettleInfo{
+					Preimage: preimg,
+				},
+			)
+			if err != nil {
+				t.Fatalf("error shouldn't have been "+
+					"received, got: %v", err)
+			}
+
+			htlc.settle = &preimg
+			assertPaymentInfo(
+				t, pControl, info.PaymentHash, info,
+				firstFailReason, htlc,
+			)
+		} else {
+			// Fail the attempt.
+			err := pControl.FailAttempt(
+				info.PaymentHash, a.AttemptID,
+				&HTLCFailInfo{
+					Reason: htlcFail,
+				},
+			)
+			if err != nil {
+				t.Fatalf("error shouldn't have been "+
+					"received, got: %v", err)
+			}
+
+			// Assert the failure was recorded.
+			htlc.failure = &htlcFail
+			assertPaymentInfo(
+				t, pControl, info.PaymentHash, info,
+				firstFailReason, htlc,
+			)
+
+			// Check that we can override any perevious terminal
+			// failure. This is to allow multiple concurrent shard
+			// write a terminal failure to the database without
+			// syncing.
+			failReason := FailureReasonPaymentDetails
+			_, err = pControl.Fail(info.PaymentHash, failReason)
+			if err != nil {
+				t.Fatalf("unable to fail payment hash: %v", err)
+			}
+		}
+
+		// If any of the two attempts settled, the payment should end
+		// up in the Succeeded state. If both failed the payment should
+		// also be Failed at this poinnt.
+		finalStatus := StatusFailed
+		expRegErr := ErrPaymentAlreadyFailed
+		if test.settleFirst || test.settleLast {
+			finalStatus = StatusSucceeded
+			expRegErr = ErrPaymentAlreadySucceeded
+		}
+
+		assertPaymentStatus(t, pControl, info.PaymentHash, finalStatus)
+
+		// Finally assert we cannot register more attempts.
+		err = pControl.RegisterAttempt(info.PaymentHash, &b)
+		if err != expRegErr {
+			t.Fatalf("expected error %v, got: %v", expRegErr, err)
+		}
+	}
+
+	for _, test := range tests {
+		test := test
+		subTest := fmt.Sprintf("first=%v, second=%v",
+			test.settleFirst, test.settleLast)
+
+		t.Run(subTest, func(t *testing.T) {
+			runSubTest(t, test)
+		})
+	}
+}
+
+func TestPaymentControlMPPRecordValidation(t *testing.T) {
+	t.Parallel()
+
+	db, err := initDB()
+	if err != nil {
+		t.Fatalf("unable to init db: %v", err)
+	}
+
+	pControl := NewPaymentControl(db)
+
+	info, attempt, _, err := genInfo()
+	if err != nil {
+		t.Fatalf("unable to generate htlc message: %v", err)
+	}
+
+	// Init the payment.
+	err = pControl.InitPayment(info.PaymentHash, info)
+	if err != nil {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+
+	// Create three unique attempts we'll use for the test, and
+	// register them with the payment control. We set each
+	// attempts's value to one third of the payment amount, and
+	// populate the MPP options.
+	shardAmt := info.Value / 3
+	attempt.Route.FinalHop().AmtToForward = shardAmt
+	attempt.Route.FinalHop().MPP = record.NewMPP(
+		info.Value, [32]byte{1},
+	)
+
+	err = pControl.RegisterAttempt(info.PaymentHash, attempt)
+	if err != nil {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+
+	// Now try to register a non-MPP attempt, which should fail.
+	b := *attempt
+	b.AttemptID = 1
+	b.Route.FinalHop().MPP = nil
+	err = pControl.RegisterAttempt(info.PaymentHash, &b)
+	if err != ErrMPPayment {
+		t.Fatalf("expected ErrMPPayment, got: %v", err)
+	}
+
+	// Try to register attempt one with a different payment address.
+	b.Route.FinalHop().MPP = record.NewMPP(
+		info.Value, [32]byte{2},
+	)
+	err = pControl.RegisterAttempt(info.PaymentHash, &b)
+	if err != ErrMPPPaymentAddrMismatch {
+		t.Fatalf("expected ErrMPPPaymentAddrMismatch, got: %v", err)
+	}
+
+	// Try registering one with a different total amount.
+	b.Route.FinalHop().MPP = record.NewMPP(
+		info.Value/2, [32]byte{1},
+	)
+	err = pControl.RegisterAttempt(info.PaymentHash, &b)
+	if err != ErrMPPTotalAmountMismatch {
+		t.Fatalf("expected ErrMPPTotalAmountMismatch, got: %v", err)
+	}
+
+	// Create and init a new payment. This time we'll check that we cannot
+	// register an MPP attempt if we already registered a non-MPP one.
+	info, attempt, _, err = genInfo()
+	if err != nil {
+		t.Fatalf("unable to generate htlc message: %v", err)
+	}
+
+	err = pControl.InitPayment(info.PaymentHash, info)
+	if err != nil {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+
+	attempt.Route.FinalHop().MPP = nil
+	err = pControl.RegisterAttempt(info.PaymentHash, attempt)
+	if err != nil {
+		t.Fatalf("unable to send htlc message: %v", err)
+	}
+
+	// Attempt to register an MPP attempt, which should fail.
+	b = *attempt
+	b.AttemptID = 1
+	b.Route.FinalHop().MPP = record.NewMPP(
+		info.Value, [32]byte{1},
+	)
+
+	err = pControl.RegisterAttempt(info.PaymentHash, &b)
+	if err != ErrNonMPPayment {
+		t.Fatalf("expected ErrNonMPPayment, got: %v", err)
+	}
+}
+
 // assertPaymentStatus retrieves the status of the payment referred to by hash
 // and compares it with the expected state.
 func assertPaymentStatus(t *testing.T, p *PaymentControl,
@@ -452,11 +840,16 @@ func assertPaymentStatus(t *testing.T, p *PaymentControl,
 	}
 }
 
+type htlcStatus struct {
+	*HTLCAttemptInfo
+	settle  *lntypes.Preimage
+	failure *HTLCFailReason
+}
+
 // assertPaymentInfo retrieves the payment referred to by hash and verifies the
 // expected values.
 func assertPaymentInfo(t *testing.T, p *PaymentControl, hash lntypes.Hash,
-	c *PaymentCreationInfo, aIdx int, a *HTLCAttemptInfo, s lntypes.Preimage,
+	c *PaymentCreationInfo, f *FailureReason, a *htlcStatus) {
-	f *FailureReason) {
 
 	t.Helper()
 
@@ -487,20 +880,35 @@ func assertPaymentInfo(t *testing.T, p *PaymentControl, hash lntypes.Hash,
 		return
 	}
 
-	htlc := payment.HTLCs[aIdx]
+	htlc := payment.HTLCs[a.AttemptID]
 	if err := assertRouteEqual(&htlc.Route, &a.Route); err != nil {
 		t.Fatal("routes do not match")
 	}
 
-	var zeroPreimage = lntypes.Preimage{}
+	if htlc.AttemptID != a.AttemptID {
-	if s != zeroPreimage {
+		t.Fatalf("unnexpected attempt ID %v, expected %v",
-		if htlc.Settle.Preimage != s {
+			htlc.AttemptID, a.AttemptID)
+	}
+
+	if a.failure != nil {
+		if htlc.Failure == nil {
+			t.Fatalf("expected HTLC to be failed")
+		}
+
+		if htlc.Failure.Reason != *a.failure {
+			t.Fatalf("expected HTLC failure %v, had %v",
+				*a.failure, htlc.Failure.Reason)
+		}
+	} else if htlc.Failure != nil {
+		t.Fatalf("expected no HTLC failure")
+	}
+
+	if a.settle != nil {
+		if htlc.Settle.Preimage != *a.settle {
 			t.Fatalf("Preimages don't match: %x vs %x",
-				htlc.Settle.Preimage, s)
+				htlc.Settle.Preimage, a.settle)
-		}
-	} else {
-		if htlc.Settle != nil {
-			t.Fatal("expected no settle info")
 		}
+	} else if htlc.Settle != nil {
+		t.Fatal("expected no settle info")
 	}
 }

channeldb/payments.go

@@ -123,7 +123,12 @@ const (
 	// LocalLiquidityInsufficient, RemoteCapacityInsufficient.
 )
 
-// String returns a human readable FailureReason
+// Error returns a human readable error string for the FailureReason.
+func (r FailureReason) Error() string {
+	return r.String()
+}
+
+// String returns a human readable FailureReason.
 func (r FailureReason) String() string {
 	switch r {
 	case FailureReasonTimeout:
@@ -247,6 +252,16 @@ func (db *DB) FetchPayments() ([]*MPPayment, error) {
 	return payments, nil
 }
 
+func fetchCreationInfo(bucket kvdb.ReadBucket) (*PaymentCreationInfo, error) {
+	b := bucket.Get(paymentCreationInfoKey)
+	if b == nil {
+		return nil, fmt.Errorf("creation info not found")
+	}
+
+	r := bytes.NewReader(b)
+	return deserializePaymentCreationInfo(r)
+}
+
 func fetchPayment(bucket kvdb.ReadBucket) (*MPPayment, error) {
 	seqBytes := bucket.Get(paymentSequenceKey)
 	if seqBytes == nil {
@@ -255,20 +270,8 @@ func fetchPayment(bucket kvdb.ReadBucket) (*MPPayment, error) {
 
 	sequenceNum := binary.BigEndian.Uint64(seqBytes)
 
-	// Get the payment status.
-	paymentStatus, err := fetchPaymentStatus(bucket)
-	if err != nil {
-		return nil, err
-	}
-
 	// Get the PaymentCreationInfo.
-	b := bucket.Get(paymentCreationInfoKey)
+	creationInfo, err := fetchCreationInfo(bucket)
-	if b == nil {
-		return nil, fmt.Errorf("creation info not found")
-	}
-
-	r := bytes.NewReader(b)
-	creationInfo, err := deserializePaymentCreationInfo(r)
 	if err != nil {
 		return nil, err
 
@@ -286,12 +289,50 @@ func fetchPayment(bucket kvdb.ReadBucket) (*MPPayment, error) {
 
 	// Get failure reason if available.
 	var failureReason *FailureReason
-	b = bucket.Get(paymentFailInfoKey)
+	b := bucket.Get(paymentFailInfoKey)
 	if b != nil {
 		reason := FailureReason(b[0])
 		failureReason = &reason
 	}
 
+	// Go through all HTLCs for this payment, noting whether we have any
+	// settled HTLC, and any still in-flight.
+	var inflight, settled bool
+	for _, h := range htlcs {
+		if h.Failure != nil {
+			continue
+		}
+
+		if h.Settle != nil {
+			settled = true
+			continue
+		}
+
+		// If any of the HTLCs are not failed nor settled, we
+		// still have inflight HTLCs.
+		inflight = true
+	}
+
+	// Use the DB state to determine the status of the payment.
+	var paymentStatus PaymentStatus
+
+	switch {
+
+	// If any of the the HTLCs did succeed and there are no HTLCs in
+	// flight, the payment succeeded.
+	case !inflight && settled:
+		paymentStatus = StatusSucceeded
+
+	// If we have no in-flight HTLCs, and the payment failure is set, the
+	// payment is considered failed.
+	case !inflight && failureReason != nil:
+		paymentStatus = StatusFailed
+
+	// Otherwise it is still in flight.
+	default:
+		paymentStatus = StatusInFlight
+	}
+
 	return &MPPayment{
 		sequenceNum:   sequenceNum,
 		Info:          creationInfo,
@@ -407,6 +448,8 @@ func (db *DB) DeletePayments() error {
 				return err
 			}
 
+			// If the status is InFlight, we cannot safely delete
+			// the payment information, so we return early.
 			if paymentStatus == StatusInFlight {
 				return nil
 			}

lntest/itest/lnd_mpp_test.go

@@ -0,0 +1,357 @@
+// +build rpctest
+
+package itest
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/lightningnetwork/lnd"
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
+	"github.com/lightningnetwork/lnd/lntest"
+	"github.com/lightningnetwork/lnd/routing/route"
+)
+
+// testSendToRouteMultiPath tests that we are able to successfully route a
+// payment using multiple shards across different paths, by using SendToRoute.
+func testSendToRouteMultiPath(net *lntest.NetworkHarness, t *harnessTest) {
+	ctxb := context.Background()
+
+	// To ensure the payment goes through seperate paths, we'll set a
+	// channel size that can only carry one shard at a time. We'll divide
+	// the payment into 3 shards.
+	const (
+		paymentAmt = btcutil.Amount(300000)
+		shardAmt   = paymentAmt / 3
+		chanAmt    = shardAmt * 3 / 2
+	)
+
+	// Set up a network with three different paths Alice <-> Bob.
+	//              _ Eve _
+	//             /       \
+	// Alice -- Carol ---- Bob
+	//      \              /
+	//       \__ Dave ____/
+	//
+	//
+	// Create the three nodes in addition to Alice and Bob.
+	alice := net.Alice
+	bob := net.Bob
+	carol, err := net.NewNode("carol", nil)
+	if err != nil {
+		t.Fatalf("unable to create carol: %v", err)
+	}
+	defer shutdownAndAssert(net, t, carol)
+
+	dave, err := net.NewNode("dave", nil)
+	if err != nil {
+		t.Fatalf("unable to create dave: %v", err)
+	}
+	defer shutdownAndAssert(net, t, dave)
+
+	eve, err := net.NewNode("eve", nil)
+	if err != nil {
+		t.Fatalf("unable to create eve: %v", err)
+	}
+	defer shutdownAndAssert(net, t, eve)
+
+	nodes := []*lntest.HarnessNode{alice, bob, carol, dave, eve}
+
+	// Connect nodes to ensure propagation of channels.
+	for i := 0; i < len(nodes); i++ {
+		for j := i + 1; j < len(nodes); j++ {
+			ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
+			if err := net.EnsureConnected(ctxt, nodes[i], nodes[j]); err != nil {
+				t.Fatalf("unable to connect nodes: %v", err)
+			}
+		}
+	}
+
+	// We'll send shards along three routes from Alice.
+	sendRoutes := [][]*lntest.HarnessNode{
+		{carol, bob},
+		{dave, bob},
+		{carol, eve, bob},
+	}
+
+	// Keep a list of all our active channels.
+	var networkChans []*lnrpc.ChannelPoint
+	var closeChannelFuncs []func()
+
+	// openChannel is a helper to open a channel from->to.
+	openChannel := func(from, to *lntest.HarnessNode, chanSize btcutil.Amount) {
+		ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
+		err := net.SendCoins(ctxt, btcutil.SatoshiPerBitcoin, from)
+		if err != nil {
+			t.Fatalf("unable to send coins : %v", err)
+		}
+
+		ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout)
+		chanPoint := openChannelAndAssert(
+			ctxt, t, net, from, to,
+			lntest.OpenChannelParams{
+				Amt: chanSize,
+			},
+		)
+
+		closeChannelFuncs = append(closeChannelFuncs, func() {
+			ctxt, _ := context.WithTimeout(ctxb, channelCloseTimeout)
+			closeChannelAndAssert(
+				ctxt, t, net, from, chanPoint, false,
+			)
+		})
+
+		networkChans = append(networkChans, chanPoint)
+	}
+
+	// Open channels between the nodes.
+	openChannel(carol, bob, chanAmt)
+	openChannel(dave, bob, chanAmt)
+	openChannel(alice, dave, chanAmt)
+	openChannel(eve, bob, chanAmt)
+	openChannel(carol, eve, chanAmt)
+
+	// Since the channel Alice-> Carol will have to carry two
+	// shards, we make it larger.
+	openChannel(alice, carol, chanAmt+shardAmt)
+
+	for _, f := range closeChannelFuncs {
+		defer f()
+	}
+
+	// Wait for all nodes to have seen all channels.
+	for _, chanPoint := range networkChans {
+		for _, node := range nodes {
+			txid, err := lnd.GetChanPointFundingTxid(chanPoint)
+			if err != nil {
+				t.Fatalf("unable to get txid: %v", err)
+			}
+			point := wire.OutPoint{
+				Hash:  *txid,
+				Index: chanPoint.OutputIndex,
+			}
+
+			ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
+			err = node.WaitForNetworkChannelOpen(ctxt, chanPoint)
+			if err != nil {
+				t.Fatalf("(%d): timeout waiting for "+
+					"channel(%s) open: %v",
+					node.NodeID, point, err)
+			}
+		}
+	}
+
+	// Make Bob create an invoice for Alice to pay.
+	payReqs, rHashes, invoices, err := createPayReqs(
+		net.Bob, paymentAmt, 1,
+	)
+	if err != nil {
+		t.Fatalf("unable to create pay reqs: %v", err)
+	}
+
+	rHash := rHashes[0]
+	payReq := payReqs[0]
+
+	ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
+	decodeResp, err := net.Bob.DecodePayReq(
+		ctxt, &lnrpc.PayReqString{PayReq: payReq},
+	)
+	if err != nil {
+		t.Fatalf("decode pay req: %v", err)
+	}
+
+	payAddr := decodeResp.PaymentAddr
+
+	// Helper function for Alice to build a route from pubkeys.
+	buildRoute := func(amt btcutil.Amount, hops []*lntest.HarnessNode) (
+		*lnrpc.Route, error) {
+
+		rpcHops := make([][]byte, 0, len(hops))
+		for _, hop := range hops {
+			k := hop.PubKeyStr
+			pubkey, err := route.NewVertexFromStr(k)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v",
+					k, err)
+			}
+			rpcHops = append(rpcHops, pubkey[:])
+		}
+
+		req := &routerrpc.BuildRouteRequest{
+			AmtMsat:        int64(amt * 1000),
+			FinalCltvDelta: lnd.DefaultBitcoinTimeLockDelta,
+			HopPubkeys:     rpcHops,
+		}
+
+		ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
+		routeResp, err := net.Alice.RouterClient.BuildRoute(ctxt, req)
+		if err != nil {
+			return nil, err
+		}
+
+		return routeResp.Route, nil
+	}
+
+	responses := make(chan *routerrpc.SendToRouteResponse, len(sendRoutes))
+	for _, hops := range sendRoutes {
+		// Build a route for the specified hops.
+		r, err := buildRoute(shardAmt, hops)
+		if err != nil {
+			t.Fatalf("unable to build route: %v", err)
+		}
+
+		// Set the MPP records to indicate this is a payment shard.
+		hop := r.Hops[len(r.Hops)-1]
+		hop.TlvPayload = true
+		hop.MppRecord = &lnrpc.MPPRecord{
+			PaymentAddr:  payAddr,
+			TotalAmtMsat: int64(paymentAmt * 1000),
+		}
+
+		// Send the shard.
+		sendReq := &routerrpc.SendToRouteRequest{
+			PaymentHash: rHash,
+			Route:       r,
+		}
+
+		// We'll send all shards in their own goroutine, since SendToRoute will
+		// block as long as the payment is in flight.
+		go func() {
+			ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
+			resp, err := net.Alice.RouterClient.SendToRoute(ctxt, sendReq)
+			if err != nil {
+				t.Fatalf("unable to send payment: %v", err)
+			}
+
+			responses <- resp
+		}()
+	}
+
+	// Wait for all responses to be back, and check that they all
+	// succeeded.
+	for range sendRoutes {
+		var resp *routerrpc.SendToRouteResponse
+		select {
+		case resp = <-responses:
+		case <-time.After(defaultTimeout):
+			t.Fatalf("response not received")
+		}
+
+		if resp.Failure != nil {
+			t.Fatalf("received payment failure : %v", resp.Failure)
+		}
+
+		// All shards should come back with the preimage.
+		if !bytes.Equal(resp.Preimage, invoices[0].RPreimage) {
+			t.Fatalf("preimage doesn't match")
+		}
+	}
+
+	// assertNumHtlcs is a helper that checks the node's latest payment,
+	// and asserts it was split into num shards.
+	assertNumHtlcs := func(node *lntest.HarnessNode, num int) {
+		req := &lnrpc.ListPaymentsRequest{
+			IncludeIncomplete: true,
+		}
+		ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
+		paymentsResp, err := node.ListPayments(ctxt, req)
+		if err != nil {
+			t.Fatalf("error when obtaining payments: %v",
+				err)
+		}
+
+		payments := paymentsResp.Payments
+		if len(payments) == 0 {
+			t.Fatalf("no payments found")
+		}
+
+		payment := payments[len(payments)-1]
+		htlcs := payment.Htlcs
+		if len(htlcs) == 0 {
+			t.Fatalf("no htlcs")
+		}
+
+		succeeded := 0
+		for _, htlc := range htlcs {
+			if htlc.Status == lnrpc.HTLCAttempt_SUCCEEDED {
+				succeeded++
+			}
+		}
+
+		if succeeded != num {
+			t.Fatalf("expected %v succussful HTLCs, got %v", num,
+				succeeded)
+		}
+	}
+
+	// assertSettledInvoice checks that the invoice for the given payment
+	// hash is settled, and has been paid using num HTLCs.
+	assertSettledInvoice := func(node *lntest.HarnessNode, rhash []byte,
+		num int) {
+
+		found := false
+		offset := uint64(0)
+		for !found {
+			ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
+			invoicesResp, err := node.ListInvoices(
+				ctxt, &lnrpc.ListInvoiceRequest{
+					IndexOffset: offset,
+				},
+			)
+			if err != nil {
+				t.Fatalf("error when obtaining payments: %v",
+					err)
+			}
+
+			if len(invoicesResp.Invoices) == 0 {
+				break
+			}
+
+			for _, inv := range invoicesResp.Invoices {
+				if !bytes.Equal(inv.RHash, rhash) {
+					continue
+				}
+
+				// Assert that the amount paid to the invoice is
+				// correct.
+				if inv.AmtPaidSat != int64(paymentAmt) {
+					t.Fatalf("incorrect payment amt for "+
+						"invoicewant: %d, got %d",
+						paymentAmt, inv.AmtPaidSat)
+				}
+
+				if inv.State != lnrpc.Invoice_SETTLED {
+					t.Fatalf("Invoice not settled: %v",
+						inv.State)
+				}
+
+				if len(inv.Htlcs) != num {
+					t.Fatalf("expected invoice to be "+
+						"settled with %v HTLCs, had %v",
+						num, len(inv.Htlcs))
+				}
+
+				found = true
+				break
+			}
+
+			offset = invoicesResp.LastIndexOffset
+		}
+
+		if !found {
+			t.Fatalf("invoice not found")
+		}
+	}
+
+	// Finally check that the payment shows up with three settled HTLCs in
+	// Alice's list of payments...
+	assertNumHtlcs(net.Alice, 3)
+
+	// ...and in Bob's list of paid invoices.
+	assertSettledInvoice(net.Bob, rHash, 3)
+}

lntest/itest/lnd_test.go

@@ -1915,8 +1915,9 @@ func testUpdateChannelPolicy(net *lntest.NetworkHarness, t *harnessTest) {
 
 	// Alice knows about the channel policy of Carol and should therefore
 	// not be able to find a path during routing.
+	expErr := channeldb.FailureReasonNoRoute.Error()
 	if err == nil ||
-		!strings.Contains(err.Error(), "unable to find a path") {
+		!strings.Contains(err.Error(), expErr) {
 		t.Fatalf("expected payment to fail, instead got %v", err)
 	}
 
@@ -3995,6 +3996,41 @@ func assertAmountSent(amt btcutil.Amount, sndr, rcvr *lntest.HarnessNode) func()
 	}
 }
 
+// assertLastHTLCError checks that the last sent HTLC of the last payment sent
+// by the given node failed with the expected failure code.
+func assertLastHTLCError(t *harnessTest, node *lntest.HarnessNode,
+	code lnrpc.Failure_FailureCode) {
+
+	req := &lnrpc.ListPaymentsRequest{
+		IncludeIncomplete: true,
+	}
+	ctxt, _ := context.WithTimeout(context.Background(), defaultTimeout)
+	paymentsResp, err := node.ListPayments(ctxt, req)
+	if err != nil {
+		t.Fatalf("error when obtaining payments: %v", err)
+	}
+
+	payments := paymentsResp.Payments
+	if len(payments) == 0 {
+		t.Fatalf("no payments found")
+	}
+
+	payment := payments[len(payments)-1]
+	htlcs := payment.Htlcs
+	if len(htlcs) == 0 {
+		t.Fatalf("no htlcs")
+	}
+
+	htlc := htlcs[len(htlcs)-1]
+	if htlc.Failure == nil {
+		t.Fatalf("expected failure")
+	}
+
+	if htlc.Failure.Code != code {
+		t.Fatalf("expected failure %v, got %v", code, htlc.Failure.Code)
+	}
+}
+
 // testSphinxReplayPersistence verifies that replayed onion packets are rejected
 // by a remote peer after a restart. We use a combination of unsafe
 // configuration arguments to force Carol to replay the same sphinx packet after
@@ -4134,11 +4170,10 @@ func testSphinxReplayPersistence(net *lntest.NetworkHarness, t *harnessTest) {
 
 	// Construct the response we expect after sending a duplicate packet
 	// that fails due to sphinx replay detection.
-	replayErr := "InvalidOnionKey"
+	if resp.PaymentError == "" {
-	if !strings.Contains(resp.PaymentError, replayErr) {
+		t.Fatalf("expected payment error")
-		t.Fatalf("received payment error: %v, expected %v",
-			resp.PaymentError, replayErr)
 	}
+	assertLastHTLCError(t, carol, lnrpc.Failure_INVALID_ONION_KEY)
 
 	// Since the payment failed, the balance should still be left
 	// unaltered.
@@ -9452,12 +9487,11 @@ out:
 		t.Fatalf("payment should have been rejected due to invalid " +
 			"payment hash")
 	}
-	expectedErrorCode := lnwire.CodeIncorrectOrUnknownPaymentDetails.String()
+
-	if !strings.Contains(resp.PaymentError, expectedErrorCode) {
+	assertLastHTLCError(
-		// TODO(roasbeef): make into proper gRPC error code
+		t, net.Alice,
-		t.Fatalf("payment should have failed due to unknown payment hash, "+
+		lnrpc.Failure_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS,
-			"instead failed due to: %v", resp.PaymentError)
+	)
-	}
 
 	// The balances of all parties should be the same as initially since
 	// the HTLC was canceled.
@@ -9484,18 +9518,11 @@ out:
 		t.Fatalf("payment should have been rejected due to wrong " +
 			"HTLC amount")
 	}
-	expectedErrorCode = lnwire.CodeIncorrectOrUnknownPaymentDetails.String()
-	if !strings.Contains(resp.PaymentError, expectedErrorCode) {
-		t.Fatalf("payment should have failed due to wrong amount, "+
-			"instead failed due to: %v", resp.PaymentError)
-	}
 
-	// We'll also ensure that the encoded error includes the invlaid HTLC
+	assertLastHTLCError(
-	// amount.
+		t, net.Alice,
-	if !strings.Contains(resp.PaymentError, htlcAmt.String()) {
+		lnrpc.Failure_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS,
-		t.Fatalf("error didn't include expected payment amt of %v: "+
+	)
-			"%v", htlcAmt, resp.PaymentError)
-	}
 
 	// The balances of all parties should be the same as initially since
 	// the HTLC was canceled.
@@ -9574,12 +9601,12 @@ out:
 	if resp.PaymentError == "" {
 		t.Fatalf("payment should fail due to insufficient "+
 			"capacity: %v", err)
-	} else if !strings.Contains(resp.PaymentError,
-		lnwire.CodeTemporaryChannelFailure.String()) {
-		t.Fatalf("payment should fail due to insufficient capacity, "+
-			"instead: %v", resp.PaymentError)
 	}
 
+	assertLastHTLCError(
+		t, net.Alice, lnrpc.Failure_TEMPORARY_CHANNEL_FAILURE,
+	)
+
 	// Generate new invoice to not pay same invoice twice.
 	ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
 	carolInvoice, err = carol.AddInvoice(ctxt, invoiceReq)
@@ -9616,11 +9643,8 @@ out:
 	if resp.PaymentError == "" {
 		t.Fatalf("payment should have failed")
 	}
-	expectedErrorCode = lnwire.CodeUnknownNextPeer.String()
+
-	if !strings.Contains(resp.PaymentError, expectedErrorCode) {
+	assertLastHTLCError(t, net.Alice, lnrpc.Failure_UNKNOWN_NEXT_PEER)
-		t.Fatalf("payment should fail due to unknown hop, instead: %v",
-			resp.PaymentError)
-	}
 
 	// Finally, immediately close the channel. This function will also
 	// block until the channel is closed and will additionally assert the
@@ -9787,9 +9811,8 @@ func testRejectHTLC(net *lntest.NetworkHarness, t *harnessTest) {
 			"should have been rejected, carol will not accept forwarded htlcs",
 		)
 	}
-	if !strings.Contains(err.Error(), lnwire.CodeChannelDisabled.String()) {
+
-		t.Fatalf("error returned should have been Channel Disabled")
+	assertLastHTLCError(t, net.Alice, lnrpc.Failure_CHANNEL_DISABLED)
-	}
 
 	// Close all channels.
 	ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout)
@@ -14885,6 +14908,10 @@ var testsCases = []*testCase{
 		name: "psbt channel funding",
 		test: testPsbtChanFunding,
 	},
+	{
+		name: "sendtoroute multi path payment",
+		test: testSendToRouteMultiPath,
+	},
 }
 
 // TestLightningNetworkDaemon performs a series of integration tests amongst a

routing/control_tower.go

@@ -34,6 +34,10 @@ type ControlTower interface {
 	// FailAttempt marks the given payment attempt failed.
 	FailAttempt(lntypes.Hash, uint64, *channeldb.HTLCFailInfo) error
 
+	// FetchPayment fetches the payment corresponding to the given payment
+	// hash.
+	FetchPayment(paymentHash lntypes.Hash) (*channeldb.MPPayment, error)
+
 	// Fail transitions a payment into the Failed state, and records the
 	// ultimate reason the payment failed. Note that this should only be
 	// called when all active active attempts are already failed. After
@@ -132,6 +136,13 @@ func (p *controlTower) FailAttempt(paymentHash lntypes.Hash,
 	return p.db.FailAttempt(paymentHash, attemptID, failInfo)
 }
 
+// FetchPayment fetches the payment corresponding to the given payment hash.
+func (p *controlTower) FetchPayment(paymentHash lntypes.Hash) (
+	*channeldb.MPPayment, error) {
+
+	return p.db.FetchPayment(paymentHash)
+}
+
 // createSuccessResult creates a success result to send to subscribers.
 func createSuccessResult(htlcs []channeldb.HTLCAttempt) *PaymentResult {
 	// Extract any preimage from the list of HTLCs.

routing/control_tower_test.go

@@ -324,7 +324,7 @@ func genInfo() (*channeldb.PaymentCreationInfo, *channeldb.HTLCAttemptInfo,
 	rhash := sha256.Sum256(preimage[:])
 	return &channeldb.PaymentCreationInfo{
 			PaymentHash:    rhash,
-			Value:          1,
+			Value:          testRoute.ReceiverAmt(),
 			CreationTime:   time.Unix(time.Now().Unix(), 0),
 			PaymentRequest: []byte("hola"),
 		},

routing/errors.go

@@ -15,10 +15,6 @@ const (
 	// this update can't bring us something new, or because a node
 	// announcement was given for node not found in any channel.
 	ErrIgnored
-
-	// ErrPaymentAttemptTimeout is an error that indicates that a payment
-	// attempt timed out before we were able to successfully route an HTLC.
-	ErrPaymentAttemptTimeout
 )
 
 // routerError is a structure that represent the error inside the routing package,

routing/mock_test.go

@@ -10,12 +10,13 @@ import (
 	"github.com/lightningnetwork/lnd/lntypes"
 	"github.com/lightningnetwork/lnd/lnwire"
 	"github.com/lightningnetwork/lnd/routing/route"
-	"github.com/lightningnetwork/lnd/zpay32"
 )
 
 type mockPaymentAttemptDispatcher struct {
 	onPayment func(firstHop lnwire.ShortChannelID) ([32]byte, error)
 	results   map[uint64]*htlcswitch.PaymentResult
+
+	sync.Mutex
 }
 
 var _ PaymentAttemptDispatcher = (*mockPaymentAttemptDispatcher)(nil)
@@ -28,10 +29,6 @@ func (m *mockPaymentAttemptDispatcher) SendHTLC(firstHop lnwire.ShortChannelID,
 		return nil
 	}
 
-	if m.results == nil {
-		m.results = make(map[uint64]*htlcswitch.PaymentResult)
-	}
-
 	var result *htlcswitch.PaymentResult
 	preimage, err := m.onPayment(firstHop)
 	if err != nil {
@@ -46,7 +43,13 @@ func (m *mockPaymentAttemptDispatcher) SendHTLC(firstHop lnwire.ShortChannelID,
 		result = &htlcswitch.PaymentResult{Preimage: preimage}
 	}
 
+	m.Lock()
+	if m.results == nil {
+		m.results = make(map[uint64]*htlcswitch.PaymentResult)
+	}
+
 	m.results[pid] = result
+	m.Unlock()
 
 	return nil
 }
@@ -56,7 +59,11 @@ func (m *mockPaymentAttemptDispatcher) GetPaymentResult(paymentID uint64,
 	<-chan *htlcswitch.PaymentResult, error) {
 
 	c := make(chan *htlcswitch.PaymentResult, 1)
+
+	m.Lock()
 	res, ok := m.results[paymentID]
+	m.Unlock()
+
 	if !ok {
 		return nil, htlcswitch.ErrPaymentIDNotFound
 	}
@@ -78,8 +85,8 @@ type mockPaymentSessionSource struct {
 
 var _ PaymentSessionSource = (*mockPaymentSessionSource)(nil)
 
-func (m *mockPaymentSessionSource) NewPaymentSession(routeHints [][]zpay32.HopHint,
+func (m *mockPaymentSessionSource) NewPaymentSession(
-	target route.Vertex) (PaymentSession, error) {
+	_ *LightningPayment) (PaymentSession, error) {
 
 	return &mockPaymentSession{m.routes}, nil
 }
@@ -102,6 +109,13 @@ func (m *mockMissionControl) ReportPaymentFail(paymentID uint64, rt *route.Route
 	failureSourceIdx *int, failure lnwire.FailureMessage) (
 	*channeldb.FailureReason, error) {
 
+	// Report a permanent failure if this is an error caused
+	// by incorrect details.
+	if failure.Code() == lnwire.CodeIncorrectOrUnknownPaymentDetails {
+		reason := channeldb.FailureReasonPaymentDetails
+		return &reason, nil
+	}
+
 	return nil, nil
 }
 
@@ -123,11 +137,11 @@ type mockPaymentSession struct {
 
 var _ PaymentSession = (*mockPaymentSession)(nil)
 
-func (m *mockPaymentSession) RequestRoute(payment *LightningPayment,
+func (m *mockPaymentSession) RequestRoute(_, _ lnwire.MilliSatoshi,
-	height uint32, finalCltvDelta uint16) (*route.Route, error) {
+	_, height uint32) (*route.Route, error) {
 
 	if len(m.routes) == 0 {
-		return nil, fmt.Errorf("no routes")
+		return nil, errNoPathFound
 	}
 
 	r := m.routes[0]
@@ -177,27 +191,38 @@ type initArgs struct {
 	c *channeldb.PaymentCreationInfo
 }
 
-type registerArgs struct {
+type registerAttemptArgs struct {
 	a *channeldb.HTLCAttemptInfo
 }
 
-type successArgs struct {
+type settleAttemptArgs struct {
 	preimg lntypes.Preimage
 }
 
-type failArgs struct {
+type failAttemptArgs struct {
+	reason *channeldb.HTLCFailInfo
+}
+
+type failPaymentArgs struct {
 	reason channeldb.FailureReason
 }
 
-type mockControlTower struct {
+type testPayment struct {
-	inflights  map[lntypes.Hash]channeldb.InFlightPayment
+	info     channeldb.PaymentCreationInfo
-	successful map[lntypes.Hash]struct{}
+	attempts []channeldb.HTLCAttempt
+}
 
-	init          chan initArgs
+type mockControlTower struct {
-	register      chan registerArgs
+	payments   map[lntypes.Hash]*testPayment
-	success       chan successArgs
+	successful map[lntypes.Hash]struct{}
-	fail          chan failArgs
+	failed     map[lntypes.Hash]channeldb.FailureReason
-	fetchInFlight chan struct{}
+
+	init            chan initArgs
+	registerAttempt chan registerAttemptArgs
+	settleAttempt   chan settleAttemptArgs
+	failAttempt     chan failAttemptArgs
+	failPayment     chan failPaymentArgs
+	fetchInFlight   chan struct{}
 
 	sync.Mutex
 }
@@ -206,8 +231,9 @@ var _ ControlTower = (*mockControlTower)(nil)
 
 func makeMockControlTower() *mockControlTower {
 	return &mockControlTower{
-		inflights:  make(map[lntypes.Hash]channeldb.InFlightPayment),
+		payments:   make(map[lntypes.Hash]*testPayment),
 		successful: make(map[lntypes.Hash]struct{}),
+		failed:     make(map[lntypes.Hash]channeldb.FailureReason),
 	}
 }
 
@@ -221,18 +247,22 @@ func (m *mockControlTower) InitPayment(phash lntypes.Hash,
 		m.init <- initArgs{c}
 	}
 
+	// Don't allow re-init a successful payment.
 	if _, ok := m.successful[phash]; ok {
-		return fmt.Errorf("already successful")
+		return channeldb.ErrAlreadyPaid
 	}
 
-	_, ok := m.inflights[phash]
+	_, failed := m.failed[phash]
-	if ok {
+	_, ok := m.payments[phash]
-		return fmt.Errorf("in flight")
+
+	// If the payment is known, only allow re-init if failed.
+	if ok && !failed {
+		return channeldb.ErrPaymentInFlight
 	}
 
-	m.inflights[phash] = channeldb.InFlightPayment{
+	delete(m.failed, phash)
-		Info:     c,
+	m.payments[phash] = &testPayment{
-		Attempts: make([]channeldb.HTLCAttemptInfo, 0),
+		info: *c,
 	}
 
 	return nil
@@ -244,17 +274,28 @@ func (m *mockControlTower) RegisterAttempt(phash lntypes.Hash,
 	m.Lock()
 	defer m.Unlock()
 
-	if m.register != nil {
+	if m.registerAttempt != nil {
-		m.register <- registerArgs{a}
+		m.registerAttempt <- registerAttemptArgs{a}
 	}
 
-	p, ok := m.inflights[phash]
+	// Cannot register attempts for successful or failed payments.
+	if _, ok := m.successful[phash]; ok {
+		return channeldb.ErrPaymentAlreadySucceeded
+	}
+
+	if _, ok := m.failed[phash]; ok {
+		return channeldb.ErrPaymentAlreadyFailed
+	}
+
+	p, ok := m.payments[phash]
 	if !ok {
-		return fmt.Errorf("not in flight")
+		return channeldb.ErrPaymentNotInitiated
 	}
 
-	p.Attempts = append(p.Attempts, *a)
+	p.attempts = append(p.attempts, channeldb.HTLCAttempt{
-	m.inflights[phash] = p
+		HTLCAttemptInfo: *a,
+	})
+	m.payments[phash] = p
 
 	return nil
 }
@@ -265,13 +306,73 @@ func (m *mockControlTower) SettleAttempt(phash lntypes.Hash,
 	m.Lock()
 	defer m.Unlock()
 
-	if m.success != nil {
+	if m.settleAttempt != nil {
-		m.success <- successArgs{settleInfo.Preimage}
+		m.settleAttempt <- settleAttemptArgs{settleInfo.Preimage}
 	}
 
-	delete(m.inflights, phash)
+	// Only allow setting attempts if the payment is known.
-	m.successful[phash] = struct{}{}
+	p, ok := m.payments[phash]
-	return nil
+	if !ok {
+		return channeldb.ErrPaymentNotInitiated
+	}
+
+	// Find the attempt with this pid, and set the settle info.
+	for i, a := range p.attempts {
+		if a.AttemptID != pid {
+			continue
+		}
+
+		if a.Settle != nil {
+			return channeldb.ErrAttemptAlreadySettled
+		}
+		if a.Failure != nil {
+			return channeldb.ErrAttemptAlreadyFailed
+		}
+
+		p.attempts[i].Settle = settleInfo
+
+		// Mark the payment successful on first settled attempt.
+		m.successful[phash] = struct{}{}
+		return nil
+	}
+
+	return fmt.Errorf("pid not found")
+}
+
+func (m *mockControlTower) FailAttempt(phash lntypes.Hash, pid uint64,
+	failInfo *channeldb.HTLCFailInfo) error {
+
+	m.Lock()
+	defer m.Unlock()
+
+	if m.failAttempt != nil {
+		m.failAttempt <- failAttemptArgs{failInfo}
+	}
+
+	// Only allow failing attempts if the payment is known.
+	p, ok := m.payments[phash]
+	if !ok {
+		return channeldb.ErrPaymentNotInitiated
+	}
+
+	// Find the attempt with this pid, and set the failure info.
+	for i, a := range p.attempts {
+		if a.AttemptID != pid {
+			continue
+		}
+
+		if a.Settle != nil {
+			return channeldb.ErrAttemptAlreadySettled
+		}
+		if a.Failure != nil {
+			return channeldb.ErrAttemptAlreadyFailed
+		}
+
+		p.attempts[i].Failure = failInfo
+		return nil
+	}
+
+	return fmt.Errorf("pid not found")
 }
 
 func (m *mockControlTower) Fail(phash lntypes.Hash,
@@ -280,14 +381,46 @@ func (m *mockControlTower) Fail(phash lntypes.Hash,
 	m.Lock()
 	defer m.Unlock()
 
-	if m.fail != nil {
+	if m.failPayment != nil {
-		m.fail <- failArgs{reason}
+		m.failPayment <- failPaymentArgs{reason}
 	}
 
-	delete(m.inflights, phash)
+	// Payment must be known.
+	if _, ok := m.payments[phash]; !ok {
+		return channeldb.ErrPaymentNotInitiated
+	}
+
+	m.failed[phash] = reason
+
 	return nil
 }
 
+func (m *mockControlTower) FetchPayment(phash lntypes.Hash) (
+	*channeldb.MPPayment, error) {
+
+	m.Lock()
+	defer m.Unlock()
+
+	p, ok := m.payments[phash]
+	if !ok {
+		return nil, channeldb.ErrPaymentNotInitiated
+	}
+
+	mp := &channeldb.MPPayment{
+		Info: &p.info,
+	}
+
+	reason, ok := m.failed[phash]
+	if ok {
+		mp.FailureReason = &reason
+	}
+
+	// Return a copy of the current attempts.
+	mp.HTLCs = append(mp.HTLCs, p.attempts...)
+
+	return mp, nil
+}
+
 func (m *mockControlTower) FetchInFlightPayments() (
 	[]*channeldb.InFlightPayment, error) {
 
@@ -298,8 +431,25 @@ func (m *mockControlTower) FetchInFlightPayments() (
 		m.fetchInFlight <- struct{}{}
 	}
 
+	// In flight are all payments not successful or failed.
 	var fl []*channeldb.InFlightPayment
-	for _, ifl := range m.inflights {
+	for hash, p := range m.payments {
+		if _, ok := m.successful[hash]; ok {
+			continue
+		}
+		if _, ok := m.failed[hash]; ok {
+			continue
+		}
+
+		var attempts []channeldb.HTLCAttemptInfo
+		for _, a := range p.attempts {
+			attempts = append(attempts, a.HTLCAttemptInfo)
+		}
+		ifl := channeldb.InFlightPayment{
+			Info:     &p.info,
+			Attempts: attempts,
+		}
+
 		fl = append(fl, &ifl)
 	}
 
@@ -311,9 +461,3 @@ func (m *mockControlTower) SubscribePayment(paymentHash lntypes.Hash) (
 
 	return false, nil, errors.New("not implemented")
 }
-
-func (m *mockControlTower) FailAttempt(hash lntypes.Hash, pid uint64,
-	failInfo *channeldb.HTLCFailInfo) error {
-
-	return nil
-}

routing/pathfind.go

@@ -58,24 +58,6 @@ var (
 	// DefaultAprioriHopProbability is the default a priori probability for
 	// a hop.
 	DefaultAprioriHopProbability = float64(0.6)
-
-	// errNoTlvPayload is returned when the destination hop does not support
-	// a tlv payload.
-	errNoTlvPayload = errors.New("destination hop doesn't " +
-		"understand new TLV payloads")
-
-	// errNoPaymentAddr is returned when the destination hop does not
-	// support payment addresses.
-	errNoPaymentAddr = errors.New("destination hop doesn't " +
-		"understand payment addresses")
-
-	// errNoPathFound is returned when a path to the target destination does
-	// not exist in the graph.
-	errNoPathFound = errors.New("unable to find a path to destination")
-
-	// errInsufficientLocalBalance is returned when none of the local
-	// channels have enough balance for the payment.
-	errInsufficientBalance = errors.New("insufficient local balance")
 )
 
 // edgePolicyWithSource is a helper struct to keep track of the source node

routing/payment_lifecycle.go

@@ -2,346 +2,607 @@ package routing
 
 import (
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
 	sphinx "github.com/lightningnetwork/lightning-onion"
 	"github.com/lightningnetwork/lnd/channeldb"
 	"github.com/lightningnetwork/lnd/htlcswitch"
+	"github.com/lightningnetwork/lnd/lntypes"
 	"github.com/lightningnetwork/lnd/lnwire"
 	"github.com/lightningnetwork/lnd/routing/route"
 )
 
-// errNoRoute is returned when all routes from the payment session have been
-// attempted.
-type errNoRoute struct {
-	// lastError is the error encountered during the last payment attempt,
-	// if at least one attempt has been made.
-	lastError error
-}
-
-// Error returns a string representation of the error.
-func (e errNoRoute) Error() string {
-	return fmt.Sprintf("unable to route payment to destination: %v",
-		e.lastError)
-}
-
 // paymentLifecycle holds all information about the current state of a payment
 // needed to resume if from any point.
 type paymentLifecycle struct {
-	router         *ChannelRouter
+	router        *ChannelRouter
-	payment        *LightningPayment
+	totalAmount   lnwire.MilliSatoshi
-	paySession     PaymentSession
+	feeLimit      lnwire.MilliSatoshi
-	timeoutChan    <-chan time.Time
+	paymentHash   lntypes.Hash
-	currentHeight  int32
+	paySession    PaymentSession
-	finalCLTVDelta uint16
+	timeoutChan   <-chan time.Time
-	attempt        *channeldb.HTLCAttemptInfo
+	currentHeight int32
-	circuit        *sphinx.Circuit
+}
-	lastError      error
+
+// payemntState holds a number of key insights learned from a given MPPayment
+// that we use to determine what to do on each payment loop iteration.
+type paymentState struct {
+	numShardsInFlight int
+	remainingAmt      lnwire.MilliSatoshi
+	remainingFees     lnwire.MilliSatoshi
+	terminate         bool
+}
+
+// paymentState uses the passed payment to find the latest information we need
+// to act on every iteration of the payment loop.
+func (p *paymentLifecycle) paymentState(payment *channeldb.MPPayment) (
+	*paymentState, error) {
+
+	// Fetch the total amount and fees that has already been sent in
+	// settled and still in-flight shards.
+	sentAmt, fees := payment.SentAmt()
+
+	// Sanity check we haven't sent a value larger than the payment amount.
+	if sentAmt > p.totalAmount {
+		return nil, fmt.Errorf("amount sent %v exceeds "+
+			"total amount %v", sentAmt, p.totalAmount)
+	}
+
+	// We'll subtract the used fee from our fee budget, but allow the fees
+	// of the already sent shards to exceed our budget (can happen after
+	// restarts).
+	feeBudget := p.feeLimit
+	if fees <= feeBudget {
+		feeBudget -= fees
+	} else {
+		feeBudget = 0
+	}
+
+	// Get any terminal info for this payment.
+	settle, failure := payment.TerminalInfo()
+
+	// If either an HTLC settled, or the payment has a payment level
+	// failure recorded, it means we should terminate the moment all shards
+	// have returned with a result.
+	terminate := settle != nil || failure != nil
+
+	activeShards := payment.InFlightHTLCs()
+	return &paymentState{
+		numShardsInFlight: len(activeShards),
+		remainingAmt:      p.totalAmount - sentAmt,
+		remainingFees:     feeBudget,
+		terminate:         terminate,
+	}, nil
 }
 
 // resumePayment resumes the paymentLifecycle from the current state.
 func (p *paymentLifecycle) resumePayment() ([32]byte, *route.Route, error) {
+	shardHandler := &shardHandler{
+		router:      p.router,
+		paymentHash: p.paymentHash,
+		shardErrors: make(chan error),
+		quit:        make(chan struct{}),
+	}
+
+	// When the payment lifecycle loop exits, we make sure to signal any
+	// sub goroutine of the shardHandler to exit, then wait for them to
+	// return.
+	defer shardHandler.stop()
+
+	// If we had any existing attempts outstanding, we'll start by spinning
+	// up goroutines that'll collect their results and deliver them to the
+	// lifecycle loop below.
+	payment, err := p.router.cfg.Control.FetchPayment(
+		p.paymentHash,
+	)
+	if err != nil {
+		return [32]byte{}, nil, err
+	}
+
+	for _, a := range payment.InFlightHTLCs() {
+		a := a
+
+		log.Debugf("Resuming payment shard %v for hash %v",
+			a.AttemptID, p.paymentHash)
+
+		shardHandler.collectResultAsync(&a.HTLCAttemptInfo)
+	}
+
 	// We'll continue until either our payment succeeds, or we encounter a
 	// critical error during path finding.
 	for {
-
+		// Start by quickly checking if there are any outcomes already
-		// If this payment had no existing payment attempt, we create
+		// available to handle before we reevaluate our state.
-		// and send one now.
+		if err := shardHandler.checkShards(); err != nil {
-		if p.attempt == nil {
-			firstHop, htlcAdd, err := p.createNewPaymentAttempt()
-			if err != nil {
-				return [32]byte{}, nil, err
-			}
-
-			// Now that the attempt is created and checkpointed to
-			// the DB, we send it.
-			sendErr := p.sendPaymentAttempt(firstHop, htlcAdd)
-			if sendErr != nil {
-				// TODO(joostjager): Distinguish unexpected
-				// internal errors from real send errors.
-				err = p.failAttempt(sendErr)
-				if err != nil {
-					return [32]byte{}, nil, err
-				}
-
-				// We must inspect the error to know whether it
-				// was critical or not, to decide whether we
-				// should continue trying.
-				err := p.handleSendError(sendErr)
-				if err != nil {
-					return [32]byte{}, nil, err
-				}
-
-				// Error was handled successfully, reset the
-				// attempt to indicate we want to make a new
-				// attempt.
-				p.attempt = nil
-				continue
-			}
-		} else {
-			// If this was a resumed attempt, we must regenerate the
-			// circuit. We don't need to check for errors resulting
-			// from an invalid route, because the sphinx packet has
-			// been successfully generated before.
-			_, c, err := generateSphinxPacket(
-				&p.attempt.Route, p.payment.PaymentHash[:],
-				p.attempt.SessionKey,
-			)
-			if err != nil {
-				return [32]byte{}, nil, err
-			}
-			p.circuit = c
-		}
-
-		// Using the created circuit, initialize the error decrypter so we can
-		// parse+decode any failures incurred by this payment within the
-		// switch.
-		errorDecryptor := &htlcswitch.SphinxErrorDecrypter{
-			OnionErrorDecrypter: sphinx.NewOnionErrorDecrypter(p.circuit),
-		}
-
-		// Now ask the switch to return the result of the payment when
-		// available.
-		resultChan, err := p.router.cfg.Payer.GetPaymentResult(
-			p.attempt.AttemptID, p.payment.PaymentHash, errorDecryptor,
-		)
-		switch {
-
-		// If this attempt ID is unknown to the Switch, it means it was
-		// never checkpointed and forwarded by the switch before a
-		// restart. In this case we can safely send a new payment
-		// attempt, and wait for its result to be available.
-		case err == htlcswitch.ErrPaymentIDNotFound:
-			log.Debugf("Payment ID %v for hash %x not found in "+
-				"the Switch, retrying.", p.attempt.AttemptID,
-				p.payment.PaymentHash)
-
-			err = p.failAttempt(err)
-			if err != nil {
-				return [32]byte{}, nil, err
-			}
-
-			// Reset the attempt to indicate we want to make a new
-			// attempt.
-			p.attempt = nil
-			continue
-
-		// A critical, unexpected error was encountered.
-		case err != nil:
-			log.Errorf("Failed getting result for attemptID %d "+
-				"from switch: %v", p.attempt.AttemptID, err)
-
 			return [32]byte{}, nil, err
 		}
 
-		// The switch knows about this payment, we'll wait for a result
+		// We start every iteration by fetching the lastest state of
-		// to be available.
+		// the payment from the ControlTower. This ensures that we will
-		var (
+		// act on the latest available information, whether we are
-			result *htlcswitch.PaymentResult
+		// resuming an existing payment or just sent a new attempt.
-			ok     bool
+		payment, err := p.router.cfg.Control.FetchPayment(
+			p.paymentHash,
 		)
+		if err != nil {
+			return [32]byte{}, nil, err
+		}
 
-		select {
+		// Using this latest state of the payment, calculate
-		case result, ok = <-resultChan:
+		// information about our active shards and terminal conditions.
-			if !ok {
+		state, err := p.paymentState(payment)
-				return [32]byte{}, nil, htlcswitch.ErrSwitchExiting
+		if err != nil {
+			return [32]byte{}, nil, err
+		}
+
+		log.Debugf("Payment %v in state terminate=%v, "+
+			"active_shards=%v, rem_value=%v, fee_limit=%v",
+			p.paymentHash, state.terminate, state.numShardsInFlight,
+			state.remainingAmt, state.remainingFees)
+
+		switch {
+
+		// We have a terminal condition and no active shards, we are
+		// ready to exit.
+		case state.terminate && state.numShardsInFlight == 0:
+			// Find the first successful shard and return
+			// the preimage and route.
+			for _, a := range payment.HTLCs {
+				if a.Settle != nil {
+					return a.Settle.Preimage, &a.Route, nil
+				}
 			}
 
+			// Payment failed.
+			return [32]byte{}, nil, *payment.FailureReason
+
+		// If we either reached a terminal error condition (but had
+		// active shards still) or there is no remaining value to send,
+		// we'll wait for a shard outcome.
+		case state.terminate || state.remainingAmt == 0:
+			// We still have outstanding shards, so wait for a new
+			// outcome to be available before re-evaluating our
+			// state.
+			if err := shardHandler.waitForShard(); err != nil {
+				return [32]byte{}, nil, err
+			}
+			continue
+		}
+
+		// Before we attempt any new shard, we'll check to see if
+		// either we've gone past the payment attempt timeout, or the
+		// router is exiting. In either case, we'll stop this payment
+		// attempt short. If a timeout is not applicable, timeoutChan
+		// will be nil.
+		select {
+		case <-p.timeoutChan:
+			log.Warnf("payment attempt not completed before " +
+				"timeout")
+
+			// By marking the payment failed with the control
+			// tower, no further shards will be launched and we'll
+			// return with an error the moment all active shards
+			// have finished.
+			saveErr := p.router.cfg.Control.Fail(
+				p.paymentHash, channeldb.FailureReasonTimeout,
+			)
+			if saveErr != nil {
+				return [32]byte{}, nil, saveErr
+			}
+
+			continue
+
 		case <-p.router.quit:
 			return [32]byte{}, nil, ErrRouterShuttingDown
+
+		// Fall through if we haven't hit our time limit.
+		default:
 		}
 
-		// In case of a payment failure, we use the error to decide
+		// Create a new payment attempt from the given payment session.
-		// whether we should retry.
+		rt, err := p.paySession.RequestRoute(
-		if result.Error != nil {
+			state.remainingAmt, state.remainingFees,
-			log.Errorf("Attempt to send payment %x failed: %v",
+			uint32(state.numShardsInFlight), uint32(p.currentHeight),
-				p.payment.PaymentHash, result.Error)
+		)
+		if err != nil {
+			log.Warnf("Failed to find route for payment %x: %v",
+				p.paymentHash, err)
 
-			err = p.failAttempt(result.Error)
+			routeErr, ok := err.(noRouteError)
-			if err != nil {
+			if !ok {
 				return [32]byte{}, nil, err
 			}
 
+			// There is no route to try, and we have no active
+			// shards. This means that there is no way for us to
+			// send the payment, so mark it failed with no route.
+			if state.numShardsInFlight == 0 {
+				failureCode := routeErr.FailureReason()
+				log.Debugf("Marking payment %v permanently "+
+					"failed with no route: %v",
+					p.paymentHash, failureCode)
+
+				saveErr := p.router.cfg.Control.Fail(
+					p.paymentHash, failureCode,
+				)
+				if saveErr != nil {
+					return [32]byte{}, nil, saveErr
+				}
+
+				continue
+			}
+
+			// We still have active shards, we'll wait for an
+			// outcome to be available before retrying.
+			if err := shardHandler.waitForShard(); err != nil {
+				return [32]byte{}, nil, err
+			}
+			continue
+		}
+
+		// We found a route to try, launch a new shard.
+		attempt, outcome, err := shardHandler.launchShard(rt)
+		if err != nil {
+			return [32]byte{}, nil, err
+		}
+
+		// If we encountered a non-critical error when launching the
+		// shard, handle it.
+		if outcome.err != nil {
+			log.Warnf("Failed to launch shard %v for "+
+				"payment %v: %v", attempt.AttemptID,
+				p.paymentHash, outcome.err)
+
 			// We must inspect the error to know whether it was
 			// critical or not, to decide whether we should
 			// continue trying.
-			if err := p.handleSendError(result.Error); err != nil {
+			err := shardHandler.handleSendError(
+				attempt, outcome.err,
+			)
+			if err != nil {
 				return [32]byte{}, nil, err
 			}
 
-			// Error was handled successfully, reset the attempt to
+			// Error was handled successfully, continue to make a
-			// indicate we want to make a new attempt.
+			// new attempt.
-			p.attempt = nil
 			continue
 		}
 
-		// We successfully got a payment result back from the switch.
+		// Now that the shard was successfully sent, launch a go
-		log.Debugf("Payment %x succeeded with pid=%v",
+		// routine that will handle its result when its back.
-			p.payment.PaymentHash, p.attempt.AttemptID)
+		shardHandler.collectResultAsync(attempt)
-
-		// Report success to mission control.
-		err = p.router.cfg.MissionControl.ReportPaymentSuccess(
-			p.attempt.AttemptID, &p.attempt.Route,
-		)
-		if err != nil {
-			log.Errorf("Error reporting payment success to mc: %v",
-				err)
-		}
-
-		// In case of success we atomically store the db payment and
-		// move the payment to the success state.
-		err = p.router.cfg.Control.SettleAttempt(
-			p.payment.PaymentHash, p.attempt.AttemptID,
-			&channeldb.HTLCSettleInfo{
-				Preimage:   result.Preimage,
-				SettleTime: p.router.cfg.Clock.Now(),
-			},
-		)
-		if err != nil {
-			log.Errorf("Unable to succeed payment "+
-				"attempt: %v", err)
-			return [32]byte{}, nil, err
-		}
-
-		// Terminal state, return the preimage and the route
-		// taken.
-		return result.Preimage, &p.attempt.Route, nil
 	}
-
 }
 
-// errorToPaymentFailure takes a path finding error and converts it into a
+// shardHandler holds what is necessary to send and collect the result of
-// payment-level failure.
+// shards.
-func errorToPaymentFailure(err error) channeldb.FailureReason {
+type shardHandler struct {
-	switch err {
+	paymentHash lntypes.Hash
-	case
+	router      *ChannelRouter
-		errNoTlvPayload,
-		errNoPaymentAddr,
-		errNoPathFound,
-		errPrebuiltRouteTried:
 
-		return channeldb.FailureReasonNoRoute
+	// shardErrors is a channel where errors collected by calling
+	// collectResultAsync will be delivered. These results are meant to be
+	// inspected by calling waitForShard or checkShards, and the channel
+	// doesn't need to be initiated if the caller is using the sync
+	// collectResult directly.
+	shardErrors chan error
 
-	case errInsufficientBalance:
+	// quit is closed to signal the sub goroutines of the payment lifecycle
-		return channeldb.FailureReasonInsufficientBalance
+	// to stop.
-	}
+	quit chan struct{}
-
+	wg   sync.WaitGroup
-	return channeldb.FailureReasonError
 }
 
-// createNewPaymentAttempt creates and stores a new payment attempt to the
+// stop signals any active shard goroutine to exit and waits for them to exit.
-// database.
+func (p *shardHandler) stop() {
-func (p *paymentLifecycle) createNewPaymentAttempt() (lnwire.ShortChannelID,
+	close(p.quit)
-	*lnwire.UpdateAddHTLC, error) {
+	p.wg.Wait()
+}
 
-	// Before we attempt this next payment, we'll check to see if either
+// waitForShard blocks until any of the outstanding shards return.
-	// we've gone past the payment attempt timeout, or the router is
+func (p *shardHandler) waitForShard() error {
-	// exiting. In either case, we'll stop this payment attempt short. If a
-	// timeout is not applicable, timeoutChan will be nil.
 	select {
-	case <-p.timeoutChan:
+	case err := <-p.shardErrors:
-		// Mark the payment as failed because of the
+		return err
-		// timeout.
-		err := p.router.cfg.Control.Fail(
-			p.payment.PaymentHash, channeldb.FailureReasonTimeout,
-		)
-		if err != nil {
-			return lnwire.ShortChannelID{}, nil, err
-		}
 
-		errStr := fmt.Sprintf("payment attempt not completed " +
+	case <-p.quit:
-			"before timeout")
+		return fmt.Errorf("shard handler quitting")
-
-		return lnwire.ShortChannelID{}, nil,
-			newErr(ErrPaymentAttemptTimeout, errStr)
 
 	case <-p.router.quit:
-		// The payment will be resumed from the current state
+		return ErrRouterShuttingDown
-		// after restart.
-		return lnwire.ShortChannelID{}, nil, ErrRouterShuttingDown
-
-	default:
-		// Fall through if we haven't hit our time limit, or
-		// are expiring.
 	}
+}
 
-	// Create a new payment attempt from the given payment session.
+// checkShards is a non-blocking method that check if any shards has finished
-	rt, err := p.paySession.RequestRoute(
+// their execution.
-		p.payment, uint32(p.currentHeight), p.finalCLTVDelta,
+func (p *shardHandler) checkShards() error {
+	for {
+		select {
+		case err := <-p.shardErrors:
+			if err != nil {
+				return err
+			}
+
+		case <-p.quit:
+			return fmt.Errorf("shard handler quitting")
+
+		case <-p.router.quit:
+			return ErrRouterShuttingDown
+
+		default:
+			return nil
+		}
+	}
+}
+
+// launchOutcome is a type returned from launchShard that indicates whether the
+// shard was successfully send onto the network.
+type launchOutcome struct {
+	// err is non-nil if a non-critical error was encountered when trying
+	// to send the shard, and we successfully updated the control tower to
+	// reflect this error. This can be errors like not enough local
+	// balance for the given route etc.
+	err error
+}
+
+// launchShard creates and sends an HTLC attempt along the given route,
+// registering it with the control tower before sending it. It returns the
+// HTLCAttemptInfo that was created for the shard, along with a launchOutcome.
+// The launchOutcome is used to indicate whether the attempt was successfully
+// sent. If the launchOutcome wraps a non-nil error, it means that the attempt
+// was not sent onto the network, so no result will be available in the future
+// for it.
+func (p *shardHandler) launchShard(rt *route.Route) (*channeldb.HTLCAttemptInfo,
+	*launchOutcome, error) {
+
+	// Using the route received from the payment session, create a new
+	// shard to send.
+	firstHop, htlcAdd, attempt, err := p.createNewPaymentAttempt(
+		rt,
 	)
 	if err != nil {
-		log.Warnf("Failed to find route for payment %x: %v",
+		return nil, nil, err
-			p.payment.PaymentHash, err)
-
-		// Convert error to payment-level failure.
-		failure := errorToPaymentFailure(err)
-
-		// If we're unable to successfully make a payment using
-		// any of the routes we've found, then mark the payment
-		// as permanently failed.
-		saveErr := p.router.cfg.Control.Fail(
-			p.payment.PaymentHash, failure,
-		)
-		if saveErr != nil {
-			return lnwire.ShortChannelID{}, nil, saveErr
-		}
-
-		// If there was an error already recorded for this
-		// payment, we'll return that.
-		if p.lastError != nil {
-			return lnwire.ShortChannelID{}, nil,
-				errNoRoute{lastError: p.lastError}
-		}
-		// Terminal state, return.
-		return lnwire.ShortChannelID{}, nil, err
 	}
 
+	// Before sending this HTLC to the switch, we checkpoint the fresh
+	// paymentID and route to the DB. This lets us know on startup the ID
+	// of the payment that we attempted to send, such that we can query the
+	// Switch for its whereabouts. The route is needed to handle the result
+	// when it eventually comes back.
+	err = p.router.cfg.Control.RegisterAttempt(p.paymentHash, attempt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Now that the attempt is created and checkpointed to the DB, we send
+	// it.
+	sendErr := p.sendPaymentAttempt(attempt, firstHop, htlcAdd)
+	if sendErr != nil {
+		// TODO(joostjager): Distinguish unexpected internal errors
+		// from real send errors.
+		err := p.failAttempt(attempt, sendErr)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// Return a launchOutcome indicating the shard failed.
+		return attempt, &launchOutcome{
+			err: sendErr,
+		}, nil
+	}
+
+	return attempt, &launchOutcome{}, nil
+}
+
+// shardResult holds the resulting outcome of a shard sent.
+type shardResult struct {
+	// preimage is the payment preimage in case of a settled HTLC. Only set
+	// if err is non-nil.
+	preimage lntypes.Preimage
+
+	// err indicates that the shard failed.
+	err error
+}
+
+// collectResultAsync launches a goroutine that will wait for the result of the
+// given HTLC attempt to be available then handle its result. Note that it will
+// fail the payment with the control tower if a terminal error is encountered.
+func (p *shardHandler) collectResultAsync(attempt *channeldb.HTLCAttemptInfo) {
+	p.wg.Add(1)
+	go func() {
+		defer p.wg.Done()
+
+		// Block until the result is available.
+		result, err := p.collectResult(attempt)
+		if err != nil {
+			if err != ErrRouterShuttingDown &&
+				err != htlcswitch.ErrSwitchExiting {
+
+				log.Errorf("Error collecting result for "+
+					"shard %v for payment %v: %v",
+					attempt.AttemptID, p.paymentHash, err)
+			}
+
+			select {
+			case p.shardErrors <- err:
+			case <-p.router.quit:
+			case <-p.quit:
+			}
+			return
+		}
+
+		// If a non-critical error was encountered handle it and mark
+		// the payment failed if the failure was terminal.
+		if result.err != nil {
+			err := p.handleSendError(attempt, result.err)
+			if err != nil {
+				select {
+				case p.shardErrors <- err:
+				case <-p.router.quit:
+				case <-p.quit:
+				}
+				return
+			}
+		}
+
+		select {
+		case p.shardErrors <- nil:
+		case <-p.router.quit:
+		case <-p.quit:
+		}
+	}()
+}
+
+// collectResult waits for the result for the given attempt to be available
+// from the Switch, then records the attempt outcome with the control tower. A
+// shardResult is returned, indicating the final outcome of this HTLC attempt.
+func (p *shardHandler) collectResult(attempt *channeldb.HTLCAttemptInfo) (
+	*shardResult, error) {
+
+	// Regenerate the circuit for this attempt.
+	_, circuit, err := generateSphinxPacket(
+		&attempt.Route, p.paymentHash[:],
+		attempt.SessionKey,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// Using the created circuit, initialize the error decrypter so we can
+	// parse+decode any failures incurred by this payment within the
+	// switch.
+	errorDecryptor := &htlcswitch.SphinxErrorDecrypter{
+		OnionErrorDecrypter: sphinx.NewOnionErrorDecrypter(circuit),
+	}
+
+	// Now ask the switch to return the result of the payment when
+	// available.
+	resultChan, err := p.router.cfg.Payer.GetPaymentResult(
+		attempt.AttemptID, p.paymentHash, errorDecryptor,
+	)
+	switch {
+
+	// If this attempt ID is unknown to the Switch, it means it was never
+	// checkpointed and forwarded by the switch before a restart. In this
+	// case we can safely send a new payment attempt, and wait for its
+	// result to be available.
+	case err == htlcswitch.ErrPaymentIDNotFound:
+		log.Debugf("Payment ID %v for hash %x not found in "+
+			"the Switch, retrying.", attempt.AttemptID,
+			p.paymentHash)
+
+		cErr := p.failAttempt(attempt, err)
+		if cErr != nil {
+			return nil, cErr
+		}
+
+		return &shardResult{
+			err: err,
+		}, nil
+
+	// A critical, unexpected error was encountered.
+	case err != nil:
+		log.Errorf("Failed getting result for attemptID %d "+
+			"from switch: %v", attempt.AttemptID, err)
+
+		return nil, err
+	}
+
+	// The switch knows about this payment, we'll wait for a result to be
+	// available.
+	var (
+		result *htlcswitch.PaymentResult
+		ok     bool
+	)
+
+	select {
+	case result, ok = <-resultChan:
+		if !ok {
+			return nil, htlcswitch.ErrSwitchExiting
+		}
+
+	case <-p.router.quit:
+		return nil, ErrRouterShuttingDown
+
+	case <-p.quit:
+		return nil, fmt.Errorf("shard handler exiting")
+	}
+
+	// In case of a payment failure, fail the attempt with the control
+	// tower and return.
+	if result.Error != nil {
+		err := p.failAttempt(attempt, result.Error)
+		if err != nil {
+			return nil, err
+		}
+
+		return &shardResult{
+			err: result.Error,
+		}, nil
+	}
+
+	// We successfully got a payment result back from the switch.
+	log.Debugf("Payment %x succeeded with pid=%v",
+		p.paymentHash, attempt.AttemptID)
+
+	// Report success to mission control.
+	err = p.router.cfg.MissionControl.ReportPaymentSuccess(
+		attempt.AttemptID, &attempt.Route,
+	)
+	if err != nil {
+		log.Errorf("Error reporting payment success to mc: %v",
+			err)
+	}
+
+	// In case of success we atomically store settle result to the DB move
+	// the shard to the settled state.
+	err = p.router.cfg.Control.SettleAttempt(
+		p.paymentHash, attempt.AttemptID,
+		&channeldb.HTLCSettleInfo{
+			Preimage:   result.Preimage,
+			SettleTime: p.router.cfg.Clock.Now(),
+		},
+	)
+	if err != nil {
+		log.Errorf("Unable to succeed payment attempt: %v", err)
+		return nil, err
+	}
+
+	return &shardResult{
+		preimage: result.Preimage,
+	}, nil
+}
+
+// createNewPaymentAttempt creates a new payment attempt from the given route.
+func (p *shardHandler) createNewPaymentAttempt(rt *route.Route) (
+	lnwire.ShortChannelID, *lnwire.UpdateAddHTLC,
+	*channeldb.HTLCAttemptInfo, error) {
+
 	// Generate a new key to be used for this attempt.
 	sessionKey, err := generateNewSessionKey()
 	if err != nil {
-		return lnwire.ShortChannelID{}, nil, err
+		return lnwire.ShortChannelID{}, nil, nil, err
 	}
 
 	// Generate the raw encoded sphinx packet to be included along
 	// with the htlcAdd message that we send directly to the
 	// switch.
-	onionBlob, c, err := generateSphinxPacket(
+	onionBlob, _, err := generateSphinxPacket(
-		rt, p.payment.PaymentHash[:], sessionKey,
+		rt, p.paymentHash[:], sessionKey,
 	)
-
-	// With SendToRoute, it can happen that the route exceeds protocol
-	// constraints. Mark the payment as failed with an internal error.
-	if err == route.ErrMaxRouteHopsExceeded ||
-		err == sphinx.ErrMaxRoutingInfoSizeExceeded {
-
-		log.Debugf("Invalid route provided for payment %x: %v",
-			p.payment.PaymentHash, err)
-
-		controlErr := p.router.cfg.Control.Fail(
-			p.payment.PaymentHash, channeldb.FailureReasonError,
-		)
-		if controlErr != nil {
-			return lnwire.ShortChannelID{}, nil, controlErr
-		}
-	}
-
-	// In any case, don't continue if there is an error.
 	if err != nil {
-		return lnwire.ShortChannelID{}, nil, err
+		return lnwire.ShortChannelID{}, nil, nil, err
 	}
 
-	// Update our cached circuit with the newly generated
-	// one.
-	p.circuit = c
-
 	// Craft an HTLC packet to send to the layer 2 switch. The
 	// metadata within this packet will be used to route the
 	// payment through the network, starting with the first-hop.
 	htlcAdd := &lnwire.UpdateAddHTLC{
 		Amount:      rt.TotalAmount,
 		Expiry:      rt.TotalTimeLock,
-		PaymentHash: p.payment.PaymentHash,
+		PaymentHash: p.paymentHash,
 	}
 	copy(htlcAdd.OnionBlob[:], onionBlob)
 
@@ -356,40 +617,30 @@ func (p *paymentLifecycle) createNewPaymentAttempt() (lnwire.ShortChannelID,
 	// this HTLC.
 	attemptID, err := p.router.cfg.NextPaymentID()
 	if err != nil {
-		return lnwire.ShortChannelID{}, nil, err
+		return lnwire.ShortChannelID{}, nil, nil, err
 	}
 
 	// We now have all the information needed to populate
 	// the current attempt information.
-	p.attempt = &channeldb.HTLCAttemptInfo{
+	attempt := &channeldb.HTLCAttemptInfo{
 		AttemptID:   attemptID,
 		AttemptTime: p.router.cfg.Clock.Now(),
 		SessionKey:  sessionKey,
 		Route:       *rt,
 	}
 
-	// Before sending this HTLC to the switch, we checkpoint the
+	return firstHop, htlcAdd, attempt, nil
-	// fresh attemptID and route to the DB. This lets us know on
-	// startup the ID of the payment that we attempted to send,
-	// such that we can query the Switch for its whereabouts. The
-	// route is needed to handle the result when it eventually
-	// comes back.
-	err = p.router.cfg.Control.RegisterAttempt(p.payment.PaymentHash, p.attempt)
-	if err != nil {
-		return lnwire.ShortChannelID{}, nil, err
-	}
-
-	return firstHop, htlcAdd, nil
 }
 
 // sendPaymentAttempt attempts to send the current attempt to the switch.
-func (p *paymentLifecycle) sendPaymentAttempt(firstHop lnwire.ShortChannelID,
+func (p *shardHandler) sendPaymentAttempt(
+	attempt *channeldb.HTLCAttemptInfo, firstHop lnwire.ShortChannelID,
 	htlcAdd *lnwire.UpdateAddHTLC) error {
 
 	log.Tracef("Attempting to send payment %x (pid=%v), "+
-		"using route: %v", p.payment.PaymentHash, p.attempt.AttemptID,
+		"using route: %v", p.paymentHash, attempt.AttemptID,
 		newLogClosure(func() string {
-			return spew.Sdump(p.attempt.Route)
+			return spew.Sdump(attempt.Route)
 		}),
 	)
 
@@ -398,63 +649,60 @@ func (p *paymentLifecycle) sendPaymentAttempt(firstHop lnwire.ShortChannelID,
 	// such that we can resume waiting for the result after a
 	// restart.
 	err := p.router.cfg.Payer.SendHTLC(
-		firstHop, p.attempt.AttemptID, htlcAdd,
+		firstHop, attempt.AttemptID, htlcAdd,
 	)
 	if err != nil {
 		log.Errorf("Failed sending attempt %d for payment "+
-			"%x to switch: %v", p.attempt.AttemptID,
+			"%x to switch: %v", attempt.AttemptID,
-			p.payment.PaymentHash, err)
+			p.paymentHash, err)
 		return err
 	}
 
 	log.Debugf("Payment %x (pid=%v) successfully sent to switch, route: %v",
-		p.payment.PaymentHash, p.attempt.AttemptID, &p.attempt.Route)
+		p.paymentHash, attempt.AttemptID, &attempt.Route)
 
 	return nil
 }
 
 // handleSendError inspects the given error from the Switch and determines
-// whether we should make another payment attempt.
+// whether we should make another payment attempt, or if it should be
-func (p *paymentLifecycle) handleSendError(sendErr error) error {
+// considered a terminal error. Terminal errors will be recorded with the
+// control tower.
+func (p *shardHandler) handleSendError(attempt *channeldb.HTLCAttemptInfo,
+	sendErr error) error {
 
 	reason := p.router.processSendError(
-		p.attempt.AttemptID, &p.attempt.Route, sendErr,
+		attempt.AttemptID, &attempt.Route, sendErr,
 	)
 	if reason == nil {
-		// Save the forwarding error so it can be returned if
-		// this turns out to be the last attempt.
-		p.lastError = sendErr
-
 		return nil
 	}
 
 	log.Debugf("Payment %x failed: final_outcome=%v, raw_err=%v",
-		p.payment.PaymentHash, *reason, sendErr)
+		p.paymentHash, *reason, sendErr)
 
-	// Mark the payment failed with no route.
+	err := p.router.cfg.Control.Fail(p.paymentHash, *reason)
-	//
-	// TODO(halseth): make payment codes for the actual reason we don't
-	// continue path finding.
-	err := p.router.cfg.Control.Fail(
-		p.payment.PaymentHash, *reason,
-	)
 	if err != nil {
 		return err
 	}
 
-	// Terminal state, return the error we encountered.
+	return nil
-	return sendErr
 }
 
 // failAttempt calls control tower to fail the current payment attempt.
-func (p *paymentLifecycle) failAttempt(sendError error) error {
+func (p *shardHandler) failAttempt(attempt *channeldb.HTLCAttemptInfo,
+	sendError error) error {
+
+	log.Warnf("Attempt %v for payment %v failed: %v", attempt.AttemptID,
+		p.paymentHash, sendError)
+
 	failInfo := marshallError(
 		sendError,
 		p.router.cfg.Clock.Now(),
 	)
 
 	return p.router.cfg.Control.FailAttempt(
-		p.payment.PaymentHash, p.attempt.AttemptID,
+		p.paymentHash, attempt.AttemptID,
 		failInfo,
 	)
 }

routing/payment_lifecycle_test.go

@@ -0,0 +1,898 @@
+package routing
+
+import (
+	"crypto/rand"
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcutil"
+	"github.com/go-errors/errors"
+	"github.com/lightningnetwork/lnd/channeldb"
+	"github.com/lightningnetwork/lnd/clock"
+	"github.com/lightningnetwork/lnd/htlcswitch"
+	"github.com/lightningnetwork/lnd/lntypes"
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/routing/route"
+)
+
+const stepTimeout = 5 * time.Second
+
+// createTestRoute builds a route a->b->c paying the given amt to c.
+func createTestRoute(amt lnwire.MilliSatoshi,
+	aliasMap map[string]route.Vertex) (*route.Route, error) {
+
+	hopFee := lnwire.NewMSatFromSatoshis(3)
+	hop1 := aliasMap["b"]
+	hop2 := aliasMap["c"]
+	hops := []*route.Hop{
+		{
+			ChannelID:     1,
+			PubKeyBytes:   hop1,
+			LegacyPayload: true,
+			AmtToForward:  amt + hopFee,
+		},
+		{
+			ChannelID:     2,
+			PubKeyBytes:   hop2,
+			LegacyPayload: true,
+			AmtToForward:  amt,
+		},
+	}
+
+	// We create a simple route that we will supply every time the router
+	// requests one.
+	return route.NewRouteFromHops(
+		amt+2*hopFee, 100, aliasMap["a"], hops,
+	)
+}
+
+// TestRouterPaymentStateMachine tests that the router interacts as expected
+// with the ControlTower during a payment lifecycle, such that it payment
+// attempts are not sent twice to the switch, and results are handled after a
+// restart.
+func TestRouterPaymentStateMachine(t *testing.T) {
+	t.Parallel()
+
+	const startingBlockHeight = 101
+
+	// Setup two simple channels such that we can mock sending along this
+	// route.
+	chanCapSat := btcutil.Amount(100000)
+	testChannels := []*testChannel{
+		symmetricTestChannel("a", "b", chanCapSat, &testChannelPolicy{
+			Expiry:  144,
+			FeeRate: 400,
+			MinHTLC: 1,
+			MaxHTLC: lnwire.NewMSatFromSatoshis(chanCapSat),
+		}, 1),
+		symmetricTestChannel("b", "c", chanCapSat, &testChannelPolicy{
+			Expiry:  144,
+			FeeRate: 400,
+			MinHTLC: 1,
+			MaxHTLC: lnwire.NewMSatFromSatoshis(chanCapSat),
+		}, 2),
+	}
+
+	testGraph, err := createTestGraphFromChannels(testChannels, "a")
+	if err != nil {
+		t.Fatalf("unable to create graph: %v", err)
+	}
+	defer testGraph.cleanUp()
+
+	paymentAmt := lnwire.NewMSatFromSatoshis(1000)
+
+	// We create a simple route that we will supply every time the router
+	// requests one.
+	rt, err := createTestRoute(paymentAmt, testGraph.aliasMap)
+	if err != nil {
+		t.Fatalf("unable to create route: %v", err)
+	}
+
+	shard, err := createTestRoute(paymentAmt/4, testGraph.aliasMap)
+	if err != nil {
+		t.Fatalf("unable to create route: %v", err)
+	}
+
+	// A payment state machine test case consists of several ordered steps,
+	// that we use for driving the scenario.
+	type testCase struct {
+		// steps is a list of steps to perform during the testcase.
+		steps []string
+
+		// routes is the sequence of routes we will provide to the
+		// router when it requests a new route.
+		routes []*route.Route
+	}
+
+	const (
+		// routerInitPayment is a test step where we expect the router
+		// to call the InitPayment method on the control tower.
+		routerInitPayment = "Router:init-payment"
+
+		// routerRegisterAttempt is a test step where we expect the
+		// router to call the RegisterAttempt method on the control
+		// tower.
+		routerRegisterAttempt = "Router:register-attempt"
+
+		// routerSettleAttempt is a test step where we expect the
+		// router to call the SettleAttempt method on the control
+		// tower.
+		routerSettleAttempt = "Router:settle-attempt"
+
+		// routerFailAttempt is a test step where we expect the router
+		// to call the FailAttempt method on the control tower.
+		routerFailAttempt = "Router:fail-attempt"
+
+		// routerFailPayment is a test step where we expect the router
+		// to call the Fail method on the control tower.
+		routerFailPayment = "Router:fail-payment"
+
+		// sendToSwitchSuccess is a step where we expect the router to
+		// call send the payment attempt to the switch, and we will
+		// respond with a non-error, indicating that the payment
+		// attempt was successfully forwarded.
+		sendToSwitchSuccess = "SendToSwitch:success"
+
+		// sendToSwitchResultFailure is a step where we expect the
+		// router to send the payment attempt to the switch, and we
+		// will respond with a forwarding error. This can happen when
+		// forwarding fail on our local links.
+		sendToSwitchResultFailure = "SendToSwitch:failure"
+
+		// getPaymentResultSuccess is a test step where we expect the
+		// router to call the GetPaymentResult method, and we will
+		// respond with a successful payment result.
+		getPaymentResultSuccess = "GetPaymentResult:success"
+
+		// getPaymentResultTempFailure is a test step where we expect the
+		// router to call the GetPaymentResult method, and we will
+		// respond with a forwarding error, expecting the router to retry.
+		getPaymentResultTempFailure = "GetPaymentResult:temp-failure"
+
+		// getPaymentResultTerminalFailure is a test step where we
+		// expect the router to call the GetPaymentResult method, and
+		// we will respond with a terminal error, expecting the router
+		// to stop making payment attempts.
+		getPaymentResultTerminalFailure = "GetPaymentResult:terminal-failure"
+
+		// resendPayment is a test step where we manually try to resend
+		// the same payment, making sure the router responds with an
+		// error indicating that it is already in flight.
+		resendPayment = "ResendPayment"
+
+		// startRouter is a step where we manually start the router,
+		// used to test that it automatically will resume payments at
+		// startup.
+		startRouter = "StartRouter"
+
+		// stopRouter is a test step where we manually make the router
+		// shut down.
+		stopRouter = "StopRouter"
+
+		// paymentSuccess is a step where assert that we receive a
+		// successful result for the original payment made.
+		paymentSuccess = "PaymentSuccess"
+
+		// paymentError is a step where assert that we receive an error
+		// for the original payment made.
+		paymentError = "PaymentError"
+
+		// resentPaymentSuccess is a step where assert that we receive
+		// a successful result for a payment that was resent.
+		resentPaymentSuccess = "ResentPaymentSuccess"
+
+		// resentPaymentError is a step where assert that we receive an
+		// error for a payment that was resent.
+		resentPaymentError = "ResentPaymentError"
+	)
+
+	tests := []testCase{
+		{
+			// Tests a normal payment flow that succeeds.
+			steps: []string{
+				routerInitPayment,
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+				getPaymentResultSuccess,
+				routerSettleAttempt,
+				paymentSuccess,
+			},
+			routes: []*route.Route{rt},
+		},
+		{
+			// A payment flow with a failure on the first attempt,
+			// but that succeeds on the second attempt.
+			steps: []string{
+				routerInitPayment,
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// Make the first sent attempt fail.
+				getPaymentResultTempFailure,
+				routerFailAttempt,
+
+				// The router should retry.
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// Make the second sent attempt succeed.
+				getPaymentResultSuccess,
+				routerSettleAttempt,
+				paymentSuccess,
+			},
+			routes: []*route.Route{rt, rt},
+		},
+		{
+			// A payment flow with a forwarding failure first time
+			// sending to the switch, but that succeeds on the
+			// second attempt.
+			steps: []string{
+				routerInitPayment,
+				routerRegisterAttempt,
+
+				// Make the first sent attempt fail.
+				sendToSwitchResultFailure,
+				routerFailAttempt,
+
+				// The router should retry.
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// Make the second sent attempt succeed.
+				getPaymentResultSuccess,
+				routerSettleAttempt,
+				paymentSuccess,
+			},
+			routes: []*route.Route{rt, rt},
+		},
+		{
+			// A payment that fails on the first attempt, and has
+			// only one route available to try. It will therefore
+			// fail permanently.
+			steps: []string{
+				routerInitPayment,
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// Make the first sent attempt fail.
+				getPaymentResultTempFailure,
+				routerFailAttempt,
+
+				// Since there are no more routes to try, the
+				// payment should fail.
+				routerFailPayment,
+				paymentError,
+			},
+			routes: []*route.Route{rt},
+		},
+		{
+			// We expect the payment to fail immediately if we have
+			// no routes to try.
+			steps: []string{
+				routerInitPayment,
+				routerFailPayment,
+				paymentError,
+			},
+			routes: []*route.Route{},
+		},
+		{
+			// A normal payment flow, where we attempt to resend
+			// the same payment after each step. This ensures that
+			// the router don't attempt to resend a payment already
+			// in flight.
+			steps: []string{
+				routerInitPayment,
+				routerRegisterAttempt,
+
+				// Manually resend the payment, the router
+				// should attempt to init with the control
+				// tower, but fail since it is already in
+				// flight.
+				resendPayment,
+				routerInitPayment,
+				resentPaymentError,
+
+				// The original payment should proceed as
+				// normal.
+				sendToSwitchSuccess,
+
+				// Again resend the payment and assert it's not
+				// allowed.
+				resendPayment,
+				routerInitPayment,
+				resentPaymentError,
+
+				// Notify about a success for the original
+				// payment.
+				getPaymentResultSuccess,
+				routerSettleAttempt,
+
+				// Now that the original payment finished,
+				// resend it again to ensure this is not
+				// allowed.
+				resendPayment,
+				routerInitPayment,
+				resentPaymentError,
+				paymentSuccess,
+			},
+			routes: []*route.Route{rt},
+		},
+		{
+			// Tests that the router is able to handle the
+			// receieved payment result after a restart.
+			steps: []string{
+				routerInitPayment,
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// Shut down the router. The original caller
+				// should get notified about this.
+				stopRouter,
+				paymentError,
+
+				// Start the router again, and ensure the
+				// router registers the success with the
+				// control tower.
+				startRouter,
+				getPaymentResultSuccess,
+				routerSettleAttempt,
+			},
+			routes: []*route.Route{rt},
+		},
+		{
+			// Tests that we are allowed to resend a payment after
+			// it has permanently failed.
+			steps: []string{
+				routerInitPayment,
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// Resending the payment at this stage should
+				// not be allowed.
+				resendPayment,
+				routerInitPayment,
+				resentPaymentError,
+
+				// Make the first attempt fail.
+				getPaymentResultTempFailure,
+				routerFailAttempt,
+
+				// Since we have no more routes to try, the
+				// original payment should fail.
+				routerFailPayment,
+				paymentError,
+
+				// Now resend the payment again. This should be
+				// allowed, since the payment has failed.
+				resendPayment,
+				routerInitPayment,
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+				getPaymentResultSuccess,
+				routerSettleAttempt,
+				resentPaymentSuccess,
+			},
+			routes: []*route.Route{rt},
+		},
+
+		// =====================================
+		// ||          MPP scenarios          ||
+		// =====================================
+		{
+			// Tests a simple successful MP payment of 4 shards.
+			steps: []string{
+				routerInitPayment,
+
+				// shard 0
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 1
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 2
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 3
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// All shards succeed.
+				getPaymentResultSuccess,
+				getPaymentResultSuccess,
+				getPaymentResultSuccess,
+				getPaymentResultSuccess,
+
+				// Router should settle them all.
+				routerSettleAttempt,
+				routerSettleAttempt,
+				routerSettleAttempt,
+				routerSettleAttempt,
+
+				// And the final result is obviously
+				// successful.
+				paymentSuccess,
+			},
+			routes: []*route.Route{shard, shard, shard, shard},
+		},
+		{
+			// An MP payment scenario where we need several extra
+			// attempts before the payment finally settle.
+			steps: []string{
+				routerInitPayment,
+
+				// shard 0
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 1
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 2
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 3
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// First two shards fail, two new ones are sent.
+				getPaymentResultTempFailure,
+				getPaymentResultTempFailure,
+				routerFailAttempt,
+				routerFailAttempt,
+
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// The four shards settle.
+				getPaymentResultSuccess,
+				getPaymentResultSuccess,
+				getPaymentResultSuccess,
+				getPaymentResultSuccess,
+				routerSettleAttempt,
+				routerSettleAttempt,
+				routerSettleAttempt,
+				routerSettleAttempt,
+
+				// Overall payment succeeds.
+				paymentSuccess,
+			},
+			routes: []*route.Route{
+				shard, shard, shard, shard, shard, shard,
+			},
+		},
+		{
+			// An MP payment scenario where 3 of the shards fail.
+			// However the last shard settle, which means we get
+			// the preimage and should consider the overall payment
+			// a success.
+			steps: []string{
+				routerInitPayment,
+
+				// shard 0
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 1
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 2
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 3
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// 3 shards fail, and should be failed by the
+				// router.
+				getPaymentResultTempFailure,
+				getPaymentResultTempFailure,
+				getPaymentResultTempFailure,
+				routerFailAttempt,
+				routerFailAttempt,
+				routerFailAttempt,
+
+				// The fourth shard succeed against all odds,
+				// making the overall payment succeed.
+				getPaymentResultSuccess,
+				routerSettleAttempt,
+				paymentSuccess,
+			},
+			routes: []*route.Route{shard, shard, shard, shard},
+		},
+		{
+			// An MP payment scenario a shard fail with a terminal
+			// error, causing the router to stop attempting.
+			steps: []string{
+				routerInitPayment,
+
+				// shard 0
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 1
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 2
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// shard 3
+				routerRegisterAttempt,
+				sendToSwitchSuccess,
+
+				// The first shard fail with a terminal error.
+				getPaymentResultTerminalFailure,
+				routerFailAttempt,
+				routerFailPayment,
+
+				// Remaining 3 shards fail.
+				getPaymentResultTempFailure,
+				getPaymentResultTempFailure,
+				getPaymentResultTempFailure,
+				routerFailAttempt,
+				routerFailAttempt,
+				routerFailAttempt,
+
+				// Payment fails.
+				paymentError,
+			},
+			routes: []*route.Route{
+				shard, shard, shard, shard, shard, shard,
+			},
+		},
+	}
+
+	// Create a mock control tower with channels set up, that we use to
+	// synchronize and listen for events.
+	control := makeMockControlTower()
+	control.init = make(chan initArgs, 20)
+	control.registerAttempt = make(chan registerAttemptArgs, 20)
+	control.settleAttempt = make(chan settleAttemptArgs, 20)
+	control.failAttempt = make(chan failAttemptArgs, 20)
+	control.failPayment = make(chan failPaymentArgs, 20)
+	control.fetchInFlight = make(chan struct{}, 20)
+
+	quit := make(chan struct{})
+	defer close(quit)
+
+	// setupRouter is a helper method that creates and starts the router in
+	// the desired configuration for this test.
+	setupRouter := func() (*ChannelRouter, chan error,
+		chan *htlcswitch.PaymentResult, chan error) {
+
+		chain := newMockChain(startingBlockHeight)
+		chainView := newMockChainView(chain)
+
+		// We set uo the use the following channels and a mock Payer to
+		// synchonize with the interaction to the Switch.
+		sendResult := make(chan error)
+		paymentResultErr := make(chan error)
+		paymentResult := make(chan *htlcswitch.PaymentResult)
+
+		payer := &mockPayer{
+			sendResult:       sendResult,
+			paymentResult:    paymentResult,
+			paymentResultErr: paymentResultErr,
+		}
+
+		router, err := New(Config{
+			Graph:              testGraph.graph,
+			Chain:              chain,
+			ChainView:          chainView,
+			Control:            control,
+			SessionSource:      &mockPaymentSessionSource{},
+			MissionControl:     &mockMissionControl{},
+			Payer:              payer,
+			ChannelPruneExpiry: time.Hour * 24,
+			GraphPruneInterval: time.Hour * 2,
+			QueryBandwidth: func(e *channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi {
+				return lnwire.NewMSatFromSatoshis(e.Capacity)
+			},
+			NextPaymentID: func() (uint64, error) {
+				next := atomic.AddUint64(&uniquePaymentID, 1)
+				return next, nil
+			},
+			Clock: clock.NewTestClock(time.Unix(1, 0)),
+		})
+		if err != nil {
+			t.Fatalf("unable to create router %v", err)
+		}
+
+		// On startup, the router should fetch all pending payments
+		// from the ControlTower, so assert that here.
+		errCh := make(chan error)
+		go func() {
+			close(errCh)
+			select {
+			case <-control.fetchInFlight:
+				return
+			case <-time.After(1 * time.Second):
+				errCh <- errors.New("router did not fetch in flight " +
+					"payments")
+			}
+		}()
+
+		if err := router.Start(); err != nil {
+			t.Fatalf("unable to start router: %v", err)
+		}
+
+		select {
+		case err := <-errCh:
+			if err != nil {
+				t.Fatalf("error in anonymous goroutine: %s", err)
+			}
+		case <-time.After(1 * time.Second):
+			t.Fatalf("did not fetch in flight payments at startup")
+		}
+
+		return router, sendResult, paymentResult, paymentResultErr
+	}
+
+	router, sendResult, getPaymentResult, getPaymentResultErr := setupRouter()
+	defer func() {
+		if err := router.Stop(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	for _, test := range tests {
+		// Craft a LightningPayment struct.
+		var preImage lntypes.Preimage
+		if _, err := rand.Read(preImage[:]); err != nil {
+			t.Fatalf("unable to generate preimage")
+		}
+
+		payHash := preImage.Hash()
+
+		payment := LightningPayment{
+			Target:      testGraph.aliasMap["c"],
+			Amount:      paymentAmt,
+			FeeLimit:    noFeeLimit,
+			PaymentHash: payHash,
+		}
+
+		router.cfg.SessionSource = &mockPaymentSessionSource{
+			routes: test.routes,
+		}
+
+		router.cfg.MissionControl = &mockMissionControl{}
+
+		// Send the payment. Since this is new payment hash, the
+		// information should be registered with the ControlTower.
+		paymentResult := make(chan error)
+		go func() {
+			_, _, err := router.SendPayment(&payment)
+			paymentResult <- err
+		}()
+
+		var resendResult chan error
+		for _, step := range test.steps {
+			switch step {
+
+			case routerInitPayment:
+				var args initArgs
+				select {
+				case args = <-control.init:
+				case <-time.After(stepTimeout):
+					t.Fatalf("no init payment with control")
+				}
+
+				if args.c == nil {
+					t.Fatalf("expected non-nil CreationInfo")
+				}
+
+			// In this step we expect the router to make a call to
+			// register a new attempt with the ControlTower.
+			case routerRegisterAttempt:
+				var args registerAttemptArgs
+				select {
+				case args = <-control.registerAttempt:
+				case <-time.After(stepTimeout):
+					t.Fatalf("attempt not registered " +
+						"with control")
+				}
+
+				if args.a == nil {
+					t.Fatalf("expected non-nil AttemptInfo")
+				}
+
+			// In this step we expect the router to call the
+			// ControlTower's SettleAttempt method with the preimage.
+			case routerSettleAttempt:
+				select {
+				case <-control.settleAttempt:
+				case <-time.After(stepTimeout):
+					t.Fatalf("attempt settle not " +
+						"registered with control")
+				}
+
+			// In this step we expect the router to call the
+			// ControlTower's FailAttempt method with a HTLC fail
+			// info.
+			case routerFailAttempt:
+				select {
+				case <-control.failAttempt:
+				case <-time.After(stepTimeout):
+					t.Fatalf("attempt fail not " +
+						"registered with control")
+				}
+
+			// In this step we expect the router to call the
+			// ControlTower's Fail method, to indicate that the
+			// payment failed.
+			case routerFailPayment:
+				select {
+				case <-control.failPayment:
+				case <-time.After(stepTimeout):
+					t.Fatalf("payment fail not " +
+						"registered with control")
+				}
+
+			// In this step we expect the SendToSwitch method to be
+			// called, and we respond with a nil-error.
+			case sendToSwitchSuccess:
+				select {
+				case sendResult <- nil:
+				case <-time.After(stepTimeout):
+					t.Fatalf("unable to send result")
+				}
+
+			// In this step we expect the SendToSwitch method to be
+			// called, and we respond with a forwarding error
+			case sendToSwitchResultFailure:
+				select {
+				case sendResult <- htlcswitch.NewForwardingError(
+					&lnwire.FailTemporaryChannelFailure{},
+					1,
+				):
+				case <-time.After(stepTimeout):
+					t.Fatalf("unable to send result")
+				}
+
+			// In this step we expect the GetPaymentResult method
+			// to be called, and we respond with the preimage to
+			// complete the payment.
+			case getPaymentResultSuccess:
+				select {
+				case getPaymentResult <- &htlcswitch.PaymentResult{
+					Preimage: preImage,
+				}:
+				case <-time.After(stepTimeout):
+					t.Fatalf("unable to send result")
+				}
+
+			// In this state we expect the GetPaymentResult method
+			// to be called, and we respond with a forwarding
+			// error, indicating that the router should retry.
+			case getPaymentResultTempFailure:
+				failure := htlcswitch.NewForwardingError(
+					&lnwire.FailTemporaryChannelFailure{},
+					1,
+				)
+
+				select {
+				case getPaymentResult <- &htlcswitch.PaymentResult{
+					Error: failure,
+				}:
+				case <-time.After(stepTimeout):
+					t.Fatalf("unable to get result")
+				}
+
+			// In this state we expect the router to call the
+			// GetPaymentResult method, and we will respond with a
+			// terminal error, indiating the router should stop
+			// making payment attempts.
+			case getPaymentResultTerminalFailure:
+				failure := htlcswitch.NewForwardingError(
+					&lnwire.FailIncorrectDetails{},
+					1,
+				)
+
+				select {
+				case getPaymentResult <- &htlcswitch.PaymentResult{
+					Error: failure,
+				}:
+				case <-time.After(stepTimeout):
+					t.Fatalf("unable to get result")
+				}
+
+			// In this step we manually try to resend the same
+			// payment, making sure the router responds with an
+			// error indicating that it is already in flight.
+			case resendPayment:
+				resendResult = make(chan error)
+				go func() {
+					_, _, err := router.SendPayment(&payment)
+					resendResult <- err
+				}()
+
+			// In this step we manually stop the router.
+			case stopRouter:
+				select {
+				case getPaymentResultErr <- fmt.Errorf(
+					"shutting down"):
+				case <-time.After(stepTimeout):
+					t.Fatalf("unable to send payment " +
+						"result error")
+				}
+
+				if err := router.Stop(); err != nil {
+					t.Fatalf("unable to restart: %v", err)
+				}
+
+			// In this step we manually start the router.
+			case startRouter:
+				router, sendResult, getPaymentResult,
+					getPaymentResultErr = setupRouter()
+
+			// In this state we expect to receive an error for the
+			// original payment made.
+			case paymentError:
+				select {
+				case err := <-paymentResult:
+					if err == nil {
+						t.Fatalf("expected error")
+					}
+
+				case <-time.After(stepTimeout):
+					t.Fatalf("got no payment result")
+				}
+
+			// In this state we expect the original payment to
+			// succeed.
+			case paymentSuccess:
+				select {
+				case err := <-paymentResult:
+					if err != nil {
+						t.Fatalf("did not expect "+
+							"error %v", err)
+					}
+
+				case <-time.After(stepTimeout):
+					t.Fatalf("got no payment result")
+				}
+
+			// In this state we expect to receive an error for the
+			// resent payment made.
+			case resentPaymentError:
+				select {
+				case err := <-resendResult:
+					if err == nil {
+						t.Fatalf("expected error")
+					}
+
+				case <-time.After(stepTimeout):
+					t.Fatalf("got no payment result")
+				}
+
+			// In this state we expect the resent payment to
+			// succeed.
+			case resentPaymentSuccess:
+				select {
+				case err := <-resendResult:
+					if err != nil {
+						t.Fatalf("did not expect error %v", err)
+					}
+
+				case <-time.After(stepTimeout):
+					t.Fatalf("got no payment result")
+				}
+
+			default:
+				t.Fatalf("unknown step %v", step)
+			}
+		}
+	}
+}

routing/payment_session.go

@@ -1,8 +1,6 @@
 package routing
 
 import (
-	"errors"
-
 	"github.com/lightningnetwork/lnd/channeldb"
 	"github.com/lightningnetwork/lnd/lnwire"
 	"github.com/lightningnetwork/lnd/routing/route"
@@ -12,20 +10,89 @@ import (
 // to prevent an HTLC being failed if some blocks are mined while it's in-flight.
 const BlockPadding uint16 = 3
 
-var (
+// noRouteError encodes a non-critical error encountered during path finding.
-	// errPrebuiltRouteTried is returned when the single pre-built route
+type noRouteError uint8
-	// failed and there is nothing more we can do.
+
-	errPrebuiltRouteTried = errors.New("pre-built route already tried")
+const (
+	// errNoTlvPayload is returned when the destination hop does not support
+	// a tlv payload.
+	errNoTlvPayload noRouteError = iota
+
+	// errNoPaymentAddr is returned when the destination hop does not
+	// support payment addresses.
+	errNoPaymentAddr
+
+	// errNoPathFound is returned when a path to the target destination does
+	// not exist in the graph.
+	errNoPathFound
+
+	// errInsufficientLocalBalance is returned when none of the local
+	// channels have enough balance for the payment.
+	errInsufficientBalance
+
+	// errEmptyPaySession is returned when the empty payment session is
+	// queried for a route.
+	errEmptyPaySession
 )
 
+// Error returns the string representation of the noRouteError
+func (e noRouteError) Error() string {
+	switch e {
+	case errNoTlvPayload:
+		return "destination hop doesn't understand new TLV payloads"
+
+	case errNoPaymentAddr:
+		return "destination hop doesn't understand payment addresses"
+
+	case errNoPathFound:
+		return "unable to find a path to destination"
+
+	case errEmptyPaySession:
+		return "empty payment session"
+
+	case errInsufficientBalance:
+		return "insufficient local balance"
+
+	default:
+		return "unknown no-route error"
+	}
+}
+
+// FailureReason converts a path finding error into a payment-level failure.
+func (e noRouteError) FailureReason() channeldb.FailureReason {
+	switch e {
+	case
+		errNoTlvPayload,
+		errNoPaymentAddr,
+		errNoPathFound,
+		errEmptyPaySession:
+
+		return channeldb.FailureReasonNoRoute
+
+	case errInsufficientBalance:
+		return channeldb.FailureReasonInsufficientBalance
+
+	default:
+		return channeldb.FailureReasonError
+	}
+}
+
 // PaymentSession is used during SendPayment attempts to provide routes to
 // attempt. It also defines methods to give the PaymentSession additional
 // information learned during the previous attempts.
 type PaymentSession interface {
 	// RequestRoute returns the next route to attempt for routing the
-	// specified HTLC payment to the target node.
+	// specified HTLC payment to the target node. The returned route should
-	RequestRoute(payment *LightningPayment,
+	// carry at most maxAmt to the target node, and pay at most feeLimit in
-		height uint32, finalCltvDelta uint16) (*route.Route, error)
+	// fees. It can carry less if the payment is MPP. The activeShards
+	// argument should be set to instruct the payment session about the
+	// number of in flight HTLCS for the payment, such that it can choose
+	// splitting strategy accordingly.
+	//
+	// A noRouteError is returned if a non-critical error is encountered
+	// during path finding.
+	RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi,
+		activeShards, height uint32) (*route.Route, error)
 }
 
 // paymentSession is used during an HTLC routings session to prune the local
@@ -43,8 +110,9 @@ type paymentSession struct {
 
 	sessionSource *SessionSource
 
-	preBuiltRoute      *route.Route
+	payment *LightningPayment
-	preBuiltRouteTried bool
+
+	empty bool
 
 	pathFinder pathFinder
 }
@@ -58,31 +126,22 @@ type paymentSession struct {
 //
 // NOTE: This function is safe for concurrent access.
 // NOTE: Part of the PaymentSession interface.
-func (p *paymentSession) RequestRoute(payment *LightningPayment,
+func (p *paymentSession) RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi,
-	height uint32, finalCltvDelta uint16) (*route.Route, error) {
+	activeShards, height uint32) (*route.Route, error) {
 
-	switch {
+	if p.empty {
-
+		return nil, errEmptyPaySession
-	// If we have a pre-built route, use that directly.
-	case p.preBuiltRoute != nil && !p.preBuiltRouteTried:
-		p.preBuiltRouteTried = true
-
-		return p.preBuiltRoute, nil
-
-	// If the pre-built route has been tried already, the payment session is
-	// over.
-	case p.preBuiltRoute != nil:
-		return nil, errPrebuiltRouteTried
 	}
 
 	// Add BlockPadding to the finalCltvDelta so that the receiving node
 	// does not reject the HTLC if some blocks are mined while it's in-flight.
+	finalCltvDelta := p.payment.FinalCLTVDelta
 	finalCltvDelta += BlockPadding
 
 	// We need to subtract the final delta before passing it into path
 	// finding. The optimal path is independent of the final cltv delta and
 	// the path finding algorithm is unaware of this value.
-	cltvLimit := payment.CltvLimit - uint32(finalCltvDelta)
+	cltvLimit := p.payment.CltvLimit - uint32(finalCltvDelta)
 
 	// TODO(roasbeef): sync logic amongst dist sys
 
@@ -93,13 +152,13 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment,
 
 	restrictions := &RestrictParams{
 		ProbabilitySource: ss.MissionControl.GetProbability,
-		FeeLimit:          payment.FeeLimit,
+		FeeLimit:          feeLimit,
-		OutgoingChannelID: payment.OutgoingChannelID,
+		OutgoingChannelID: p.payment.OutgoingChannelID,
-		LastHop:           payment.LastHop,
+		LastHop:           p.payment.LastHop,
 		CltvLimit:         cltvLimit,
-		DestCustomRecords: payment.DestCustomRecords,
+		DestCustomRecords: p.payment.DestCustomRecords,
-		DestFeatures:      payment.DestFeatures,
+		DestFeatures:      p.payment.DestFeatures,
-		PaymentAddr:       payment.PaymentAddr,
+		PaymentAddr:       p.payment.PaymentAddr,
 	}
 
 	// We'll also obtain a set of bandwidthHints from the lower layer for
@@ -122,8 +181,8 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment,
 			bandwidthHints:  bandwidthHints,
 		},
 		restrictions, &ss.PathFindingConfig,
-		ss.SelfNode.PubKeyBytes, payment.Target,
+		ss.SelfNode.PubKeyBytes, p.payment.Target,
-		payment.Amount, finalHtlcExpiry,
+		maxAmt, finalHtlcExpiry,
 	)
 	if err != nil {
 		return nil, err
@@ -135,10 +194,10 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment,
 	route, err := newRoute(
 		sourceVertex, path, height,
 		finalHopParams{
-			amt:         payment.Amount,
+			amt:         maxAmt,
 			cltvDelta:   finalCltvDelta,
-			records:     payment.DestCustomRecords,
+			records:     p.payment.DestCustomRecords,
-			paymentAddr: payment.PaymentAddr,
+			paymentAddr: p.payment.PaymentAddr,
 		},
 	)
 	if err != nil {

routing/payment_session_source.go

@@ -47,10 +47,10 @@ type SessionSource struct {
 // view from Mission Control. An optional set of routing hints can be provided
 // in order to populate additional edges to explore when finding a path to the
 // payment's destination.
-func (m *SessionSource) NewPaymentSession(routeHints [][]zpay32.HopHint,
+func (m *SessionSource) NewPaymentSession(p *LightningPayment) (
-	target route.Vertex) (PaymentSession, error) {
+	PaymentSession, error) {
 
-	edges, err := RouteHintsToEdges(routeHints, target)
+	edges, err := RouteHintsToEdges(p.RouteHints, p.Target)
 	if err != nil {
 		return nil, err
 	}
@@ -70,27 +70,18 @@ func (m *SessionSource) NewPaymentSession(routeHints [][]zpay32.HopHint,
 		additionalEdges:   edges,
 		getBandwidthHints: getBandwidthHints,
 		sessionSource:     m,
+		payment:           p,
 		pathFinder:        findPath,
 	}, nil
 }
 
-// NewPaymentSessionForRoute creates a new paymentSession instance that is just
-// used for failure reporting to missioncontrol.
-func (m *SessionSource) NewPaymentSessionForRoute(preBuiltRoute *route.Route) PaymentSession {
-	return &paymentSession{
-		sessionSource: m,
-		preBuiltRoute: preBuiltRoute,
-	}
-}
-
 // NewPaymentSessionEmpty creates a new paymentSession instance that is empty,
 // and will be exhausted immediately. Used for failure reporting to
 // missioncontrol for resumed payment we don't want to make more attempts for.
 func (m *SessionSource) NewPaymentSessionEmpty() PaymentSession {
 	return &paymentSession{
-		sessionSource:      m,
+		sessionSource: m,
-		preBuiltRoute:      &route.Route{},
+		empty:         true,
-		preBuiltRouteTried: true,
 	}
 }
 

routing/payment_session_test.go

@@ -44,6 +44,16 @@ func TestRequestRoute(t *testing.T) {
 		},
 	}
 
+	cltvLimit := uint32(30)
+	finalCltvDelta := uint16(8)
+
+	payment := &LightningPayment{
+		CltvLimit:      cltvLimit,
+		FinalCLTVDelta: finalCltvDelta,
+		Amount:         1000,
+		FeeLimit:       1000,
+	}
+
 	session := &paymentSession{
 		getBandwidthHints: func() (map[uint64]lnwire.MilliSatoshi,
 			error) {
@@ -51,18 +61,13 @@ func TestRequestRoute(t *testing.T) {
 			return nil, nil
 		},
 		sessionSource: sessionSource,
+		payment:       payment,
 		pathFinder:    findPath,
 	}
 
-	cltvLimit := uint32(30)
+	route, err := session.RequestRoute(
-	finalCltvDelta := uint16(8)
+		payment.Amount, payment.FeeLimit, 0, height,
-
+	)
-	payment := &LightningPayment{
-		CltvLimit:      cltvLimit,
-		FinalCLTVDelta: finalCltvDelta,
-	}
-
-	route, err := session.RequestRoute(payment, height, finalCltvDelta)
 	if err != nil {
 		t.Fatal(err)
 	}

routing/route/route.go

@@ -129,6 +129,23 @@ type Hop struct {
 	LegacyPayload bool
 }
 
+// Copy returns a deep copy of the Hop.
+func (h *Hop) Copy() *Hop {
+	c := *h
+
+	if h.MPP != nil {
+		m := *h.MPP
+		c.MPP = &m
+	}
+
+	if h.AMP != nil {
+		a := *h.AMP
+		c.AMP = &a
+	}
+
+	return &c
+}
+
 // PackHopPayload writes to the passed io.Writer, the series of byes that can
 // be placed directly into the per-hop payload (EOB) for this hop. This will
 // include the required routing fields, as well as serializing any of the
@@ -287,6 +304,18 @@ type Route struct {
 	Hops []*Hop
 }
 
+// Copy returns a deep copy of the Route.
+func (r *Route) Copy() *Route {
+	c := *r
+
+	c.Hops = make([]*Hop, len(r.Hops))
+	for i := range r.Hops {
+		c.Hops[i] = r.Hops[i].Copy()
+	}
+
+	return &c
+}
+
 // HopFee returns the fee charged by the route hop indicated by hopIndex.
 func (r *Route) HopFee(hopIndex int) lnwire.MilliSatoshi {
 	var incomingAmt lnwire.MilliSatoshi
@@ -308,7 +337,25 @@ func (r *Route) TotalFees() lnwire.MilliSatoshi {
 		return 0
 	}
 
-	return r.TotalAmount - r.Hops[len(r.Hops)-1].AmtToForward
+	return r.TotalAmount - r.ReceiverAmt()
+}
+
+// ReceiverAmt is the amount received by the final hop of this route.
+func (r *Route) ReceiverAmt() lnwire.MilliSatoshi {
+	if len(r.Hops) == 0 {
+		return 0
+	}
+
+	return r.Hops[len(r.Hops)-1].AmtToForward
+}
+
+// FinalHop returns the last hop of the route, or nil if the route is empty.
+func (r *Route) FinalHop() *Hop {
+	if len(r.Hops) == 0 {
+		return nil
+	}
+
+	return r.Hops[len(r.Hops)-1]
 }
 
 // NewRouteFromHops creates a new Route structure from the minimally required

routing/route/route_test.go

@@ -20,15 +20,24 @@ var (
 func TestRouteTotalFees(t *testing.T) {
 	t.Parallel()
 
-	// Make sure empty route returns a 0 fee.
+	// Make sure empty route returns a 0 fee, and zero amount.
 	r := &Route{}
 	if r.TotalFees() != 0 {
 		t.Fatalf("expected 0 fees, got %v", r.TotalFees())
 	}
+	if r.ReceiverAmt() != 0 {
+		t.Fatalf("expected 0 amt, got %v", r.ReceiverAmt())
+	}
+
+	// Make sure empty route won't be allowed in the constructor.
+	amt := lnwire.MilliSatoshi(1000)
+	_, err := NewRouteFromHops(amt, 100, Vertex{}, []*Hop{})
+	if err != ErrNoRouteHopsProvided {
+		t.Fatalf("expected ErrNoRouteHopsProvided, got %v", err)
+	}
 
 	// For one-hop routes the fee should be 0, since the last node will
 	// receive the full amount.
-	amt := lnwire.MilliSatoshi(1000)
 	hops := []*Hop{
 		{
 			PubKeyBytes:      Vertex{},
@@ -37,7 +46,7 @@ func TestRouteTotalFees(t *testing.T) {
 			AmtToForward:     amt,
 		},
 	}
-	r, err := NewRouteFromHops(amt, 100, Vertex{}, hops)
+	r, err = NewRouteFromHops(amt, 100, Vertex{}, hops)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -46,6 +55,10 @@ func TestRouteTotalFees(t *testing.T) {
 		t.Fatalf("expected 0 fees, got %v", r.TotalFees())
 	}
 
+	if r.ReceiverAmt() != amt {
+		t.Fatalf("expected %v amt, got %v", amt, r.ReceiverAmt())
+	}
+
 	// Append the route with a node, making the first one take a fee.
 	fee := lnwire.MilliSatoshi(100)
 	hops = append(hops, &Hop{
@@ -64,6 +77,10 @@ func TestRouteTotalFees(t *testing.T) {
 	if r.TotalFees() != fee {
 		t.Fatalf("expected %v fees, got %v", fee, r.TotalFees())
 	}
+
+	if r.ReceiverAmt() != amt-fee {
+		t.Fatalf("expected %v amt, got %v", amt-fee, r.ReceiverAmt())
+	}
 }
 
 var (

routing/router.go

@@ -159,13 +159,7 @@ type PaymentSessionSource interface {
 	// routes to the given target. An optional set of routing hints can be
 	// provided in order to populate additional edges to explore when
 	// finding a path to the payment's destination.
-	NewPaymentSession(routeHints [][]zpay32.HopHint,
+	NewPaymentSession(p *LightningPayment) (PaymentSession, error)
-		target route.Vertex) (PaymentSession, error)
-
-	// NewPaymentSessionForRoute creates a new paymentSession instance that
-	// is just used for failure reporting to missioncontrol, and will only
-	// attempt the given route.
-	NewPaymentSessionForRoute(preBuiltRoute *route.Route) PaymentSession
 
 	// NewPaymentSessionEmpty creates a new paymentSession instance that is
 	// empty, and will be exhausted immediately. Used for failure reporting
@@ -532,23 +526,17 @@ func (r *ChannelRouter) Start() error {
 			// We create a dummy, empty payment session such that
 			// we won't make another payment attempt when the
 			// result for the in-flight attempt is received.
-			//
-			// PayAttemptTime doesn't need to be set, as there is
-			// only a single attempt.
 			paySession := r.cfg.SessionSource.NewPaymentSessionEmpty()
 
-			lPayment := &LightningPayment{
+			// We pass in a zero timeout value, to indicate we
-				PaymentHash: payment.Info.PaymentHash,
+			// don't need it to timeout. It will stop immediately
-			}
+			// after the existing attempt has finished anyway. We
-
+			// also set a zero fee limit, as no more routes should
-			// TODO(joostjager): For mpp, possibly relaunch multiple
+			// be tried.
-			// in-flight htlcs here.
+			_, _, err := r.sendPayment(
-			var attempt *channeldb.HTLCAttemptInfo
+				payment.Info.Value, 0,
-			if len(payment.Attempts) > 0 {
+				payment.Info.PaymentHash, 0, paySession,
-				attempt = &payment.Attempts[0]
+			)
-			}
-
-			_, _, err := r.sendPayment(attempt, lPayment, paySession)
 			if err != nil {
 				log.Errorf("Resuming payment with hash %v "+
 					"failed: %v.", payment.Info.PaymentHash, err)
@@ -1640,9 +1628,15 @@ func (r *ChannelRouter) SendPayment(payment *LightningPayment) ([32]byte,
 		return [32]byte{}, nil, err
 	}
 
+	log.Tracef("Dispatching SendPayment for lightning payment: %v",
+		spewPayment(payment))
+
 	// Since this is the first time this payment is being made, we pass nil
 	// for the existing attempt.
-	return r.sendPayment(nil, payment, paySession)
+	return r.sendPayment(
+		payment.Amount, payment.FeeLimit, payment.PaymentHash,
+		payment.PayAttemptTimeout, paySession,
+	)
 }
 
 // SendPaymentAsync is the non-blocking version of SendPayment. The payment
@@ -1659,7 +1653,13 @@ func (r *ChannelRouter) SendPaymentAsync(payment *LightningPayment) error {
 	go func() {
 		defer r.wg.Done()
 
-		_, _, err := r.sendPayment(nil, payment, paySession)
+		log.Tracef("Dispatching SendPayment for lightning payment: %v",
+			spewPayment(payment))
+
+		_, _, err := r.sendPayment(
+			payment.Amount, payment.FeeLimit, payment.PaymentHash,
+			payment.PayAttemptTimeout, paySession,
+		)
 		if err != nil {
 			log.Errorf("Payment with hash %x failed: %v",
 				payment.PaymentHash, err)
@@ -1669,6 +1669,28 @@ func (r *ChannelRouter) SendPaymentAsync(payment *LightningPayment) error {
 	return nil
 }
 
+// spewPayment returns a log closures that provides a spewed string
+// representation of the passed payment.
+func spewPayment(payment *LightningPayment) logClosure {
+	return newLogClosure(func() string {
+		// Make a copy of the payment with a nilled Curve
+		// before spewing.
+		var routeHints [][]zpay32.HopHint
+		for _, routeHint := range payment.RouteHints {
+			var hopHints []zpay32.HopHint
+			for _, hopHint := range routeHint {
+				h := hopHint.Copy()
+				h.NodeID.Curve = nil
+				hopHints = append(hopHints, h)
+			}
+			routeHints = append(routeHints, hopHints)
+		}
+		p := *payment
+		p.RouteHints = routeHints
+		return spew.Sdump(p)
+	})
+}
+
 // preparePayment creates the payment session and registers the payment with the
 // control tower.
 func (r *ChannelRouter) preparePayment(payment *LightningPayment) (
@@ -1677,9 +1699,7 @@ func (r *ChannelRouter) preparePayment(payment *LightningPayment) (
 	// Before starting the HTLC routing attempt, we'll create a fresh
 	// payment session which will report our errors back to mission
 	// control.
-	paySession, err := r.cfg.SessionSource.NewPaymentSession(
+	paySession, err := r.cfg.SessionSource.NewPaymentSession(payment)
-		payment.RouteHints, payment.Target,
-	)
 	if err != nil {
 		return nil, err
 	}
@@ -1706,14 +1726,19 @@ func (r *ChannelRouter) preparePayment(payment *LightningPayment) (
 // SendToRoute attempts to send a payment with the given hash through the
 // provided route. This function is blocking and will return the obtained
 // preimage if the payment is successful or the full error in case of a failure.
-func (r *ChannelRouter) SendToRoute(hash lntypes.Hash, route *route.Route) (
+func (r *ChannelRouter) SendToRoute(hash lntypes.Hash, rt *route.Route) (
 	lntypes.Preimage, error) {
 
-	// Create a payment session for just this route.
-	paySession := r.cfg.SessionSource.NewPaymentSessionForRoute(route)
-
 	// Calculate amount paid to receiver.
-	amt := route.TotalAmount - route.TotalFees()
+	amt := rt.ReceiverAmt()
+
+	// If this is meant as a MP payment shard, we set the amount
+	// for the creating info to the total amount of the payment.
+	finalHop := rt.Hops[len(rt.Hops)-1]
+	mpp := finalHop.MPP
+	if mpp != nil {
+		amt = mpp.TotalMsat()
+	}
 
 	// Record this payment hash with the ControlTower, ensuring it is not
 	// already in-flight.
@@ -1725,50 +1750,100 @@ func (r *ChannelRouter) SendToRoute(hash lntypes.Hash, route *route.Route) (
 	}
 
 	err := r.cfg.Control.InitPayment(hash, info)
-	if err != nil {
+	switch {
+	// If this is an MPP attempt and the hash is already registered with
+	// the database, we can go on to launch the shard.
+	case err == channeldb.ErrPaymentInFlight && mpp != nil:
+
+	// Any other error is not tolerated.
+	case err != nil:
 		return [32]byte{}, err
 	}
 
-	// Create a (mostly) dummy payment, as the created payment session is
+	log.Tracef("Dispatching SendToRoute for hash %v: %v",
-	// not going to do path finding.
+		hash, newLogClosure(func() string {
-	// TODO(halseth): sendPayment doesn't really need LightningPayment, make
+			return spew.Sdump(rt)
-	// it take just needed fields instead.
+		}),
-	//
+	)
-	// PayAttemptTime doesn't need to be set, as there is only a single
+
-	// attempt.
+	// Launch a shard along the given route.
-	payment := &LightningPayment{
+	sh := &shardHandler{
-		PaymentHash: hash,
+		router:      r,
+		paymentHash: hash,
 	}
 
-	// Since this is the first time this payment is being made, we pass nil
+	var shardError error
-	// for the existing attempt.
+	attempt, outcome, err := sh.launchShard(rt)
-	preimage, _, err := r.sendPayment(nil, payment, paySession)
-	if err != nil {
-		// SendToRoute should return a structured error. In case the
-		// provided route fails, payment lifecycle will return a
-		// noRouteError with the structured error embedded.
-		if noRouteError, ok := err.(errNoRoute); ok {
-			if noRouteError.lastError == nil {
-				return lntypes.Preimage{},
-					errors.New("failure message missing")
-			}
 
-			return lntypes.Preimage{}, noRouteError.lastError
+	// With SendToRoute, it can happen that the route exceeds protocol
+	// constraints. Mark the payment as failed with an internal error.
+	if err == route.ErrMaxRouteHopsExceeded ||
+		err == sphinx.ErrMaxRoutingInfoSizeExceeded {
+
+		log.Debugf("Invalid route provided for payment %x: %v",
+			hash, err)
+
+		controlErr := r.cfg.Control.Fail(
+			hash, channeldb.FailureReasonError,
+		)
+		if controlErr != nil {
+			return [32]byte{}, controlErr
 		}
+	}
 
+	// In any case, don't continue if there is an error.
+	if err != nil {
 		return lntypes.Preimage{}, err
 	}
 
-	return preimage, nil
+	switch {
+	// Failed to launch shard.
+	case outcome.err != nil:
+		shardError = outcome.err
+
+	// Shard successfully launched, wait for the result to be available.
+	default:
+		result, err := sh.collectResult(attempt)
+		if err != nil {
+			return lntypes.Preimage{}, err
+		}
+
+		// We got a successful result.
+		if result.err == nil {
+			return result.preimage, nil
+		}
+
+		// The shard failed, break switch to handle it.
+		shardError = result.err
+	}
+
+	// Since for SendToRoute we won't retry in case the shard fails, we'll
+	// mark the payment failed with the control tower immediately. Process
+	// the error to check if it maps into a terminal error code, if not use
+	// a generic NO_ROUTE error.
+	reason := r.processSendError(
+		attempt.AttemptID, &attempt.Route, shardError,
+	)
+	if reason == nil {
+		r := channeldb.FailureReasonNoRoute
+		reason = &r
+	}
+
+	err = r.cfg.Control.Fail(hash, *reason)
+	if err != nil {
+		return lntypes.Preimage{}, err
+	}
+
+	return lntypes.Preimage{}, shardError
 }
 
-// sendPayment attempts to send a payment as described within the passed
+// sendPayment attempts to send a payment to the passed payment hash. This
-// LightningPayment. This function is blocking and will return either: when the
+// function is blocking and will return either: when the payment is successful,
-// payment is successful, or all candidates routes have been attempted and
+// or all candidates routes have been attempted and resulted in a failed
-// resulted in a failed payment. If the payment succeeds, then a non-nil Route
+// payment. If the payment succeeds, then a non-nil Route will be returned
-// will be returned which describes the path the successful payment traversed
+// which describes the path the successful payment traversed within the network
-// within the network to reach the destination. Additionally, the payment
+// to reach the destination. Additionally, the payment preimage will also be
-// preimage will also be returned.
+// returned.
 //
 // The existing attempt argument should be set to nil if this is a payment that
 // haven't had any payment attempt sent to the switch yet. If it has had an
@@ -1779,29 +1854,9 @@ func (r *ChannelRouter) SendToRoute(hash lntypes.Hash, route *route.Route) (
 // router will call this method for every payment still in-flight according to
 // the ControlTower.
 func (r *ChannelRouter) sendPayment(
-	existingAttempt *channeldb.HTLCAttemptInfo,
+	totalAmt, feeLimit lnwire.MilliSatoshi, paymentHash lntypes.Hash,
-	payment *LightningPayment, paySession PaymentSession) (
+	timeout time.Duration,
-	[32]byte, *route.Route, error) {
+	paySession PaymentSession) ([32]byte, *route.Route, error) {
-
-	log.Tracef("Dispatching route for lightning payment: %v",
-		newLogClosure(func() string {
-			// Make a copy of the payment with a nilled Curve
-			// before spewing.
-			var routeHints [][]zpay32.HopHint
-			for _, routeHint := range payment.RouteHints {
-				var hopHints []zpay32.HopHint
-				for _, hopHint := range routeHint {
-					h := hopHint.Copy()
-					h.NodeID.Curve = nil
-					hopHints = append(hopHints, h)
-				}
-				routeHints = append(routeHints, hopHints)
-			}
-			p := *payment
-			p.RouteHints = routeHints
-			return spew.Sdump(p)
-		}),
-	)
 
 	// We'll also fetch the current block height so we can properly
 	// calculate the required HTLC time locks within the route.
@@ -1813,21 +1868,19 @@ func (r *ChannelRouter) sendPayment(
 	// Now set up a paymentLifecycle struct with these params, such that we
 	// can resume the payment from the current state.
 	p := &paymentLifecycle{
-		router:         r,
+		router:        r,
-		payment:        payment,
+		totalAmount:   totalAmt,
-		paySession:     paySession,
+		feeLimit:      feeLimit,
-		currentHeight:  currentHeight,
+		paymentHash:   paymentHash,
-		finalCLTVDelta: uint16(payment.FinalCLTVDelta),
+		paySession:    paySession,
-		attempt:        existingAttempt,
+		currentHeight: currentHeight,
-		circuit:        nil,
-		lastError:      nil,
 	}
 
 	// If a timeout is specified, create a timeout channel. If no timeout is
 	// specified, the channel is left nil and will never abort the payment
 	// loop.
-	if payment.PayAttemptTimeout != 0 {
+	if timeout != 0 {
-		p.timeoutChan = time.After(payment.PayAttemptTimeout)
+		p.timeoutChan = time.After(timeout)
 	}
 
 	return p.resumePayment()

routing/router_test.go

@@ -2,12 +2,10 @@ package routing
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"image/color"
 	"math"
 	"math/rand"
-	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -23,6 +21,7 @@ import (
 	"github.com/lightningnetwork/lnd/htlcswitch"
 	"github.com/lightningnetwork/lnd/lntypes"
 	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/record"
 	"github.com/lightningnetwork/lnd/routing/route"
 	"github.com/lightningnetwork/lnd/zpay32"
 )
@@ -792,8 +791,30 @@ func TestSendPaymentErrorPathPruning(t *testing.T) {
 
 	// The final error returned should also indicate that the peer wasn't
 	// online (the last error we returned).
-	if !strings.Contains(err.Error(), "UnknownNextPeer") {
+	if err != channeldb.FailureReasonNoRoute {
-		t.Fatalf("expected UnknownNextPeer instead got: %v", err)
+		t.Fatalf("expected no route instead got: %v", err)
+	}
+
+	// Inspect the two attempts that were made before the payment failed.
+	p, err := ctx.router.cfg.Control.FetchPayment(payHash)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(p.HTLCs) != 2 {
+		t.Fatalf("expected two attempts got %v", len(p.HTLCs))
+	}
+
+	// We expect the first attempt to have failed with a
+	// TemporaryChannelFailure, the second with UnknownNextPeer.
+	msg := p.HTLCs[0].Failure.Message
+	if _, ok := msg.(*lnwire.FailTemporaryChannelFailure); !ok {
+		t.Fatalf("unexpected fail message: %T", msg)
+	}
+
+	msg = p.HTLCs[1].Failure.Message
+	if _, ok := msg.(*lnwire.FailUnknownNextPeer); !ok {
+		t.Fatalf("unexpected fail message: %T", msg)
 	}
 
 	ctx.router.cfg.MissionControl.(*MissionControl).ResetHistory()
@@ -2595,639 +2616,6 @@ func assertChannelsPruned(t *testing.T, graph *channeldb.ChannelGraph,
 	}
 }
 
-// TestRouterPaymentStateMachine tests that the router interacts as expected
-// with the ControlTower during a payment lifecycle, such that it payment
-// attempts are not sent twice to the switch, and results are handled after a
-// restart.
-func TestRouterPaymentStateMachine(t *testing.T) {
-	t.Parallel()
-
-	const startingBlockHeight = 101
-
-	// Setup two simple channels such that we can mock sending along this
-	// route.
-	chanCapSat := btcutil.Amount(100000)
-	testChannels := []*testChannel{
-		symmetricTestChannel("a", "b", chanCapSat, &testChannelPolicy{
-			Expiry:  144,
-			FeeRate: 400,
-			MinHTLC: 1,
-			MaxHTLC: lnwire.NewMSatFromSatoshis(chanCapSat),
-		}, 1),
-		symmetricTestChannel("b", "c", chanCapSat, &testChannelPolicy{
-			Expiry:  144,
-			FeeRate: 400,
-			MinHTLC: 1,
-			MaxHTLC: lnwire.NewMSatFromSatoshis(chanCapSat),
-		}, 2),
-	}
-
-	testGraph, err := createTestGraphFromChannels(testChannels, "a")
-	if err != nil {
-		t.Fatalf("unable to create graph: %v", err)
-	}
-	defer testGraph.cleanUp()
-
-	hop1 := testGraph.aliasMap["b"]
-	hop2 := testGraph.aliasMap["c"]
-	hops := []*route.Hop{
-		{
-			ChannelID:     1,
-			PubKeyBytes:   hop1,
-			LegacyPayload: true,
-		},
-		{
-			ChannelID:     2,
-			PubKeyBytes:   hop2,
-			LegacyPayload: true,
-		},
-	}
-
-	// We create a simple route that we will supply every time the router
-	// requests one.
-	rt, err := route.NewRouteFromHops(
-		lnwire.MilliSatoshi(10000), 100, testGraph.aliasMap["a"], hops,
-	)
-	if err != nil {
-		t.Fatalf("unable to create route: %v", err)
-	}
-
-	// A payment state machine test case consists of several ordered steps,
-	// that we use for driving the scenario.
-	type testCase struct {
-		// steps is a list of steps to perform during the testcase.
-		steps []string
-
-		// routes is the sequence of routes we will provide to the
-		// router when it requests a new route.
-		routes []*route.Route
-	}
-
-	const (
-		// routerInitPayment is a test step where we expect the router
-		// to call the InitPayment method on the control tower.
-		routerInitPayment = "Router:init-payment"
-
-		// routerRegisterAttempt is a test step where we expect the
-		// router to call the RegisterAttempt method on the control
-		// tower.
-		routerRegisterAttempt = "Router:register-attempt"
-
-		// routerSuccess is a test step where we expect the router to
-		// call the Success method on the control tower.
-		routerSuccess = "Router:success"
-
-		// routerFail is a test step where we expect the router to call
-		// the Fail method on the control tower.
-		routerFail = "Router:fail"
-
-		// sendToSwitchSuccess is a step where we expect the router to
-		// call send the payment attempt to the switch, and we will
-		// respond with a non-error, indicating that the payment
-		// attempt was successfully forwarded.
-		sendToSwitchSuccess = "SendToSwitch:success"
-
-		// sendToSwitchResultFailure is a step where we expect the
-		// router to send the payment attempt to the switch, and we
-		// will respond with a forwarding error. This can happen when
-		// forwarding fail on our local links.
-		sendToSwitchResultFailure = "SendToSwitch:failure"
-
-		// getPaymentResultSuccess is a test step where we expect the
-		// router to call the GetPaymentResult method, and we will
-		// respond with a successful payment result.
-		getPaymentResultSuccess = "GetPaymentResult:success"
-
-		// getPaymentResultFailure is a test step where we expect the
-		// router to call the GetPaymentResult method, and we will
-		// respond with a forwarding error.
-		getPaymentResultFailure = "GetPaymentResult:failure"
-
-		// resendPayment is a test step where we manually try to resend
-		// the same payment, making sure the router responds with an
-		// error indicating that it is alreayd in flight.
-		resendPayment = "ResendPayment"
-
-		// startRouter is a step where we manually start the router,
-		// used to test that it automatically will resume payments at
-		// startup.
-		startRouter = "StartRouter"
-
-		// stopRouter is a test step where we manually make the router
-		// shut down.
-		stopRouter = "StopRouter"
-
-		// paymentSuccess is a step where assert that we receive a
-		// successful result for the original payment made.
-		paymentSuccess = "PaymentSuccess"
-
-		// paymentError is a step where assert that we receive an error
-		// for the original payment made.
-		paymentError = "PaymentError"
-
-		// resentPaymentSuccess is a step where assert that we receive
-		// a successful result for a payment that was resent.
-		resentPaymentSuccess = "ResentPaymentSuccess"
-
-		// resentPaymentError is a step where assert that we receive an
-		// error for a payment that was resent.
-		resentPaymentError = "ResentPaymentError"
-	)
-
-	tests := []testCase{
-		{
-			// Tests a normal payment flow that succeeds.
-			steps: []string{
-				routerInitPayment,
-				routerRegisterAttempt,
-				sendToSwitchSuccess,
-				getPaymentResultSuccess,
-				routerSuccess,
-				paymentSuccess,
-			},
-			routes: []*route.Route{rt},
-		},
-		{
-			// A payment flow with a failure on the first attempt,
-			// but that succeeds on the second attempt.
-			steps: []string{
-				routerInitPayment,
-				routerRegisterAttempt,
-				sendToSwitchSuccess,
-
-				// Make the first sent attempt fail.
-				getPaymentResultFailure,
-
-				// The router should retry.
-				routerRegisterAttempt,
-				sendToSwitchSuccess,
-
-				// Make the second sent attempt succeed.
-				getPaymentResultSuccess,
-				routerSuccess,
-				paymentSuccess,
-			},
-			routes: []*route.Route{rt, rt},
-		},
-		{
-			// A payment flow with a forwarding failure first time
-			// sending to the switch, but that succeeds on the
-			// second attempt.
-			steps: []string{
-				routerInitPayment,
-				routerRegisterAttempt,
-
-				// Make the first sent attempt fail.
-				sendToSwitchResultFailure,
-
-				// The router should retry.
-				routerRegisterAttempt,
-				sendToSwitchSuccess,
-
-				// Make the second sent attempt succeed.
-				getPaymentResultSuccess,
-				routerSuccess,
-				paymentSuccess,
-			},
-			routes: []*route.Route{rt, rt},
-		},
-		{
-			// A payment that fails on the first attempt, and has
-			// only one route available to try. It will therefore
-			// fail permanently.
-			steps: []string{
-				routerInitPayment,
-				routerRegisterAttempt,
-				sendToSwitchSuccess,
-
-				// Make the first sent attempt fail.
-				getPaymentResultFailure,
-
-				// Since there are no more routes to try, the
-				// payment should fail.
-				routerFail,
-				paymentError,
-			},
-			routes: []*route.Route{rt},
-		},
-		{
-			// We expect the payment to fail immediately if we have
-			// no routes to try.
-			steps: []string{
-				routerInitPayment,
-				routerFail,
-				paymentError,
-			},
-			routes: []*route.Route{},
-		},
-		{
-			// A normal payment flow, where we attempt to resend
-			// the same payment after each step. This ensures that
-			// the router don't attempt to resend a payment already
-			// in flight.
-			steps: []string{
-				routerInitPayment,
-				routerRegisterAttempt,
-
-				// Manually resend the payment, the router
-				// should attempt to init with the control
-				// tower, but fail since it is already in
-				// flight.
-				resendPayment,
-				routerInitPayment,
-				resentPaymentError,
-
-				// The original payment should proceed as
-				// normal.
-				sendToSwitchSuccess,
-
-				// Again resend the payment and assert it's not
-				// allowed.
-				resendPayment,
-				routerInitPayment,
-				resentPaymentError,
-
-				// Notify about a success for the original
-				// payment.
-				getPaymentResultSuccess,
-				routerSuccess,
-
-				// Now that the original payment finished,
-				// resend it again to ensure this is not
-				// allowed.
-				resendPayment,
-				routerInitPayment,
-				resentPaymentError,
-				paymentSuccess,
-			},
-			routes: []*route.Route{rt},
-		},
-		{
-			// Tests that the router is able to handle the
-			// receieved payment result after a restart.
-			steps: []string{
-				routerInitPayment,
-				routerRegisterAttempt,
-				sendToSwitchSuccess,
-
-				// Shut down the router. The original caller
-				// should get notified about this.
-				stopRouter,
-				paymentError,
-
-				// Start the router again, and ensure the
-				// router registers the success with the
-				// control tower.
-				startRouter,
-				getPaymentResultSuccess,
-				routerSuccess,
-			},
-			routes: []*route.Route{rt},
-		},
-		{
-			// Tests that we are allowed to resend a payment after
-			// it has permanently failed.
-			steps: []string{
-				routerInitPayment,
-				routerRegisterAttempt,
-				sendToSwitchSuccess,
-
-				// Resending the payment at this stage should
-				// not be allowed.
-				resendPayment,
-				routerInitPayment,
-				resentPaymentError,
-
-				// Make the first attempt fail.
-				getPaymentResultFailure,
-				routerFail,
-
-				// Since we have no more routes to try, the
-				// original payment should fail.
-				paymentError,
-
-				// Now resend the payment again. This should be
-				// allowed, since the payment has failed.
-				resendPayment,
-				routerInitPayment,
-				routerRegisterAttempt,
-				sendToSwitchSuccess,
-				getPaymentResultSuccess,
-				routerSuccess,
-				resentPaymentSuccess,
-			},
-			routes: []*route.Route{rt},
-		},
-	}
-
-	// Create a mock control tower with channels set up, that we use to
-	// synchronize and listen for events.
-	control := makeMockControlTower()
-	control.init = make(chan initArgs)
-	control.register = make(chan registerArgs)
-	control.success = make(chan successArgs)
-	control.fail = make(chan failArgs)
-	control.fetchInFlight = make(chan struct{})
-
-	quit := make(chan struct{})
-	defer close(quit)
-
-	// setupRouter is a helper method that creates and starts the router in
-	// the desired configuration for this test.
-	setupRouter := func() (*ChannelRouter, chan error,
-		chan *htlcswitch.PaymentResult, chan error) {
-
-		chain := newMockChain(startingBlockHeight)
-		chainView := newMockChainView(chain)
-
-		// We set uo the use the following channels and a mock Payer to
-		// synchonize with the interaction to the Switch.
-		sendResult := make(chan error)
-		paymentResultErr := make(chan error)
-		paymentResult := make(chan *htlcswitch.PaymentResult)
-
-		payer := &mockPayer{
-			sendResult:       sendResult,
-			paymentResult:    paymentResult,
-			paymentResultErr: paymentResultErr,
-		}
-
-		router, err := New(Config{
-			Graph:              testGraph.graph,
-			Chain:              chain,
-			ChainView:          chainView,
-			Control:            control,
-			SessionSource:      &mockPaymentSessionSource{},
-			MissionControl:     &mockMissionControl{},
-			Payer:              payer,
-			ChannelPruneExpiry: time.Hour * 24,
-			GraphPruneInterval: time.Hour * 2,
-			QueryBandwidth: func(e *channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi {
-				return lnwire.NewMSatFromSatoshis(e.Capacity)
-			},
-			NextPaymentID: func() (uint64, error) {
-				next := atomic.AddUint64(&uniquePaymentID, 1)
-				return next, nil
-			},
-			Clock: clock.NewTestClock(time.Unix(1, 0)),
-		})
-		if err != nil {
-			t.Fatalf("unable to create router %v", err)
-		}
-
-		// On startup, the router should fetch all pending payments
-		// from the ControlTower, so assert that here.
-		errCh := make(chan error)
-		go func() {
-			close(errCh)
-			select {
-			case <-control.fetchInFlight:
-				return
-			case <-time.After(1 * time.Second):
-				errCh <- errors.New("router did not fetch in flight " +
-					"payments")
-			}
-		}()
-
-		if err := router.Start(); err != nil {
-			t.Fatalf("unable to start router: %v", err)
-		}
-
-		select {
-		case err := <-errCh:
-			if err != nil {
-				t.Fatalf("error in anonymous goroutine: %s", err)
-			}
-		case <-time.After(1 * time.Second):
-			t.Fatalf("did not fetch in flight payments at startup")
-		}
-
-		return router, sendResult, paymentResult, paymentResultErr
-	}
-
-	router, sendResult, getPaymentResult, getPaymentResultErr := setupRouter()
-	defer router.Stop()
-
-	for _, test := range tests {
-		// Craft a LightningPayment struct.
-		var preImage lntypes.Preimage
-		if _, err := rand.Read(preImage[:]); err != nil {
-			t.Fatalf("unable to generate preimage")
-		}
-
-		payHash := preImage.Hash()
-
-		paymentAmt := lnwire.NewMSatFromSatoshis(1000)
-		payment := LightningPayment{
-			Target:      testGraph.aliasMap["c"],
-			Amount:      paymentAmt,
-			FeeLimit:    noFeeLimit,
-			PaymentHash: payHash,
-		}
-
-		copy(preImage[:], bytes.Repeat([]byte{9}, 32))
-
-		router.cfg.SessionSource = &mockPaymentSessionSource{
-			routes: test.routes,
-		}
-
-		router.cfg.MissionControl = &mockMissionControl{}
-
-		// Send the payment. Since this is new payment hash, the
-		// information should be registered with the ControlTower.
-		paymentResult := make(chan error)
-		go func() {
-			_, _, err := router.SendPayment(&payment)
-			paymentResult <- err
-		}()
-
-		var resendResult chan error
-		for _, step := range test.steps {
-			switch step {
-
-			case routerInitPayment:
-				var args initArgs
-				select {
-				case args = <-control.init:
-				case <-time.After(1 * time.Second):
-					t.Fatalf("no init payment with control")
-				}
-
-				if args.c == nil {
-					t.Fatalf("expected non-nil CreationInfo")
-				}
-
-			// In this step we expect the router to make a call to
-			// register a new attempt with the ControlTower.
-			case routerRegisterAttempt:
-				var args registerArgs
-				select {
-				case args = <-control.register:
-				case <-time.After(1 * time.Second):
-					t.Fatalf("not registered with control")
-				}
-
-				if args.a == nil {
-					t.Fatalf("expected non-nil AttemptInfo")
-				}
-
-			// In this step we expect the router to call the
-			// ControlTower's Succcess method with the preimage.
-			case routerSuccess:
-				select {
-				case _ = <-control.success:
-				case <-time.After(1 * time.Second):
-					t.Fatalf("not registered with control")
-				}
-
-			// In this step we expect the router to call the
-			// ControlTower's Fail method, to indicate that the
-			// payment failed.
-			case routerFail:
-				select {
-				case _ = <-control.fail:
-				case <-time.After(1 * time.Second):
-					t.Fatalf("not registered with control")
-				}
-
-			// In this step we expect the SendToSwitch method to be
-			// called, and we respond with a nil-error.
-			case sendToSwitchSuccess:
-				select {
-				case sendResult <- nil:
-				case <-time.After(1 * time.Second):
-					t.Fatalf("unable to send result")
-				}
-
-			// In this step we expect the SendToSwitch method to be
-			// called, and we respond with a forwarding error
-			case sendToSwitchResultFailure:
-				select {
-				case sendResult <- htlcswitch.NewForwardingError(
-					&lnwire.FailTemporaryChannelFailure{},
-					1,
-				):
-				case <-time.After(1 * time.Second):
-					t.Fatalf("unable to send result")
-				}
-
-			// In this step we expect the GetPaymentResult method
-			// to be called, and we respond with the preimage to
-			// complete the payment.
-			case getPaymentResultSuccess:
-				select {
-				case getPaymentResult <- &htlcswitch.PaymentResult{
-					Preimage: preImage,
-				}:
-				case <-time.After(1 * time.Second):
-					t.Fatalf("unable to send result")
-				}
-
-			// In this state we expect the GetPaymentResult method
-			// to be called, and we respond with a forwarding
-			// error, indicating that the router should retry.
-			case getPaymentResultFailure:
-				failure := htlcswitch.NewForwardingError(
-					&lnwire.FailTemporaryChannelFailure{},
-					1,
-				)
-
-				select {
-				case getPaymentResult <- &htlcswitch.PaymentResult{
-					Error: failure,
-				}:
-				case <-time.After(1 * time.Second):
-					t.Fatalf("unable to get result")
-				}
-
-			// In this step we manually try to resend the same
-			// payment, making sure the router responds with an
-			// error indicating that it is alreayd in flight.
-			case resendPayment:
-				resendResult = make(chan error)
-				go func() {
-					_, _, err := router.SendPayment(&payment)
-					resendResult <- err
-				}()
-
-			// In this step we manually stop the router.
-			case stopRouter:
-				select {
-				case getPaymentResultErr <- fmt.Errorf(
-					"shutting down"):
-				case <-time.After(1 * time.Second):
-					t.Fatalf("unable to send payment " +
-						"result error")
-				}
-
-				if err := router.Stop(); err != nil {
-					t.Fatalf("unable to restart: %v", err)
-				}
-
-			// In this step we manually start the router.
-			case startRouter:
-				router, sendResult, getPaymentResult,
-					getPaymentResultErr = setupRouter()
-
-			// In this state we expect to receive an error for the
-			// original payment made.
-			case paymentError:
-				select {
-				case err := <-paymentResult:
-					if err == nil {
-						t.Fatalf("expected error")
-					}
-
-				case <-time.After(1 * time.Second):
-					t.Fatalf("got no payment result")
-				}
-
-			// In this state we expect the original payment to
-			// succeed.
-			case paymentSuccess:
-				select {
-				case err := <-paymentResult:
-					if err != nil {
-						t.Fatalf("did not expecte error %v", err)
-					}
-
-				case <-time.After(1 * time.Second):
-					t.Fatalf("got no payment result")
-				}
-
-			// In this state we expect to receive an error for the
-			// resent payment made.
-			case resentPaymentError:
-				select {
-				case err := <-resendResult:
-					if err == nil {
-						t.Fatalf("expected error")
-					}
-
-				case <-time.After(1 * time.Second):
-					t.Fatalf("got no payment result")
-				}
-
-			// In this state we expect the resent payment to
-			// succeed.
-			case resentPaymentSuccess:
-				select {
-				case err := <-resendResult:
-					if err != nil {
-						t.Fatalf("did not expect error %v", err)
-					}
-
-				case <-time.After(1 * time.Second):
-					t.Fatalf("got no payment result")
-				}
-
-			default:
-				t.Fatalf("unknown step %v", step)
-			}
-		}
-	}
-}
-
 // TestSendToRouteStructuredError asserts that SendToRoute returns a structured
 // error.
 func TestSendToRouteStructuredError(t *testing.T) {
@@ -3338,6 +2726,138 @@ func TestSendToRouteStructuredError(t *testing.T) {
 	}
 }
 
+// TestSendToRouteMultiShardSend checks that a 3-shard payment can be executed
+// using SendToRoute.
+func TestSendToRouteMultiShardSend(t *testing.T) {
+	t.Parallel()
+
+	ctx, cleanup, err := createTestCtxSingleNode(0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+
+	const numShards = 3
+	const payAmt = lnwire.MilliSatoshi(numShards * 10000)
+	node, err := createTestNode()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a simple 1-hop route that we will use for all three shards.
+	hops := []*route.Hop{
+		{
+			ChannelID:    1,
+			PubKeyBytes:  node.PubKeyBytes,
+			AmtToForward: payAmt / numShards,
+			MPP:          record.NewMPP(payAmt, [32]byte{}),
+		},
+	}
+
+	sourceNode, err := ctx.graph.SourceNode()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rt, err := route.NewRouteFromHops(
+		payAmt, 100, sourceNode.PubKeyBytes, hops,
+	)
+	if err != nil {
+		t.Fatalf("unable to create route: %v", err)
+	}
+
+	// The first shard we send we'll fail immediately, to check that we are
+	// still allowed to retry with other shards after a failed one.
+	ctx.router.cfg.Payer.(*mockPaymentAttemptDispatcher).setPaymentResult(
+		func(firstHop lnwire.ShortChannelID) ([32]byte, error) {
+			return [32]byte{}, htlcswitch.NewForwardingError(
+				&lnwire.FailFeeInsufficient{
+					Update: lnwire.ChannelUpdate{},
+				}, 1,
+			)
+		})
+
+	// The payment parameter is mostly redundant in SendToRoute. Can be left
+	// empty for this test.
+	var payment lntypes.Hash
+
+	// Send the shard using the created route, and expect an error to be
+	// returned.
+	_, err = ctx.router.SendToRoute(payment, rt)
+	if err == nil {
+		t.Fatalf("expected forwarding error")
+	}
+
+	// Now we'll modify the SendToSwitch method again to wait until all
+	// three shards are initiated before returning a result. We do this by
+	// signalling when the method has been called, and then stop to wait
+	// for the test to deliver the final result on the channel below.
+	waitForResultSignal := make(chan struct{}, numShards)
+	results := make(chan lntypes.Preimage, numShards)
+
+	ctx.router.cfg.Payer.(*mockPaymentAttemptDispatcher).setPaymentResult(
+		func(firstHop lnwire.ShortChannelID) ([32]byte, error) {
+
+			// Signal that the shard has been initiated and is
+			// waiting for a result.
+			waitForResultSignal <- struct{}{}
+
+			// Wait for a result before returning it.
+			res, ok := <-results
+			if !ok {
+				return [32]byte{}, fmt.Errorf("failure")
+			}
+			return res, nil
+		})
+
+	// Launch three shards by calling SendToRoute in three goroutines,
+	// returning their final error on the channel.
+	errChan := make(chan error)
+	successes := make(chan lntypes.Preimage)
+
+	for i := 0; i < numShards; i++ {
+		go func() {
+			preimg, err := ctx.router.SendToRoute(payment, rt)
+			if err != nil {
+				errChan <- err
+				return
+			}
+
+			successes <- preimg
+		}()
+	}
+
+	// Wait for all shards to signal they have been initiated.
+	for i := 0; i < numShards; i++ {
+		select {
+		case <-waitForResultSignal:
+		case <-time.After(5 * time.Second):
+			t.Fatalf("not waiting for results")
+		}
+	}
+
+	// Deliver a dummy preimage to all the shard handlers.
+	preimage := lntypes.Preimage{}
+	preimage[4] = 42
+	for i := 0; i < numShards; i++ {
+		results <- preimage
+	}
+
+	// Finally expect all shards to return with the above preimage.
+	for i := 0; i < numShards; i++ {
+		select {
+		case p := <-successes:
+			if p != preimage {
+				t.Fatalf("preimage mismatch")
+			}
+		case err := <-errChan:
+			t.Fatalf("unexpected error from SendToRoute: %v", err)
+		case <-time.After(5 * time.Second):
+			t.Fatalf("result not received")
+		}
+	}
+}
+
 // TestSendToRouteMaxHops asserts that SendToRoute fails when using a route that
 // exceeds the maximum number of hops.
 func TestSendToRouteMaxHops(t *testing.T) {