diff --git a/channeldb/mp_payment.go b/channeldb/mp_payment.go index d46e59091..e359ff03d 100644 --- a/channeldb/mp_payment.go +++ b/channeldb/mp_payment.go @@ -131,6 +131,52 @@ type MPPayment struct { Status PaymentStatus } +// TerminalInfo returns any HTLC settle info recorded. If no settle info is +// recorded, any payment level failure will be returned. If neither a settle +// nor a failure is recorded, both return values will be nil. +func (m *MPPayment) TerminalInfo() (*HTLCSettleInfo, *FailureReason) { + for _, h := range m.HTLCs { + if h.Settle != nil { + return h.Settle, nil + } + } + + return nil, m.FailureReason +} + +// SentAmt returns the sum of sent amount and fees for HTLCs that are either +// settled or still in flight. +func (m *MPPayment) SentAmt() (lnwire.MilliSatoshi, lnwire.MilliSatoshi) { + var sent, fees lnwire.MilliSatoshi + for _, h := range m.HTLCs { + if h.Failure != nil { + continue + } + + // The attempt was not failed, meaning the amount was + // potentially sent to the receiver. + sent += h.Route.ReceiverAmt() + fees += h.Route.TotalFees() + } + + return sent, fees +} + +// InFlightHTLCs returns the HTLCs that are still in-flight, meaning they have +// not been settled or failed. +func (m *MPPayment) InFlightHTLCs() []HTLCAttempt { + var inflights []HTLCAttempt + for _, h := range m.HTLCs { + if h.Settle != nil || h.Failure != nil { + continue + } + + inflights = append(inflights, h) + } + + return inflights +} + // serializeHTLCSettleInfo serializes the details of a settled htlc. func serializeHTLCSettleInfo(w io.Writer, s *HTLCSettleInfo) error { if _, err := w.Write(s.Preimage[:]); err != nil { diff --git a/channeldb/payment_control.go b/channeldb/payment_control.go index 64d7a3919..ca5b6998b 100644 --- a/channeldb/payment_control.go +++ b/channeldb/payment_control.go @@ -18,22 +18,59 @@ var ( // already "in flight" on the network. ErrPaymentInFlight = errors.New("payment is in transition") - // ErrPaymentNotInitiated is returned if payment wasn't initiated in - // switch. + // ErrPaymentNotInitiated is returned if the payment wasn't initiated. ErrPaymentNotInitiated = errors.New("payment isn't initiated") // ErrPaymentAlreadySucceeded is returned in the event we attempt to // change the status of a payment already succeeded. ErrPaymentAlreadySucceeded = errors.New("payment is already succeeded") - // ErrPaymentAlreadyFailed is returned in the event we attempt to - // re-fail a failed payment. + // ErrPaymentAlreadyFailed is returned in the event we attempt to alter + // a failed payment. ErrPaymentAlreadyFailed = errors.New("payment has already failed") // ErrUnknownPaymentStatus is returned when we do not recognize the // existing state of a payment. ErrUnknownPaymentStatus = errors.New("unknown payment status") + // ErrPaymentTerminal is returned if we attempt to alter a payment that + // already has reached a terminal condition. + ErrPaymentTerminal = errors.New("payment has reached terminal condition") + + // ErrAttemptAlreadySettled is returned if we try to alter an already + // settled HTLC attempt. + ErrAttemptAlreadySettled = errors.New("attempt already settled") + + // ErrAttemptAlreadyFailed is returned if we try to alter an already + // failed HTLC attempt. + ErrAttemptAlreadyFailed = errors.New("attempt already failed") + + // ErrValueMismatch is returned if we try to register a non-MPP attempt + // with an amount that doesn't match the payment amount. + ErrValueMismatch = errors.New("attempted value doesn't match payment" + + "amount") + + // ErrValueExceedsAmt is returned if we try to register an attempt that + // would take the total sent amount above the payment amount. + ErrValueExceedsAmt = errors.New("attempted value exceeds payment" + + "amount") + + // ErrNonMPPayment is returned if we try to register an MPP attempt for + // a payment that already has a non-MPP attempt regitered. + ErrNonMPPayment = errors.New("payment has non-MPP attempts") + + // ErrMPPayment is returned if we try to register a non-MPP attempt for + // a payment that already has an MPP attempt regitered. + ErrMPPayment = errors.New("payment has MPP attempts") + + // ErrMPPPaymentAddrMismatch is returned if we try to register an MPP + // shard where the payment address doesn't match existing shards. + ErrMPPPaymentAddrMismatch = errors.New("payment address mismatch") + + // ErrMPPTotalAmountMismatch is returned if we try to register an MPP + // shard where the total amount doesn't match existing shards. + ErrMPPTotalAmountMismatch = errors.New("mp payment total amount mismatch") + // errNoAttemptInfo is returned when no attempt info is stored yet. errNoAttemptInfo = errors.New("unable to find attempt info for " + "inflight payment") @@ -52,7 +89,7 @@ func NewPaymentControl(db *DB) *PaymentControl { } // InitPayment checks or records the given PaymentCreationInfo with the DB, -// making sure it does not already exist as an in-flight payment. Then this +// making sure it does not already exist as an in-flight payment. When this // method returns successfully, the payment is guranteeed to be in the InFlight // state. func (p *PaymentControl) InitPayment(paymentHash lntypes.Hash, @@ -168,12 +205,69 @@ func (p *PaymentControl) RegisterAttempt(paymentHash lntypes.Hash, return err } - // We can only register attempts for payments that are - // in-flight. - if err := ensureInFlight(bucket); err != nil { + payment, err := fetchPayment(bucket) + if err != nil { return err } + // Ensure the payment is in-flight. + if err := ensureInFlight(payment); err != nil { + return err + } + + // We cannot register a new attempt if the payment already has + // reached a terminal condition: + settle, fail := payment.TerminalInfo() + if settle != nil || fail != nil { + return ErrPaymentTerminal + } + + // Make sure any existing shards match the new one with regards + // to MPP options. + mpp := attempt.Route.FinalHop().MPP + for _, h := range payment.InFlightHTLCs() { + hMpp := h.Route.FinalHop().MPP + + switch { + + // We tried to register a non-MPP attempt for a MPP + // payment. + case mpp == nil && hMpp != nil: + return ErrMPPayment + + // We tried to register a MPP shard for a non-MPP + // payment. + case mpp != nil && hMpp == nil: + return ErrNonMPPayment + + // Non-MPP payment, nothing more to validate. + case mpp == nil: + continue + } + + // Check that MPP options match. + if mpp.PaymentAddr() != hMpp.PaymentAddr() { + return ErrMPPPaymentAddrMismatch + } + + if mpp.TotalMsat() != hMpp.TotalMsat() { + return ErrMPPTotalAmountMismatch + } + } + + // If this is a non-MPP attempt, it must match the total amount + // exactly. + amt := attempt.Route.ReceiverAmt() + if mpp == nil && amt != payment.Info.Value { + return ErrValueMismatch + } + + // Ensure we aren't sending more than the total payment amount. + sentAmt, _ := payment.SentAmt() + if sentAmt+amt > payment.Info.Value { + return ErrValueExceedsAmt + } + htlcsBucket, err := bucket.CreateBucketIfNotExists( paymentHtlcsBucket, ) @@ -241,8 +335,15 @@ func (p *PaymentControl) updateHtlcKey(paymentHash lntypes.Hash, return err } - // We can only update keys of in-flight payments. - if err := ensureInFlight(bucket); err != nil { + p, err := fetchPayment(bucket) + if err != nil { + return err + } + + // We can only update keys of in-flight payments. We allow + // updating keys even if the payment has reached a terminal + // condition, since the HTLC outcomes must still be updated. + if err := ensureInFlight(p); err != nil { return err } @@ -257,6 +358,15 @@ func (p *PaymentControl) updateHtlcKey(paymentHash lntypes.Hash, attemptID) } + // Make sure the shard is not already failed or settled. + if htlcBucket.Get(htlcFailInfoKey) != nil { + return ErrAttemptAlreadyFailed + } + + if htlcBucket.Get(htlcSettleInfoKey) != nil { + return ErrAttemptAlreadySettled + } + // Add or update the key for this htlc. err = htlcBucket.Put(key, value) if err != nil { @@ -299,9 +409,17 @@ func (p *PaymentControl) Fail(paymentHash lntypes.Hash, return err } - // We can only mark in-flight payments as failed. - if err := ensureInFlight(bucket); err != nil { - updateErr = err + // We mark the payent as failed as long as it is known. This + // lets the last attempt to fail with a terminal write its + // failure to the PaymentControl without synchronizing with + // other attempts. + paymentStatus, err := fetchPaymentStatus(bucket) + if err != nil { + return err + } + + if paymentStatus == StatusUnknown { + updateErr = ErrPaymentNotInitiated return nil } @@ -318,14 +436,6 @@ func (p *PaymentControl) Fail(paymentHash lntypes.Hash, return err } - // Final sanity check to see if there are no in-flight htlcs. - for _, htlc := range payment.HTLCs { - if htlc.Settle == nil && htlc.Failure == nil { - return errors.New("payment failed with " + - "in-flight htlc(s)") - } - } - return nil }) if err != nil { @@ -428,45 +538,29 @@ func nextPaymentSequence(tx kvdb.RwTx) ([]byte, error) { // fetchPaymentStatus fetches the payment status of the payment. If the payment // isn't found, it will default to "StatusUnknown". func fetchPaymentStatus(bucket kvdb.ReadBucket) (PaymentStatus, error) { - htlcsBucket := bucket.NestedReadBucket(paymentHtlcsBucket) - if htlcsBucket != nil { - htlcs, err := fetchHtlcAttempts(htlcsBucket) - if err != nil { - return 0, err - } - - // Go through all HTLCs, and return StatusSucceeded if any of - // them did succeed. - for _, h := range htlcs { - if h.Settle != nil { - return StatusSucceeded, nil - } - } + // Creation info should be set for all payments, regardless of state. + // If not, it is unknown. + if bucket.Get(paymentCreationInfoKey) == nil { + return StatusUnknown, nil } - if bucket.Get(paymentFailInfoKey) != nil { - return StatusFailed, nil + payment, err := fetchPayment(bucket) + if err != nil { + return 0, err } - if bucket.Get(paymentCreationInfoKey) != nil { - return StatusInFlight, nil - } - - return StatusUnknown, nil + return payment.Status, nil } // ensureInFlight checks whether the payment found in the given bucket has // status InFlight, and returns an error otherwise. This should be used to // ensure we only mark in-flight payments as succeeded or failed. -func ensureInFlight(bucket kvdb.ReadBucket) error { - paymentStatus, err := fetchPaymentStatus(bucket) - if err != nil { - return err - } +func ensureInFlight(payment *MPPayment) error { + paymentStatus := payment.Status switch { - // The payment was indeed InFlight, return. + // The payment was indeed InFlight. case paymentStatus == StatusInFlight: return nil @@ -528,14 +622,7 @@ func (p *PaymentControl) FetchInFlightPayments() ([]*InFlightPayment, error) { inFlight := &InFlightPayment{} // Get the CreationInfo. - b := bucket.Get(paymentCreationInfoKey) - if b == nil { - return fmt.Errorf("unable to find creation " + - "info for inflight payment") - } - - r := bytes.NewReader(b) - inFlight.Info, err = deserializePaymentCreationInfo(r) + inFlight.Info, err = fetchCreationInfo(bucket) if err != nil { return err } diff --git a/channeldb/payment_control_test.go b/channeldb/payment_control_test.go index 272de68c9..abd2722a5 100644 --- a/channeldb/payment_control_test.go +++ b/channeldb/payment_control_test.go @@ -12,6 +12,7 @@ import ( "github.com/btcsuite/fastsha256" "github.com/davecgh/go-spew/spew" "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/record" ) func initDB() (*DB, error) { @@ -48,14 +49,14 @@ func genInfo() (*PaymentCreationInfo, *HTLCAttemptInfo, rhash := fastsha256.Sum256(preimage[:]) return &PaymentCreationInfo{ PaymentHash: rhash, - Value: 1, + Value: testRoute.ReceiverAmt(), CreationTime: time.Unix(time.Now().Unix(), 0), PaymentRequest: []byte("hola"), }, &HTLCAttemptInfo{ - AttemptID: 1, + AttemptID: 0, SessionKey: priv, - Route: testRoute, + Route: *testRoute.Copy(), }, preimage, nil } @@ -85,8 +86,7 @@ func TestPaymentControlSwitchFail(t *testing.T) { assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) assertPaymentInfo( - t, pControl, info.PaymentHash, info, 0, nil, lntypes.Preimage{}, - nil, + t, pControl, info.PaymentHash, info, nil, nil, ) // Fail the payment, which should moved it to Failed. @@ -99,8 +99,7 @@ func TestPaymentControlSwitchFail(t *testing.T) { // Verify the status is indeed Failed. assertPaymentStatus(t, pControl, info.PaymentHash, StatusFailed) assertPaymentInfo( - t, pControl, info.PaymentHash, info, 0, nil, lntypes.Preimage{}, - &failReason, + t, pControl, info.PaymentHash, info, &failReason, nil, ) // Sends the htlc again, which should succeed since the prior payment @@ -112,44 +111,57 @@ func TestPaymentControlSwitchFail(t *testing.T) { assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) assertPaymentInfo( - t, pControl, info.PaymentHash, info, 0, nil, lntypes.Preimage{}, - nil, + t, pControl, info.PaymentHash, info, nil, nil, ) // Record a new attempt. In this test scenario, the attempt fails. // However, this is not communicated to control tower in the current // implementation. It only registers the initiation of the attempt. - attempt.AttemptID = 2 err = pControl.RegisterAttempt(info.PaymentHash, attempt) if err != nil { t.Fatalf("unable to register attempt: %v", err) } + htlcReason := HTLCFailUnreadable err = pControl.FailAttempt( - info.PaymentHash, 2, &HTLCFailInfo{ - Reason: HTLCFailUnreadable, + info.PaymentHash, attempt.AttemptID, + &HTLCFailInfo{ + Reason: htlcReason, }, ) if err != nil { t.Fatal(err) } + assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) + + htlc := &htlcStatus{ + HTLCAttemptInfo: attempt, + failure: &htlcReason, + } + + assertPaymentInfo(t, pControl, info.PaymentHash, info, nil, htlc) // Record another attempt. - attempt.AttemptID = 3 + attempt.AttemptID = 1 err = pControl.RegisterAttempt(info.PaymentHash, attempt) if err != nil { t.Fatalf("unable to send htlc message: %v", err) } assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) + + htlc = &htlcStatus{ + HTLCAttemptInfo: attempt, + } + assertPaymentInfo( - t, pControl, info.PaymentHash, info, 0, attempt, lntypes.Preimage{}, - nil, + t, pControl, info.PaymentHash, info, nil, htlc, ) - // Settle the attempt and verify that status was changed to StatusSucceeded. + // Settle the attempt and verify that status was changed to + // StatusSucceeded. var payment *MPPayment payment, err = pControl.SettleAttempt( - info.PaymentHash, 3, + info.PaymentHash, attempt.AttemptID, &HTLCSettleInfo{ Preimage: preimg, }, @@ -171,7 +183,11 @@ func TestPaymentControlSwitchFail(t *testing.T) { } assertPaymentStatus(t, pControl, info.PaymentHash, StatusSucceeded) - assertPaymentInfo(t, pControl, info.PaymentHash, info, 1, attempt, preimg, nil) + + htlc.settle = &preimg + assertPaymentInfo( + t, pControl, info.PaymentHash, info, nil, htlc, + ) // Attempt a final payment, which should now fail since the prior // payment succeed. @@ -207,8 +223,7 @@ func TestPaymentControlSwitchDoubleSend(t *testing.T) { assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) assertPaymentInfo( - t, pControl, info.PaymentHash, info, 0, nil, lntypes.Preimage{}, - nil, + t, pControl, info.PaymentHash, info, nil, nil, ) // Try to initiate double sending of htlc message with the same @@ -226,9 +241,12 @@ func TestPaymentControlSwitchDoubleSend(t *testing.T) { t.Fatalf("unable to send htlc message: %v", err) } assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) + + htlc := &htlcStatus{ + HTLCAttemptInfo: attempt, + } assertPaymentInfo( - t, pControl, info.PaymentHash, info, 0, attempt, lntypes.Preimage{}, - nil, + t, pControl, info.PaymentHash, info, nil, htlc, ) // Sends base htlc message which initiate StatusInFlight. @@ -240,7 +258,7 @@ func TestPaymentControlSwitchDoubleSend(t *testing.T) { // After settling, the error should be ErrAlreadyPaid. _, err = pControl.SettleAttempt( - info.PaymentHash, 1, + info.PaymentHash, attempt.AttemptID, &HTLCSettleInfo{ Preimage: preimg, }, @@ -249,7 +267,9 @@ func TestPaymentControlSwitchDoubleSend(t *testing.T) { t.Fatalf("error shouldn't have been received, got: %v", err) } assertPaymentStatus(t, pControl, info.PaymentHash, StatusSucceeded) - assertPaymentInfo(t, pControl, info.PaymentHash, info, 0, attempt, preimg, nil) + + htlc.settle = &preimg + assertPaymentInfo(t, pControl, info.PaymentHash, info, nil, htlc) err = pControl.InitPayment(info.PaymentHash, info) if err != ErrAlreadyPaid { @@ -360,12 +380,17 @@ func TestPaymentControlDeleteNonInFligt(t *testing.T) { t.Fatalf("unable to send htlc message: %v", err) } + htlc := &htlcStatus{ + HTLCAttemptInfo: attempt, + } + if p.failed { // Fail the payment attempt. + htlcFailure := HTLCFailUnreadable err := pControl.FailAttempt( info.PaymentHash, attempt.AttemptID, &HTLCFailInfo{ - Reason: HTLCFailUnreadable, + Reason: htlcFailure, }, ) if err != nil { @@ -381,14 +406,16 @@ func TestPaymentControlDeleteNonInFligt(t *testing.T) { // Verify the status is indeed Failed. assertPaymentStatus(t, pControl, info.PaymentHash, StatusFailed) + + htlc.failure = &htlcFailure assertPaymentInfo( - t, pControl, info.PaymentHash, info, 0, attempt, - lntypes.Preimage{}, &failReason, + t, pControl, info.PaymentHash, info, + &failReason, htlc, ) } else if p.success { // Verifies that status was changed to StatusSucceeded. _, err := pControl.SettleAttempt( - info.PaymentHash, 1, + info.PaymentHash, attempt.AttemptID, &HTLCSettleInfo{ Preimage: preimg, }, @@ -398,14 +425,15 @@ func TestPaymentControlDeleteNonInFligt(t *testing.T) { } assertPaymentStatus(t, pControl, info.PaymentHash, StatusSucceeded) + + htlc.settle = &preimg assertPaymentInfo( - t, pControl, info.PaymentHash, info, 0, attempt, preimg, nil, + t, pControl, info.PaymentHash, info, nil, htlc, ) } else { assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) assertPaymentInfo( - t, pControl, info.PaymentHash, info, 0, attempt, - lntypes.Preimage{}, nil, + t, pControl, info.PaymentHash, info, nil, htlc, ) } } @@ -431,6 +459,366 @@ func TestPaymentControlDeleteNonInFligt(t *testing.T) { } } +// TestPaymentControlMultiShard checks the ability of payment control to +// have multiple in-flight HTLCs for a single payment. +func TestPaymentControlMultiShard(t *testing.T) { + t.Parallel() + + // We will register three HTLC attempts, and always fail the second + // one. We'll generate all combinations of settling/failing the first + // and third HTLC, and assert that the payment status end up as we + // expect. + type testCase struct { + settleFirst bool + settleLast bool + } + + var tests []testCase + for _, f := range []bool{true, false} { + for _, l := range []bool{true, false} { + tests = append(tests, testCase{f, l}) + } + } + + runSubTest := func(t *testing.T, test testCase) { + db, err := initDB() + if err != nil { + t.Fatalf("unable to init db: %v", err) + } + + pControl := NewPaymentControl(db) + + info, attempt, preimg, err := genInfo() + if err != nil { + t.Fatalf("unable to generate htlc message: %v", err) + } + + // Init the payment, moving it to the StatusInFlight state. + err = pControl.InitPayment(info.PaymentHash, info) + if err != nil { + t.Fatalf("unable to send htlc message: %v", err) + } + + assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) + assertPaymentInfo( + t, pControl, info.PaymentHash, info, nil, nil, + ) + + // Create three unique attempts we'll use for the test, and + // register them with the payment control. We set each + // attempts's value to one third of the payment amount, and + // populate the MPP options. + shardAmt := info.Value / 3 + attempt.Route.FinalHop().AmtToForward = shardAmt + attempt.Route.FinalHop().MPP = record.NewMPP( + info.Value, [32]byte{1}, + ) + + var attempts []*HTLCAttemptInfo + for i := uint64(0); i < 3; i++ { + a := *attempt + a.AttemptID = i + attempts = append(attempts, &a) + + err = pControl.RegisterAttempt(info.PaymentHash, &a) + if err != nil { + t.Fatalf("unable to send htlc message: %v", err) + } + assertPaymentStatus( + t, pControl, info.PaymentHash, StatusInFlight, + ) + + htlc := &htlcStatus{ + HTLCAttemptInfo: &a, + } + assertPaymentInfo( + t, pControl, info.PaymentHash, info, nil, htlc, + ) + } + + // For a fourth attempt, check that attempting to + // register it will fail since the total sent amount + // will be too large. + b := *attempt + b.AttemptID = 3 + err = pControl.RegisterAttempt(info.PaymentHash, &b) + if err != ErrValueExceedsAmt { + t.Fatalf("expected ErrValueExceedsAmt, got: %v", + err) + } + + // Fail the second attempt. + a := attempts[1] + htlcFail := HTLCFailUnreadable + err = pControl.FailAttempt( + info.PaymentHash, a.AttemptID, + &HTLCFailInfo{ + Reason: htlcFail, + }, + ) + if err != nil { + t.Fatal(err) + } + + htlc := &htlcStatus{ + HTLCAttemptInfo: a, + failure: &htlcFail, + } + assertPaymentInfo( + t, pControl, info.PaymentHash, info, nil, htlc, + ) + + // Payment should still be in-flight. + assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) + + // Depending on the test case, settle or fail the first attempt. + a = attempts[0] + htlc = &htlcStatus{ + HTLCAttemptInfo: a, + } + + var firstFailReason *FailureReason + if test.settleFirst { + _, err := pControl.SettleAttempt( + info.PaymentHash, a.AttemptID, + &HTLCSettleInfo{ + Preimage: preimg, + }, + ) + if err != nil { + t.Fatalf("error shouldn't have been "+ + "received, got: %v", err) + } + + // Assert that the HTLC has had the preimage recorded. + htlc.settle = &preimg + assertPaymentInfo( + t, pControl, info.PaymentHash, info, nil, htlc, + ) + } else { + err := pControl.FailAttempt( + info.PaymentHash, a.AttemptID, + &HTLCFailInfo{ + Reason: htlcFail, + }, + ) + if err != nil { + t.Fatalf("error shouldn't have been "+ + "received, got: %v", err) + } + + // Assert the failure was recorded. + htlc.failure = &htlcFail + assertPaymentInfo( + t, pControl, info.PaymentHash, info, nil, htlc, + ) + + // We also record a payment level fail, to move it into + // a terminal state. + failReason := FailureReasonNoRoute + _, err = pControl.Fail(info.PaymentHash, failReason) + if err != nil { + t.Fatalf("unable to fail payment hash: %v", err) + } + + // Record the reason we failed the payment, such that + // we can assert this later in the test. + firstFailReason = &failReason + } + + // The payment should still be considered in-flight, since there + // is still an active HTLC. + assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) + + // Try to register yet another attempt. This should fail now + // that the payment has reached a terminal condition. + b = *attempt + b.AttemptID = 3 + err = pControl.RegisterAttempt(info.PaymentHash, &b) + if err != ErrPaymentTerminal { + t.Fatalf("expected ErrPaymentTerminal, got: %v", err) + } + + assertPaymentStatus(t, pControl, info.PaymentHash, StatusInFlight) + + // Settle or fail the remaining attempt based on the testcase. + a = attempts[2] + htlc = &htlcStatus{ + HTLCAttemptInfo: a, + } + if test.settleLast { + // Settle the last outstanding attempt. + _, err = pControl.SettleAttempt( + info.PaymentHash, a.AttemptID, + &HTLCSettleInfo{ + Preimage: preimg, + }, + ) + if err != nil { + t.Fatalf("error shouldn't have been "+ + "received, got: %v", err) + } + + htlc.settle = &preimg + assertPaymentInfo( + t, pControl, info.PaymentHash, info, + firstFailReason, htlc, + ) + } else { + // Fail the attempt. + err := pControl.FailAttempt( + info.PaymentHash, a.AttemptID, + &HTLCFailInfo{ + Reason: htlcFail, + }, + ) + if err != nil { + t.Fatalf("error shouldn't have been "+ + "received, got: %v", err) + } + + // Assert the failure was recorded. + htlc.failure = &htlcFail + assertPaymentInfo( + t, pControl, info.PaymentHash, info, + firstFailReason, htlc, + ) + + // Check that we can override any perevious terminal + // failure. This is to allow multiple concurrent shard + // write a terminal failure to the database without + // syncing. + failReason := FailureReasonPaymentDetails + _, err = pControl.Fail(info.PaymentHash, failReason) + if err != nil { + t.Fatalf("unable to fail payment hash: %v", err) + } + } + + // If any of the two attempts settled, the payment should end + // up in the Succeeded state. If both failed the payment should + // also be Failed at this poinnt. + finalStatus := StatusFailed + expRegErr := ErrPaymentAlreadyFailed + if test.settleFirst || test.settleLast { + finalStatus = StatusSucceeded + expRegErr = ErrPaymentAlreadySucceeded + } + + assertPaymentStatus(t, pControl, info.PaymentHash, finalStatus) + + // Finally assert we cannot register more attempts. + err = pControl.RegisterAttempt(info.PaymentHash, &b) + if err != expRegErr { + t.Fatalf("expected error %v, got: %v", expRegErr, err) + } + } + + for _, test := range tests { + test := test + subTest := fmt.Sprintf("first=%v, second=%v", + test.settleFirst, test.settleLast) + + t.Run(subTest, func(t *testing.T) { + runSubTest(t, test) + }) + } +} + +func TestPaymentControlMPPRecordValidation(t *testing.T) { + t.Parallel() + + db, err := initDB() + if err != nil { + t.Fatalf("unable to init db: %v", err) + } + + pControl := NewPaymentControl(db) + + info, attempt, _, err := genInfo() + if err != nil { + t.Fatalf("unable to generate htlc message: %v", err) + } + + // Init the payment. + err = pControl.InitPayment(info.PaymentHash, info) + if err != nil { + t.Fatalf("unable to send htlc message: %v", err) + } + + // Create three unique attempts we'll use for the test, and + // register them with the payment control. We set each + // attempts's value to one third of the payment amount, and + // populate the MPP options. + shardAmt := info.Value / 3 + attempt.Route.FinalHop().AmtToForward = shardAmt + attempt.Route.FinalHop().MPP = record.NewMPP( + info.Value, [32]byte{1}, + ) + + err = pControl.RegisterAttempt(info.PaymentHash, attempt) + if err != nil { + t.Fatalf("unable to send htlc message: %v", err) + } + + // Now try to register a non-MPP attempt, which should fail. + b := *attempt + b.AttemptID = 1 + b.Route.FinalHop().MPP = nil + err = pControl.RegisterAttempt(info.PaymentHash, &b) + if err != ErrMPPayment { + t.Fatalf("expected ErrMPPayment, got: %v", err) + } + + // Try to register attempt one with a different payment address. + b.Route.FinalHop().MPP = record.NewMPP( + info.Value, [32]byte{2}, + ) + err = pControl.RegisterAttempt(info.PaymentHash, &b) + if err != ErrMPPPaymentAddrMismatch { + t.Fatalf("expected ErrMPPPaymentAddrMismatch, got: %v", err) + } + + // Try registering one with a different total amount. + b.Route.FinalHop().MPP = record.NewMPP( + info.Value/2, [32]byte{1}, + ) + err = pControl.RegisterAttempt(info.PaymentHash, &b) + if err != ErrMPPTotalAmountMismatch { + t.Fatalf("expected ErrMPPTotalAmountMismatch, got: %v", err) + } + + // Create and init a new payment. This time we'll check that we cannot + // register an MPP attempt if we already registered a non-MPP one. + info, attempt, _, err = genInfo() + if err != nil { + t.Fatalf("unable to generate htlc message: %v", err) + } + + err = pControl.InitPayment(info.PaymentHash, info) + if err != nil { + t.Fatalf("unable to send htlc message: %v", err) + } + + attempt.Route.FinalHop().MPP = nil + err = pControl.RegisterAttempt(info.PaymentHash, attempt) + if err != nil { + t.Fatalf("unable to send htlc message: %v", err) + } + + // Attempt to register an MPP attempt, which should fail. + b = *attempt + b.AttemptID = 1 + b.Route.FinalHop().MPP = record.NewMPP( + info.Value, [32]byte{1}, + ) + + err = pControl.RegisterAttempt(info.PaymentHash, &b) + if err != ErrNonMPPayment { + t.Fatalf("expected ErrNonMPPayment, got: %v", err) + } +} + // assertPaymentStatus retrieves the status of the payment referred to by hash // and compares it with the expected state. func assertPaymentStatus(t *testing.T, p *PaymentControl, @@ -452,11 +840,16 @@ func assertPaymentStatus(t *testing.T, p *PaymentControl, } } +type htlcStatus struct { + *HTLCAttemptInfo + settle *lntypes.Preimage + failure *HTLCFailReason +} + // assertPaymentInfo retrieves the payment referred to by hash and verifies the // expected values. func assertPaymentInfo(t *testing.T, p *PaymentControl, hash lntypes.Hash, - c *PaymentCreationInfo, aIdx int, a *HTLCAttemptInfo, s lntypes.Preimage, - f *FailureReason) { + c *PaymentCreationInfo, f *FailureReason, a *htlcStatus) { t.Helper() @@ -487,20 +880,35 @@ func assertPaymentInfo(t *testing.T, p *PaymentControl, hash lntypes.Hash, return } - htlc := payment.HTLCs[aIdx] + htlc := payment.HTLCs[a.AttemptID] if err := assertRouteEqual(&htlc.Route, &a.Route); err != nil { t.Fatal("routes do not match") } - var zeroPreimage = lntypes.Preimage{} - if s != zeroPreimage { - if htlc.Settle.Preimage != s { + if htlc.AttemptID != a.AttemptID { + t.Fatalf("unnexpected attempt ID %v, expected %v", + htlc.AttemptID, a.AttemptID) + } + + if a.failure != nil { + if htlc.Failure == nil { + t.Fatalf("expected HTLC to be failed") + } + + if htlc.Failure.Reason != *a.failure { + t.Fatalf("expected HTLC failure %v, had %v", + *a.failure, htlc.Failure.Reason) + } + } else if htlc.Failure != nil { + t.Fatalf("expected no HTLC failure") + } + + if a.settle != nil { + if htlc.Settle.Preimage != *a.settle { t.Fatalf("Preimages don't match: %x vs %x", - htlc.Settle.Preimage, s) - } - } else { - if htlc.Settle != nil { - t.Fatal("expected no settle info") + htlc.Settle.Preimage, a.settle) } + } else if htlc.Settle != nil { + t.Fatal("expected no settle info") } } diff --git a/channeldb/payments.go b/channeldb/payments.go index a1ea379b2..6c5730435 100644 --- a/channeldb/payments.go +++ b/channeldb/payments.go @@ -123,7 +123,12 @@ const ( // LocalLiquidityInsufficient, RemoteCapacityInsufficient. ) -// String returns a human readable FailureReason +// Error returns a human readable error string for the FailureReason. +func (r FailureReason) Error() string { + return r.String() +} + +// String returns a human readable FailureReason. func (r FailureReason) String() string { switch r { case FailureReasonTimeout: @@ -247,6 +252,16 @@ func (db *DB) FetchPayments() ([]*MPPayment, error) { return payments, nil } +func fetchCreationInfo(bucket kvdb.ReadBucket) (*PaymentCreationInfo, error) { + b := bucket.Get(paymentCreationInfoKey) + if b == nil { + return nil, fmt.Errorf("creation info not found") + } + + r := bytes.NewReader(b) + return deserializePaymentCreationInfo(r) +} + func fetchPayment(bucket kvdb.ReadBucket) (*MPPayment, error) { seqBytes := bucket.Get(paymentSequenceKey) if seqBytes == nil { @@ -255,20 +270,8 @@ func fetchPayment(bucket kvdb.ReadBucket) (*MPPayment, error) { sequenceNum := binary.BigEndian.Uint64(seqBytes) - // Get the payment status. - paymentStatus, err := fetchPaymentStatus(bucket) - if err != nil { - return nil, err - } - // Get the PaymentCreationInfo. - b := bucket.Get(paymentCreationInfoKey) - if b == nil { - return nil, fmt.Errorf("creation info not found") - } - - r := bytes.NewReader(b) - creationInfo, err := deserializePaymentCreationInfo(r) + creationInfo, err := fetchCreationInfo(bucket) if err != nil { return nil, err @@ -286,12 +289,50 @@ func fetchPayment(bucket kvdb.ReadBucket) (*MPPayment, error) { // Get failure reason if available. var failureReason *FailureReason - b = bucket.Get(paymentFailInfoKey) + b := bucket.Get(paymentFailInfoKey) if b != nil { reason := FailureReason(b[0]) failureReason = &reason } + // Go through all HTLCs for this payment, noting whether we have any + // settled HTLC, and any still in-flight. + var inflight, settled bool + for _, h := range htlcs { + if h.Failure != nil { + continue + } + + if h.Settle != nil { + settled = true + continue + } + + // If any of the HTLCs are not failed nor settled, we + // still have inflight HTLCs. + inflight = true + } + + // Use the DB state to determine the status of the payment. + var paymentStatus PaymentStatus + + switch { + + // If any of the the HTLCs did succeed and there are no HTLCs in + // flight, the payment succeeded. + case !inflight && settled: + paymentStatus = StatusSucceeded + + // If we have no in-flight HTLCs, and the payment failure is set, the + // payment is considered failed. + case !inflight && failureReason != nil: + paymentStatus = StatusFailed + + // Otherwise it is still in flight. + default: + paymentStatus = StatusInFlight + } + return &MPPayment{ sequenceNum: sequenceNum, Info: creationInfo, @@ -407,6 +448,8 @@ func (db *DB) DeletePayments() error { return err } + // If the status is InFlight, we cannot safely delete + // the payment information, so we return early. if paymentStatus == StatusInFlight { return nil } diff --git a/lntest/itest/lnd_mpp_test.go b/lntest/itest/lnd_mpp_test.go new file mode 100644 index 000000000..9b338baf0 --- /dev/null +++ b/lntest/itest/lnd_mpp_test.go @@ -0,0 +1,357 @@ +// +build rpctest + +package itest + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + "github.com/lightningnetwork/lnd" + "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" + "github.com/lightningnetwork/lnd/lntest" + "github.com/lightningnetwork/lnd/routing/route" +) + +// testSendToRouteMultiPath tests that we are able to successfully route a +// payment using multiple shards across different paths, by using SendToRoute. +func testSendToRouteMultiPath(net *lntest.NetworkHarness, t *harnessTest) { + ctxb := context.Background() + + // To ensure the payment goes through seperate paths, we'll set a + // channel size that can only carry one shard at a time. We'll divide + // the payment into 3 shards. + const ( + paymentAmt = btcutil.Amount(300000) + shardAmt = paymentAmt / 3 + chanAmt = shardAmt * 3 / 2 + ) + + // Set up a network with three different paths Alice <-> Bob. + // _ Eve _ + // / \ + // Alice -- Carol ---- Bob + // \ / + // \__ Dave ____/ + // + // + // Create the three nodes in addition to Alice and Bob. + alice := net.Alice + bob := net.Bob + carol, err := net.NewNode("carol", nil) + if err != nil { + t.Fatalf("unable to create carol: %v", err) + } + defer shutdownAndAssert(net, t, carol) + + dave, err := net.NewNode("dave", nil) + if err != nil { + t.Fatalf("unable to create dave: %v", err) + } + defer shutdownAndAssert(net, t, dave) + + eve, err := net.NewNode("eve", nil) + if err != nil { + t.Fatalf("unable to create eve: %v", err) + } + defer shutdownAndAssert(net, t, eve) + + nodes := []*lntest.HarnessNode{alice, bob, carol, dave, eve} + + // Connect nodes to ensure propagation of channels. + for i := 0; i < len(nodes); i++ { + for j := i + 1; j < len(nodes); j++ { + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + if err := net.EnsureConnected(ctxt, nodes[i], nodes[j]); err != nil { + t.Fatalf("unable to connect nodes: %v", err) + } + } + } + + // We'll send shards along three routes from Alice. + sendRoutes := [][]*lntest.HarnessNode{ + {carol, bob}, + {dave, bob}, + {carol, eve, bob}, + } + + // Keep a list of all our active channels. + var networkChans []*lnrpc.ChannelPoint + var closeChannelFuncs []func() + + // openChannel is a helper to open a channel from->to. + openChannel := func(from, to *lntest.HarnessNode, chanSize btcutil.Amount) { + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + err := net.SendCoins(ctxt, btcutil.SatoshiPerBitcoin, from) + if err != nil { + t.Fatalf("unable to send coins : %v", err) + } + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPoint := openChannelAndAssert( + ctxt, t, net, from, to, + lntest.OpenChannelParams{ + Amt: chanSize, + }, + ) + + closeChannelFuncs = append(closeChannelFuncs, func() { + ctxt, _ := context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert( + ctxt, t, net, from, chanPoint, false, + ) + }) + + networkChans = append(networkChans, chanPoint) + } + + // Open channels between the nodes. + openChannel(carol, bob, chanAmt) + openChannel(dave, bob, chanAmt) + openChannel(alice, dave, chanAmt) + openChannel(eve, bob, chanAmt) + openChannel(carol, eve, chanAmt) + + // Since the channel Alice-> Carol will have to carry two + // shards, we make it larger. + openChannel(alice, carol, chanAmt+shardAmt) + + for _, f := range closeChannelFuncs { + defer f() + } + + // Wait for all nodes to have seen all channels. + for _, chanPoint := range networkChans { + for _, node := range nodes { + txid, err := lnd.GetChanPointFundingTxid(chanPoint) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + point := wire.OutPoint{ + Hash: *txid, + Index: chanPoint.OutputIndex, + } + + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) + if err != nil { + t.Fatalf("(%d): timeout waiting for "+ + "channel(%s) open: %v", + node.NodeID, point, err) + } + } + } + + // Make Bob create an invoice for Alice to pay. + payReqs, rHashes, invoices, err := createPayReqs( + net.Bob, paymentAmt, 1, + ) + if err != nil { + t.Fatalf("unable to create pay reqs: %v", err) + } + + rHash := rHashes[0] + payReq := payReqs[0] + + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + decodeResp, err := net.Bob.DecodePayReq( + ctxt, &lnrpc.PayReqString{PayReq: payReq}, + ) + if err != nil { + t.Fatalf("decode pay req: %v", err) + } + + payAddr := decodeResp.PaymentAddr + + // Helper function for Alice to build a route from pubkeys. + buildRoute := func(amt btcutil.Amount, hops []*lntest.HarnessNode) ( + *lnrpc.Route, error) { + + rpcHops := make([][]byte, 0, len(hops)) + for _, hop := range hops { + k := hop.PubKeyStr + pubkey, err := route.NewVertexFromStr(k) + if err != nil { + return nil, fmt.Errorf("error parsing %v: %v", + k, err) + } + rpcHops = append(rpcHops, pubkey[:]) + } + + req := &routerrpc.BuildRouteRequest{ + AmtMsat: int64(amt * 1000), + FinalCltvDelta: lnd.DefaultBitcoinTimeLockDelta, + HopPubkeys: rpcHops, + } + + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + routeResp, err := net.Alice.RouterClient.BuildRoute(ctxt, req) + if err != nil { + return nil, err + } + + return routeResp.Route, nil + } + + responses := make(chan *routerrpc.SendToRouteResponse, len(sendRoutes)) + for _, hops := range sendRoutes { + // Build a route for the specified hops. + r, err := buildRoute(shardAmt, hops) + if err != nil { + t.Fatalf("unable to build route: %v", err) + } + + // Set the MPP records to indicate this is a payment shard. + hop := r.Hops[len(r.Hops)-1] + hop.TlvPayload = true + hop.MppRecord = &lnrpc.MPPRecord{ + PaymentAddr: payAddr, + TotalAmtMsat: int64(paymentAmt * 1000), + } + + // Send the shard. + sendReq := &routerrpc.SendToRouteRequest{ + PaymentHash: rHash, + Route: r, + } + + // We'll send all shards in their own goroutine, since SendToRoute will + // block as long as the payment is in flight. + go func() { + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + resp, err := net.Alice.RouterClient.SendToRoute(ctxt, sendReq) + if err != nil { + t.Fatalf("unable to send payment: %v", err) + } + + responses <- resp + }() + } + + // Wait for all responses to be back, and check that they all + // succeeded. + for range sendRoutes { + var resp *routerrpc.SendToRouteResponse + select { + case resp = <-responses: + case <-time.After(defaultTimeout): + t.Fatalf("response not received") + } + + if resp.Failure != nil { + t.Fatalf("received payment failure : %v", resp.Failure) + } + + // All shards should come back with the preimage. + if !bytes.Equal(resp.Preimage, invoices[0].RPreimage) { + t.Fatalf("preimage doesn't match") + } + } + + // assertNumHtlcs is a helper that checks the node's latest payment, + // and asserts it was split into num shards. + assertNumHtlcs := func(node *lntest.HarnessNode, num int) { + req := &lnrpc.ListPaymentsRequest{ + IncludeIncomplete: true, + } + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + paymentsResp, err := node.ListPayments(ctxt, req) + if err != nil { + t.Fatalf("error when obtaining payments: %v", + err) + } + + payments := paymentsResp.Payments + if len(payments) == 0 { + t.Fatalf("no payments found") + } + + payment := payments[len(payments)-1] + htlcs := payment.Htlcs + if len(htlcs) == 0 { + t.Fatalf("no htlcs") + } + + succeeded := 0 + for _, htlc := range htlcs { + if htlc.Status == lnrpc.HTLCAttempt_SUCCEEDED { + succeeded++ + } + } + + if succeeded != num { + t.Fatalf("expected %v succussful HTLCs, got %v", num, + succeeded) + } + } + + // assertSettledInvoice checks that the invoice for the given payment + // hash is settled, and has been paid using num HTLCs. + assertSettledInvoice := func(node *lntest.HarnessNode, rhash []byte, + num int) { + + found := false + offset := uint64(0) + for !found { + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + invoicesResp, err := node.ListInvoices( + ctxt, &lnrpc.ListInvoiceRequest{ + IndexOffset: offset, + }, + ) + if err != nil { + t.Fatalf("error when obtaining payments: %v", + err) + } + + if len(invoicesResp.Invoices) == 0 { + break + } + + for _, inv := range invoicesResp.Invoices { + if !bytes.Equal(inv.RHash, rhash) { + continue + } + + // Assert that the amount paid to the invoice is + // correct. + if inv.AmtPaidSat != int64(paymentAmt) { + t.Fatalf("incorrect payment amt for "+ + "invoicewant: %d, got %d", + paymentAmt, inv.AmtPaidSat) + } + + if inv.State != lnrpc.Invoice_SETTLED { + t.Fatalf("Invoice not settled: %v", + inv.State) + } + + if len(inv.Htlcs) != num { + t.Fatalf("expected invoice to be "+ + "settled with %v HTLCs, had %v", + num, len(inv.Htlcs)) + } + + found = true + break + } + + offset = invoicesResp.LastIndexOffset + } + + if !found { + t.Fatalf("invoice not found") + } + } + + // Finally check that the payment shows up with three settled HTLCs in + // Alice's list of payments... + assertNumHtlcs(net.Alice, 3) + + // ...and in Bob's list of paid invoices. + assertSettledInvoice(net.Bob, rHash, 3) +} diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index 1a1133997..d35fca729 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -1915,8 +1915,9 @@ func testUpdateChannelPolicy(net *lntest.NetworkHarness, t *harnessTest) { // Alice knows about the channel policy of Carol and should therefore // not be able to find a path during routing. + expErr := channeldb.FailureReasonNoRoute.Error() if err == nil || - !strings.Contains(err.Error(), "unable to find a path") { + !strings.Contains(err.Error(), expErr) { t.Fatalf("expected payment to fail, instead got %v", err) } @@ -3995,6 +3996,41 @@ func assertAmountSent(amt btcutil.Amount, sndr, rcvr *lntest.HarnessNode) func() } } +// assertLastHTLCError checks that the last sent HTLC of the last payment sent +// by the given node failed with the expected failure code. +func assertLastHTLCError(t *harnessTest, node *lntest.HarnessNode, + code lnrpc.Failure_FailureCode) { + + req := &lnrpc.ListPaymentsRequest{ + IncludeIncomplete: true, + } + ctxt, _ := context.WithTimeout(context.Background(), defaultTimeout) + paymentsResp, err := node.ListPayments(ctxt, req) + if err != nil { + t.Fatalf("error when obtaining payments: %v", err) + } + + payments := paymentsResp.Payments + if len(payments) == 0 { + t.Fatalf("no payments found") + } + + payment := payments[len(payments)-1] + htlcs := payment.Htlcs + if len(htlcs) == 0 { + t.Fatalf("no htlcs") + } + + htlc := htlcs[len(htlcs)-1] + if htlc.Failure == nil { + t.Fatalf("expected failure") + } + + if htlc.Failure.Code != code { + t.Fatalf("expected failure %v, got %v", code, htlc.Failure.Code) + } +} + // testSphinxReplayPersistence verifies that replayed onion packets are rejected // by a remote peer after a restart. We use a combination of unsafe // configuration arguments to force Carol to replay the same sphinx packet after @@ -4134,11 +4170,10 @@ func testSphinxReplayPersistence(net *lntest.NetworkHarness, t *harnessTest) { // Construct the response we expect after sending a duplicate packet // that fails due to sphinx replay detection. - replayErr := "InvalidOnionKey" - if !strings.Contains(resp.PaymentError, replayErr) { - t.Fatalf("received payment error: %v, expected %v", - resp.PaymentError, replayErr) + if resp.PaymentError == "" { + t.Fatalf("expected payment error") } + assertLastHTLCError(t, carol, lnrpc.Failure_INVALID_ONION_KEY) // Since the payment failed, the balance should still be left // unaltered. @@ -9452,12 +9487,11 @@ out: t.Fatalf("payment should have been rejected due to invalid " + "payment hash") } - expectedErrorCode := lnwire.CodeIncorrectOrUnknownPaymentDetails.String() - if !strings.Contains(resp.PaymentError, expectedErrorCode) { - // TODO(roasbeef): make into proper gRPC error code - t.Fatalf("payment should have failed due to unknown payment hash, "+ - "instead failed due to: %v", resp.PaymentError) - } + + assertLastHTLCError( + t, net.Alice, + lnrpc.Failure_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS, + ) // The balances of all parties should be the same as initially since // the HTLC was canceled. @@ -9484,18 +9518,11 @@ out: t.Fatalf("payment should have been rejected due to wrong " + "HTLC amount") } - expectedErrorCode = lnwire.CodeIncorrectOrUnknownPaymentDetails.String() - if !strings.Contains(resp.PaymentError, expectedErrorCode) { - t.Fatalf("payment should have failed due to wrong amount, "+ - "instead failed due to: %v", resp.PaymentError) - } - // We'll also ensure that the encoded error includes the invlaid HTLC - // amount. - if !strings.Contains(resp.PaymentError, htlcAmt.String()) { - t.Fatalf("error didn't include expected payment amt of %v: "+ - "%v", htlcAmt, resp.PaymentError) - } + assertLastHTLCError( + t, net.Alice, + lnrpc.Failure_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS, + ) // The balances of all parties should be the same as initially since // the HTLC was canceled. @@ -9574,12 +9601,12 @@ out: if resp.PaymentError == "" { t.Fatalf("payment should fail due to insufficient "+ "capacity: %v", err) - } else if !strings.Contains(resp.PaymentError, - lnwire.CodeTemporaryChannelFailure.String()) { - t.Fatalf("payment should fail due to insufficient capacity, "+ - "instead: %v", resp.PaymentError) } + assertLastHTLCError( + t, net.Alice, lnrpc.Failure_TEMPORARY_CHANNEL_FAILURE, + ) + // Generate new invoice to not pay same invoice twice. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) carolInvoice, err = carol.AddInvoice(ctxt, invoiceReq) @@ -9616,11 +9643,8 @@ out: if resp.PaymentError == "" { t.Fatalf("payment should have failed") } - expectedErrorCode = lnwire.CodeUnknownNextPeer.String() - if !strings.Contains(resp.PaymentError, expectedErrorCode) { - t.Fatalf("payment should fail due to unknown hop, instead: %v", - resp.PaymentError) - } + + assertLastHTLCError(t, net.Alice, lnrpc.Failure_UNKNOWN_NEXT_PEER) // Finally, immediately close the channel. This function will also // block until the channel is closed and will additionally assert the @@ -9787,9 +9811,8 @@ func testRejectHTLC(net *lntest.NetworkHarness, t *harnessTest) { "should have been rejected, carol will not accept forwarded htlcs", ) } - if !strings.Contains(err.Error(), lnwire.CodeChannelDisabled.String()) { - t.Fatalf("error returned should have been Channel Disabled") - } + + assertLastHTLCError(t, net.Alice, lnrpc.Failure_CHANNEL_DISABLED) // Close all channels. ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) @@ -14885,6 +14908,10 @@ var testsCases = []*testCase{ name: "psbt channel funding", test: testPsbtChanFunding, }, + { + name: "sendtoroute multi path payment", + test: testSendToRouteMultiPath, + }, } // TestLightningNetworkDaemon performs a series of integration tests amongst a diff --git a/routing/control_tower.go b/routing/control_tower.go index 5ade611cc..da7d4d2bc 100644 --- a/routing/control_tower.go +++ b/routing/control_tower.go @@ -34,6 +34,10 @@ type ControlTower interface { // FailAttempt marks the given payment attempt failed. FailAttempt(lntypes.Hash, uint64, *channeldb.HTLCFailInfo) error + // FetchPayment fetches the payment corresponding to the given payment + // hash. + FetchPayment(paymentHash lntypes.Hash) (*channeldb.MPPayment, error) + // Fail transitions a payment into the Failed state, and records the // ultimate reason the payment failed. Note that this should only be // called when all active active attempts are already failed. After @@ -132,6 +136,13 @@ func (p *controlTower) FailAttempt(paymentHash lntypes.Hash, return p.db.FailAttempt(paymentHash, attemptID, failInfo) } +// FetchPayment fetches the payment corresponding to the given payment hash. +func (p *controlTower) FetchPayment(paymentHash lntypes.Hash) ( + *channeldb.MPPayment, error) { + + return p.db.FetchPayment(paymentHash) +} + // createSuccessResult creates a success result to send to subscribers. func createSuccessResult(htlcs []channeldb.HTLCAttempt) *PaymentResult { // Extract any preimage from the list of HTLCs. diff --git a/routing/control_tower_test.go b/routing/control_tower_test.go index 6bc8ffd7d..82dc2706f 100644 --- a/routing/control_tower_test.go +++ b/routing/control_tower_test.go @@ -324,7 +324,7 @@ func genInfo() (*channeldb.PaymentCreationInfo, *channeldb.HTLCAttemptInfo, rhash := sha256.Sum256(preimage[:]) return &channeldb.PaymentCreationInfo{ PaymentHash: rhash, - Value: 1, + Value: testRoute.ReceiverAmt(), CreationTime: time.Unix(time.Now().Unix(), 0), PaymentRequest: []byte("hola"), }, diff --git a/routing/errors.go b/routing/errors.go index 3166f7390..3beac229a 100644 --- a/routing/errors.go +++ b/routing/errors.go @@ -15,10 +15,6 @@ const ( // this update can't bring us something new, or because a node // announcement was given for node not found in any channel. ErrIgnored - - // ErrPaymentAttemptTimeout is an error that indicates that a payment - // attempt timed out before we were able to successfully route an HTLC. - ErrPaymentAttemptTimeout ) // routerError is a structure that represent the error inside the routing package, diff --git a/routing/mock_test.go b/routing/mock_test.go index 6332bea8f..9645f73bf 100644 --- a/routing/mock_test.go +++ b/routing/mock_test.go @@ -10,12 +10,13 @@ import ( "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/routing/route" - "github.com/lightningnetwork/lnd/zpay32" ) type mockPaymentAttemptDispatcher struct { onPayment func(firstHop lnwire.ShortChannelID) ([32]byte, error) results map[uint64]*htlcswitch.PaymentResult + + sync.Mutex } var _ PaymentAttemptDispatcher = (*mockPaymentAttemptDispatcher)(nil) @@ -28,10 +29,6 @@ func (m *mockPaymentAttemptDispatcher) SendHTLC(firstHop lnwire.ShortChannelID, return nil } - if m.results == nil { - m.results = make(map[uint64]*htlcswitch.PaymentResult) - } - var result *htlcswitch.PaymentResult preimage, err := m.onPayment(firstHop) if err != nil { @@ -46,7 +43,13 @@ func (m *mockPaymentAttemptDispatcher) SendHTLC(firstHop lnwire.ShortChannelID, result = &htlcswitch.PaymentResult{Preimage: preimage} } + m.Lock() + if m.results == nil { + m.results = make(map[uint64]*htlcswitch.PaymentResult) + } + m.results[pid] = result + m.Unlock() return nil } @@ -56,7 +59,11 @@ func (m *mockPaymentAttemptDispatcher) GetPaymentResult(paymentID uint64, <-chan *htlcswitch.PaymentResult, error) { c := make(chan *htlcswitch.PaymentResult, 1) + + m.Lock() res, ok := m.results[paymentID] + m.Unlock() + if !ok { return nil, htlcswitch.ErrPaymentIDNotFound } @@ -78,8 +85,8 @@ type mockPaymentSessionSource struct { var _ PaymentSessionSource = (*mockPaymentSessionSource)(nil) -func (m *mockPaymentSessionSource) NewPaymentSession(routeHints [][]zpay32.HopHint, - target route.Vertex) (PaymentSession, error) { +func (m *mockPaymentSessionSource) NewPaymentSession( + _ *LightningPayment) (PaymentSession, error) { return &mockPaymentSession{m.routes}, nil } @@ -102,6 +109,13 @@ func (m *mockMissionControl) ReportPaymentFail(paymentID uint64, rt *route.Route failureSourceIdx *int, failure lnwire.FailureMessage) ( *channeldb.FailureReason, error) { + // Report a permanent failure if this is an error caused + // by incorrect details. + if failure.Code() == lnwire.CodeIncorrectOrUnknownPaymentDetails { + reason := channeldb.FailureReasonPaymentDetails + return &reason, nil + } + return nil, nil } @@ -123,11 +137,11 @@ type mockPaymentSession struct { var _ PaymentSession = (*mockPaymentSession)(nil) -func (m *mockPaymentSession) RequestRoute(payment *LightningPayment, - height uint32, finalCltvDelta uint16) (*route.Route, error) { +func (m *mockPaymentSession) RequestRoute(_, _ lnwire.MilliSatoshi, + _, height uint32) (*route.Route, error) { if len(m.routes) == 0 { - return nil, fmt.Errorf("no routes") + return nil, errNoPathFound } r := m.routes[0] @@ -177,27 +191,38 @@ type initArgs struct { c *channeldb.PaymentCreationInfo } -type registerArgs struct { +type registerAttemptArgs struct { a *channeldb.HTLCAttemptInfo } -type successArgs struct { +type settleAttemptArgs struct { preimg lntypes.Preimage } -type failArgs struct { +type failAttemptArgs struct { + reason *channeldb.HTLCFailInfo +} + +type failPaymentArgs struct { reason channeldb.FailureReason } -type mockControlTower struct { - inflights map[lntypes.Hash]channeldb.InFlightPayment - successful map[lntypes.Hash]struct{} +type testPayment struct { + info channeldb.PaymentCreationInfo + attempts []channeldb.HTLCAttempt +} - init chan initArgs - register chan registerArgs - success chan successArgs - fail chan failArgs - fetchInFlight chan struct{} +type mockControlTower struct { + payments map[lntypes.Hash]*testPayment + successful map[lntypes.Hash]struct{} + failed map[lntypes.Hash]channeldb.FailureReason + + init chan initArgs + registerAttempt chan registerAttemptArgs + settleAttempt chan settleAttemptArgs + failAttempt chan failAttemptArgs + failPayment chan failPaymentArgs + fetchInFlight chan struct{} sync.Mutex } @@ -206,8 +231,9 @@ var _ ControlTower = (*mockControlTower)(nil) func makeMockControlTower() *mockControlTower { return &mockControlTower{ - inflights: make(map[lntypes.Hash]channeldb.InFlightPayment), + payments: make(map[lntypes.Hash]*testPayment), successful: make(map[lntypes.Hash]struct{}), + failed: make(map[lntypes.Hash]channeldb.FailureReason), } } @@ -221,18 +247,22 @@ func (m *mockControlTower) InitPayment(phash lntypes.Hash, m.init <- initArgs{c} } + // Don't allow re-init a successful payment. if _, ok := m.successful[phash]; ok { - return fmt.Errorf("already successful") + return channeldb.ErrAlreadyPaid } - _, ok := m.inflights[phash] - if ok { - return fmt.Errorf("in flight") + _, failed := m.failed[phash] + _, ok := m.payments[phash] + + // If the payment is known, only allow re-init if failed. + if ok && !failed { + return channeldb.ErrPaymentInFlight } - m.inflights[phash] = channeldb.InFlightPayment{ - Info: c, - Attempts: make([]channeldb.HTLCAttemptInfo, 0), + delete(m.failed, phash) + m.payments[phash] = &testPayment{ + info: *c, } return nil @@ -244,17 +274,28 @@ func (m *mockControlTower) RegisterAttempt(phash lntypes.Hash, m.Lock() defer m.Unlock() - if m.register != nil { - m.register <- registerArgs{a} + if m.registerAttempt != nil { + m.registerAttempt <- registerAttemptArgs{a} } - p, ok := m.inflights[phash] + // Cannot register attempts for successful or failed payments. + if _, ok := m.successful[phash]; ok { + return channeldb.ErrPaymentAlreadySucceeded + } + + if _, ok := m.failed[phash]; ok { + return channeldb.ErrPaymentAlreadyFailed + } + + p, ok := m.payments[phash] if !ok { - return fmt.Errorf("not in flight") + return channeldb.ErrPaymentNotInitiated } - p.Attempts = append(p.Attempts, *a) - m.inflights[phash] = p + p.attempts = append(p.attempts, channeldb.HTLCAttempt{ + HTLCAttemptInfo: *a, + }) + m.payments[phash] = p return nil } @@ -265,13 +306,73 @@ func (m *mockControlTower) SettleAttempt(phash lntypes.Hash, m.Lock() defer m.Unlock() - if m.success != nil { - m.success <- successArgs{settleInfo.Preimage} + if m.settleAttempt != nil { + m.settleAttempt <- settleAttemptArgs{settleInfo.Preimage} } - delete(m.inflights, phash) - m.successful[phash] = struct{}{} - return nil + // Only allow setting attempts if the payment is known. + p, ok := m.payments[phash] + if !ok { + return channeldb.ErrPaymentNotInitiated + } + + // Find the attempt with this pid, and set the settle info. + for i, a := range p.attempts { + if a.AttemptID != pid { + continue + } + + if a.Settle != nil { + return channeldb.ErrAttemptAlreadySettled + } + if a.Failure != nil { + return channeldb.ErrAttemptAlreadyFailed + } + + p.attempts[i].Settle = settleInfo + + // Mark the payment successful on first settled attempt. + m.successful[phash] = struct{}{} + return nil + } + + return fmt.Errorf("pid not found") +} + +func (m *mockControlTower) FailAttempt(phash lntypes.Hash, pid uint64, + failInfo *channeldb.HTLCFailInfo) error { + + m.Lock() + defer m.Unlock() + + if m.failAttempt != nil { + m.failAttempt <- failAttemptArgs{failInfo} + } + + // Only allow failing attempts if the payment is known. + p, ok := m.payments[phash] + if !ok { + return channeldb.ErrPaymentNotInitiated + } + + // Find the attempt with this pid, and set the failure info. + for i, a := range p.attempts { + if a.AttemptID != pid { + continue + } + + if a.Settle != nil { + return channeldb.ErrAttemptAlreadySettled + } + if a.Failure != nil { + return channeldb.ErrAttemptAlreadyFailed + } + + p.attempts[i].Failure = failInfo + return nil + } + + return fmt.Errorf("pid not found") } func (m *mockControlTower) Fail(phash lntypes.Hash, @@ -280,14 +381,46 @@ func (m *mockControlTower) Fail(phash lntypes.Hash, m.Lock() defer m.Unlock() - if m.fail != nil { - m.fail <- failArgs{reason} + if m.failPayment != nil { + m.failPayment <- failPaymentArgs{reason} } - delete(m.inflights, phash) + // Payment must be known. + if _, ok := m.payments[phash]; !ok { + return channeldb.ErrPaymentNotInitiated + } + + m.failed[phash] = reason + return nil } +func (m *mockControlTower) FetchPayment(phash lntypes.Hash) ( + *channeldb.MPPayment, error) { + + m.Lock() + defer m.Unlock() + + p, ok := m.payments[phash] + if !ok { + return nil, channeldb.ErrPaymentNotInitiated + } + + mp := &channeldb.MPPayment{ + Info: &p.info, + } + + reason, ok := m.failed[phash] + if ok { + mp.FailureReason = &reason + } + + // Return a copy of the current attempts. + mp.HTLCs = append(mp.HTLCs, p.attempts...) + + return mp, nil +} + func (m *mockControlTower) FetchInFlightPayments() ( []*channeldb.InFlightPayment, error) { @@ -298,8 +431,25 @@ func (m *mockControlTower) FetchInFlightPayments() ( m.fetchInFlight <- struct{}{} } + // In flight are all payments not successful or failed. var fl []*channeldb.InFlightPayment - for _, ifl := range m.inflights { + for hash, p := range m.payments { + if _, ok := m.successful[hash]; ok { + continue + } + if _, ok := m.failed[hash]; ok { + continue + } + + var attempts []channeldb.HTLCAttemptInfo + for _, a := range p.attempts { + attempts = append(attempts, a.HTLCAttemptInfo) + } + ifl := channeldb.InFlightPayment{ + Info: &p.info, + Attempts: attempts, + } + fl = append(fl, &ifl) } @@ -311,9 +461,3 @@ func (m *mockControlTower) SubscribePayment(paymentHash lntypes.Hash) ( return false, nil, errors.New("not implemented") } - -func (m *mockControlTower) FailAttempt(hash lntypes.Hash, pid uint64, - failInfo *channeldb.HTLCFailInfo) error { - - return nil -} diff --git a/routing/pathfind.go b/routing/pathfind.go index 549d34166..1365958c6 100644 --- a/routing/pathfind.go +++ b/routing/pathfind.go @@ -58,24 +58,6 @@ var ( // DefaultAprioriHopProbability is the default a priori probability for // a hop. DefaultAprioriHopProbability = float64(0.6) - - // errNoTlvPayload is returned when the destination hop does not support - // a tlv payload. - errNoTlvPayload = errors.New("destination hop doesn't " + - "understand new TLV payloads") - - // errNoPaymentAddr is returned when the destination hop does not - // support payment addresses. - errNoPaymentAddr = errors.New("destination hop doesn't " + - "understand payment addresses") - - // errNoPathFound is returned when a path to the target destination does - // not exist in the graph. - errNoPathFound = errors.New("unable to find a path to destination") - - // errInsufficientLocalBalance is returned when none of the local - // channels have enough balance for the payment. - errInsufficientBalance = errors.New("insufficient local balance") ) // edgePolicyWithSource is a helper struct to keep track of the source node diff --git a/routing/payment_lifecycle.go b/routing/payment_lifecycle.go index 96441d179..6ac79400a 100644 --- a/routing/payment_lifecycle.go +++ b/routing/payment_lifecycle.go @@ -2,346 +2,607 @@ package routing import ( "fmt" + "sync" "time" "github.com/davecgh/go-spew/spew" sphinx "github.com/lightningnetwork/lightning-onion" "github.com/lightningnetwork/lnd/channeldb" "github.com/lightningnetwork/lnd/htlcswitch" + "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/routing/route" ) -// errNoRoute is returned when all routes from the payment session have been -// attempted. -type errNoRoute struct { - // lastError is the error encountered during the last payment attempt, - // if at least one attempt has been made. - lastError error -} - -// Error returns a string representation of the error. -func (e errNoRoute) Error() string { - return fmt.Sprintf("unable to route payment to destination: %v", - e.lastError) -} - // paymentLifecycle holds all information about the current state of a payment // needed to resume if from any point. type paymentLifecycle struct { - router *ChannelRouter - payment *LightningPayment - paySession PaymentSession - timeoutChan <-chan time.Time - currentHeight int32 - finalCLTVDelta uint16 - attempt *channeldb.HTLCAttemptInfo - circuit *sphinx.Circuit - lastError error + router *ChannelRouter + totalAmount lnwire.MilliSatoshi + feeLimit lnwire.MilliSatoshi + paymentHash lntypes.Hash + paySession PaymentSession + timeoutChan <-chan time.Time + currentHeight int32 +} + +// payemntState holds a number of key insights learned from a given MPPayment +// that we use to determine what to do on each payment loop iteration. +type paymentState struct { + numShardsInFlight int + remainingAmt lnwire.MilliSatoshi + remainingFees lnwire.MilliSatoshi + terminate bool +} + +// paymentState uses the passed payment to find the latest information we need +// to act on every iteration of the payment loop. +func (p *paymentLifecycle) paymentState(payment *channeldb.MPPayment) ( + *paymentState, error) { + + // Fetch the total amount and fees that has already been sent in + // settled and still in-flight shards. + sentAmt, fees := payment.SentAmt() + + // Sanity check we haven't sent a value larger than the payment amount. + if sentAmt > p.totalAmount { + return nil, fmt.Errorf("amount sent %v exceeds "+ + "total amount %v", sentAmt, p.totalAmount) + } + + // We'll subtract the used fee from our fee budget, but allow the fees + // of the already sent shards to exceed our budget (can happen after + // restarts). + feeBudget := p.feeLimit + if fees <= feeBudget { + feeBudget -= fees + } else { + feeBudget = 0 + } + + // Get any terminal info for this payment. + settle, failure := payment.TerminalInfo() + + // If either an HTLC settled, or the payment has a payment level + // failure recorded, it means we should terminate the moment all shards + // have returned with a result. + terminate := settle != nil || failure != nil + + activeShards := payment.InFlightHTLCs() + return &paymentState{ + numShardsInFlight: len(activeShards), + remainingAmt: p.totalAmount - sentAmt, + remainingFees: feeBudget, + terminate: terminate, + }, nil } // resumePayment resumes the paymentLifecycle from the current state. func (p *paymentLifecycle) resumePayment() ([32]byte, *route.Route, error) { + shardHandler := &shardHandler{ + router: p.router, + paymentHash: p.paymentHash, + shardErrors: make(chan error), + quit: make(chan struct{}), + } + + // When the payment lifecycle loop exits, we make sure to signal any + // sub goroutine of the shardHandler to exit, then wait for them to + // return. + defer shardHandler.stop() + + // If we had any existing attempts outstanding, we'll start by spinning + // up goroutines that'll collect their results and deliver them to the + // lifecycle loop below. + payment, err := p.router.cfg.Control.FetchPayment( + p.paymentHash, + ) + if err != nil { + return [32]byte{}, nil, err + } + + for _, a := range payment.InFlightHTLCs() { + a := a + + log.Debugf("Resuming payment shard %v for hash %v", + a.AttemptID, p.paymentHash) + + shardHandler.collectResultAsync(&a.HTLCAttemptInfo) + } + // We'll continue until either our payment succeeds, or we encounter a // critical error during path finding. for { - - // If this payment had no existing payment attempt, we create - // and send one now. - if p.attempt == nil { - firstHop, htlcAdd, err := p.createNewPaymentAttempt() - if err != nil { - return [32]byte{}, nil, err - } - - // Now that the attempt is created and checkpointed to - // the DB, we send it. - sendErr := p.sendPaymentAttempt(firstHop, htlcAdd) - if sendErr != nil { - // TODO(joostjager): Distinguish unexpected - // internal errors from real send errors. - err = p.failAttempt(sendErr) - if err != nil { - return [32]byte{}, nil, err - } - - // We must inspect the error to know whether it - // was critical or not, to decide whether we - // should continue trying. - err := p.handleSendError(sendErr) - if err != nil { - return [32]byte{}, nil, err - } - - // Error was handled successfully, reset the - // attempt to indicate we want to make a new - // attempt. - p.attempt = nil - continue - } - } else { - // If this was a resumed attempt, we must regenerate the - // circuit. We don't need to check for errors resulting - // from an invalid route, because the sphinx packet has - // been successfully generated before. - _, c, err := generateSphinxPacket( - &p.attempt.Route, p.payment.PaymentHash[:], - p.attempt.SessionKey, - ) - if err != nil { - return [32]byte{}, nil, err - } - p.circuit = c - } - - // Using the created circuit, initialize the error decrypter so we can - // parse+decode any failures incurred by this payment within the - // switch. - errorDecryptor := &htlcswitch.SphinxErrorDecrypter{ - OnionErrorDecrypter: sphinx.NewOnionErrorDecrypter(p.circuit), - } - - // Now ask the switch to return the result of the payment when - // available. - resultChan, err := p.router.cfg.Payer.GetPaymentResult( - p.attempt.AttemptID, p.payment.PaymentHash, errorDecryptor, - ) - switch { - - // If this attempt ID is unknown to the Switch, it means it was - // never checkpointed and forwarded by the switch before a - // restart. In this case we can safely send a new payment - // attempt, and wait for its result to be available. - case err == htlcswitch.ErrPaymentIDNotFound: - log.Debugf("Payment ID %v for hash %x not found in "+ - "the Switch, retrying.", p.attempt.AttemptID, - p.payment.PaymentHash) - - err = p.failAttempt(err) - if err != nil { - return [32]byte{}, nil, err - } - - // Reset the attempt to indicate we want to make a new - // attempt. - p.attempt = nil - continue - - // A critical, unexpected error was encountered. - case err != nil: - log.Errorf("Failed getting result for attemptID %d "+ - "from switch: %v", p.attempt.AttemptID, err) - + // Start by quickly checking if there are any outcomes already + // available to handle before we reevaluate our state. + if err := shardHandler.checkShards(); err != nil { return [32]byte{}, nil, err } - // The switch knows about this payment, we'll wait for a result - // to be available. - var ( - result *htlcswitch.PaymentResult - ok bool + // We start every iteration by fetching the lastest state of + // the payment from the ControlTower. This ensures that we will + // act on the latest available information, whether we are + // resuming an existing payment or just sent a new attempt. + payment, err := p.router.cfg.Control.FetchPayment( + p.paymentHash, ) + if err != nil { + return [32]byte{}, nil, err + } - select { - case result, ok = <-resultChan: - if !ok { - return [32]byte{}, nil, htlcswitch.ErrSwitchExiting + // Using this latest state of the payment, calculate + // information about our active shards and terminal conditions. + state, err := p.paymentState(payment) + if err != nil { + return [32]byte{}, nil, err + } + + log.Debugf("Payment %v in state terminate=%v, "+ + "active_shards=%v, rem_value=%v, fee_limit=%v", + p.paymentHash, state.terminate, state.numShardsInFlight, + state.remainingAmt, state.remainingFees) + + switch { + + // We have a terminal condition and no active shards, we are + // ready to exit. + case state.terminate && state.numShardsInFlight == 0: + // Find the first successful shard and return + // the preimage and route. + for _, a := range payment.HTLCs { + if a.Settle != nil { + return a.Settle.Preimage, &a.Route, nil + } } + // Payment failed. + return [32]byte{}, nil, *payment.FailureReason + + // If we either reached a terminal error condition (but had + // active shards still) or there is no remaining value to send, + // we'll wait for a shard outcome. + case state.terminate || state.remainingAmt == 0: + // We still have outstanding shards, so wait for a new + // outcome to be available before re-evaluating our + // state. + if err := shardHandler.waitForShard(); err != nil { + return [32]byte{}, nil, err + } + continue + } + + // Before we attempt any new shard, we'll check to see if + // either we've gone past the payment attempt timeout, or the + // router is exiting. In either case, we'll stop this payment + // attempt short. If a timeout is not applicable, timeoutChan + // will be nil. + select { + case <-p.timeoutChan: + log.Warnf("payment attempt not completed before " + + "timeout") + + // By marking the payment failed with the control + // tower, no further shards will be launched and we'll + // return with an error the moment all active shards + // have finished. + saveErr := p.router.cfg.Control.Fail( + p.paymentHash, channeldb.FailureReasonTimeout, + ) + if saveErr != nil { + return [32]byte{}, nil, saveErr + } + + continue + case <-p.router.quit: return [32]byte{}, nil, ErrRouterShuttingDown + + // Fall through if we haven't hit our time limit. + default: } - // In case of a payment failure, we use the error to decide - // whether we should retry. - if result.Error != nil { - log.Errorf("Attempt to send payment %x failed: %v", - p.payment.PaymentHash, result.Error) + // Create a new payment attempt from the given payment session. + rt, err := p.paySession.RequestRoute( + state.remainingAmt, state.remainingFees, + uint32(state.numShardsInFlight), uint32(p.currentHeight), + ) + if err != nil { + log.Warnf("Failed to find route for payment %x: %v", + p.paymentHash, err) - err = p.failAttempt(result.Error) - if err != nil { + routeErr, ok := err.(noRouteError) + if !ok { return [32]byte{}, nil, err } + // There is no route to try, and we have no active + // shards. This means that there is no way for us to + // send the payment, so mark it failed with no route. + if state.numShardsInFlight == 0 { + failureCode := routeErr.FailureReason() + log.Debugf("Marking payment %v permanently "+ + "failed with no route: %v", + p.paymentHash, failureCode) + + saveErr := p.router.cfg.Control.Fail( + p.paymentHash, failureCode, + ) + if saveErr != nil { + return [32]byte{}, nil, saveErr + } + + continue + } + + // We still have active shards, we'll wait for an + // outcome to be available before retrying. + if err := shardHandler.waitForShard(); err != nil { + return [32]byte{}, nil, err + } + continue + } + + // We found a route to try, launch a new shard. + attempt, outcome, err := shardHandler.launchShard(rt) + if err != nil { + return [32]byte{}, nil, err + } + + // If we encountered a non-critical error when launching the + // shard, handle it. + if outcome.err != nil { + log.Warnf("Failed to launch shard %v for "+ + "payment %v: %v", attempt.AttemptID, + p.paymentHash, outcome.err) + // We must inspect the error to know whether it was // critical or not, to decide whether we should // continue trying. - if err := p.handleSendError(result.Error); err != nil { + err := shardHandler.handleSendError( + attempt, outcome.err, + ) + if err != nil { return [32]byte{}, nil, err } - // Error was handled successfully, reset the attempt to - // indicate we want to make a new attempt. - p.attempt = nil + // Error was handled successfully, continue to make a + // new attempt. continue } - // We successfully got a payment result back from the switch. - log.Debugf("Payment %x succeeded with pid=%v", - p.payment.PaymentHash, p.attempt.AttemptID) - - // Report success to mission control. - err = p.router.cfg.MissionControl.ReportPaymentSuccess( - p.attempt.AttemptID, &p.attempt.Route, - ) - if err != nil { - log.Errorf("Error reporting payment success to mc: %v", - err) - } - - // In case of success we atomically store the db payment and - // move the payment to the success state. - err = p.router.cfg.Control.SettleAttempt( - p.payment.PaymentHash, p.attempt.AttemptID, - &channeldb.HTLCSettleInfo{ - Preimage: result.Preimage, - SettleTime: p.router.cfg.Clock.Now(), - }, - ) - if err != nil { - log.Errorf("Unable to succeed payment "+ - "attempt: %v", err) - return [32]byte{}, nil, err - } - - // Terminal state, return the preimage and the route - // taken. - return result.Preimage, &p.attempt.Route, nil + // Now that the shard was successfully sent, launch a go + // routine that will handle its result when its back. + shardHandler.collectResultAsync(attempt) } - } -// errorToPaymentFailure takes a path finding error and converts it into a -// payment-level failure. -func errorToPaymentFailure(err error) channeldb.FailureReason { - switch err { - case - errNoTlvPayload, - errNoPaymentAddr, - errNoPathFound, - errPrebuiltRouteTried: +// shardHandler holds what is necessary to send and collect the result of +// shards. +type shardHandler struct { + paymentHash lntypes.Hash + router *ChannelRouter - return channeldb.FailureReasonNoRoute + // shardErrors is a channel where errors collected by calling + // collectResultAsync will be delivered. These results are meant to be + // inspected by calling waitForShard or checkShards, and the channel + // doesn't need to be initiated if the caller is using the sync + // collectResult directly. + shardErrors chan error - case errInsufficientBalance: - return channeldb.FailureReasonInsufficientBalance - } - - return channeldb.FailureReasonError + // quit is closed to signal the sub goroutines of the payment lifecycle + // to stop. + quit chan struct{} + wg sync.WaitGroup } -// createNewPaymentAttempt creates and stores a new payment attempt to the -// database. -func (p *paymentLifecycle) createNewPaymentAttempt() (lnwire.ShortChannelID, - *lnwire.UpdateAddHTLC, error) { +// stop signals any active shard goroutine to exit and waits for them to exit. +func (p *shardHandler) stop() { + close(p.quit) + p.wg.Wait() +} - // Before we attempt this next payment, we'll check to see if either - // we've gone past the payment attempt timeout, or the router is - // exiting. In either case, we'll stop this payment attempt short. If a - // timeout is not applicable, timeoutChan will be nil. +// waitForShard blocks until any of the outstanding shards return. +func (p *shardHandler) waitForShard() error { select { - case <-p.timeoutChan: - // Mark the payment as failed because of the - // timeout. - err := p.router.cfg.Control.Fail( - p.payment.PaymentHash, channeldb.FailureReasonTimeout, - ) - if err != nil { - return lnwire.ShortChannelID{}, nil, err - } + case err := <-p.shardErrors: + return err - errStr := fmt.Sprintf("payment attempt not completed " + - "before timeout") - - return lnwire.ShortChannelID{}, nil, - newErr(ErrPaymentAttemptTimeout, errStr) + case <-p.quit: + return fmt.Errorf("shard handler quitting") case <-p.router.quit: - // The payment will be resumed from the current state - // after restart. - return lnwire.ShortChannelID{}, nil, ErrRouterShuttingDown - - default: - // Fall through if we haven't hit our time limit, or - // are expiring. + return ErrRouterShuttingDown } +} - // Create a new payment attempt from the given payment session. - rt, err := p.paySession.RequestRoute( - p.payment, uint32(p.currentHeight), p.finalCLTVDelta, +// checkShards is a non-blocking method that check if any shards has finished +// their execution. +func (p *shardHandler) checkShards() error { + for { + select { + case err := <-p.shardErrors: + if err != nil { + return err + } + + case <-p.quit: + return fmt.Errorf("shard handler quitting") + + case <-p.router.quit: + return ErrRouterShuttingDown + + default: + return nil + } + } +} + +// launchOutcome is a type returned from launchShard that indicates whether the +// shard was successfully send onto the network. +type launchOutcome struct { + // err is non-nil if a non-critical error was encountered when trying + // to send the shard, and we successfully updated the control tower to + // reflect this error. This can be errors like not enough local + // balance for the given route etc. + err error +} + +// launchShard creates and sends an HTLC attempt along the given route, +// registering it with the control tower before sending it. It returns the +// HTLCAttemptInfo that was created for the shard, along with a launchOutcome. +// The launchOutcome is used to indicate whether the attempt was successfully +// sent. If the launchOutcome wraps a non-nil error, it means that the attempt +// was not sent onto the network, so no result will be available in the future +// for it. +func (p *shardHandler) launchShard(rt *route.Route) (*channeldb.HTLCAttemptInfo, + *launchOutcome, error) { + + // Using the route received from the payment session, create a new + // shard to send. + firstHop, htlcAdd, attempt, err := p.createNewPaymentAttempt( + rt, ) if err != nil { - log.Warnf("Failed to find route for payment %x: %v", - p.payment.PaymentHash, err) - - // Convert error to payment-level failure. - failure := errorToPaymentFailure(err) - - // If we're unable to successfully make a payment using - // any of the routes we've found, then mark the payment - // as permanently failed. - saveErr := p.router.cfg.Control.Fail( - p.payment.PaymentHash, failure, - ) - if saveErr != nil { - return lnwire.ShortChannelID{}, nil, saveErr - } - - // If there was an error already recorded for this - // payment, we'll return that. - if p.lastError != nil { - return lnwire.ShortChannelID{}, nil, - errNoRoute{lastError: p.lastError} - } - // Terminal state, return. - return lnwire.ShortChannelID{}, nil, err + return nil, nil, err } + // Before sending this HTLC to the switch, we checkpoint the fresh + // paymentID and route to the DB. This lets us know on startup the ID + // of the payment that we attempted to send, such that we can query the + // Switch for its whereabouts. The route is needed to handle the result + // when it eventually comes back. + err = p.router.cfg.Control.RegisterAttempt(p.paymentHash, attempt) + if err != nil { + return nil, nil, err + } + + // Now that the attempt is created and checkpointed to the DB, we send + // it. + sendErr := p.sendPaymentAttempt(attempt, firstHop, htlcAdd) + if sendErr != nil { + // TODO(joostjager): Distinguish unexpected internal errors + // from real send errors. + err := p.failAttempt(attempt, sendErr) + if err != nil { + return nil, nil, err + } + + // Return a launchOutcome indicating the shard failed. + return attempt, &launchOutcome{ + err: sendErr, + }, nil + } + + return attempt, &launchOutcome{}, nil +} + +// shardResult holds the resulting outcome of a shard sent. +type shardResult struct { + // preimage is the payment preimage in case of a settled HTLC. Only set + // if err is non-nil. + preimage lntypes.Preimage + + // err indicates that the shard failed. + err error +} + +// collectResultAsync launches a goroutine that will wait for the result of the +// given HTLC attempt to be available then handle its result. Note that it will +// fail the payment with the control tower if a terminal error is encountered. +func (p *shardHandler) collectResultAsync(attempt *channeldb.HTLCAttemptInfo) { + p.wg.Add(1) + go func() { + defer p.wg.Done() + + // Block until the result is available. + result, err := p.collectResult(attempt) + if err != nil { + if err != ErrRouterShuttingDown && + err != htlcswitch.ErrSwitchExiting { + + log.Errorf("Error collecting result for "+ + "shard %v for payment %v: %v", + attempt.AttemptID, p.paymentHash, err) + } + + select { + case p.shardErrors <- err: + case <-p.router.quit: + case <-p.quit: + } + return + } + + // If a non-critical error was encountered handle it and mark + // the payment failed if the failure was terminal. + if result.err != nil { + err := p.handleSendError(attempt, result.err) + if err != nil { + select { + case p.shardErrors <- err: + case <-p.router.quit: + case <-p.quit: + } + return + } + } + + select { + case p.shardErrors <- nil: + case <-p.router.quit: + case <-p.quit: + } + }() +} + +// collectResult waits for the result for the given attempt to be available +// from the Switch, then records the attempt outcome with the control tower. A +// shardResult is returned, indicating the final outcome of this HTLC attempt. +func (p *shardHandler) collectResult(attempt *channeldb.HTLCAttemptInfo) ( + *shardResult, error) { + + // Regenerate the circuit for this attempt. + _, circuit, err := generateSphinxPacket( + &attempt.Route, p.paymentHash[:], + attempt.SessionKey, + ) + if err != nil { + return nil, err + } + + // Using the created circuit, initialize the error decrypter so we can + // parse+decode any failures incurred by this payment within the + // switch. + errorDecryptor := &htlcswitch.SphinxErrorDecrypter{ + OnionErrorDecrypter: sphinx.NewOnionErrorDecrypter(circuit), + } + + // Now ask the switch to return the result of the payment when + // available. + resultChan, err := p.router.cfg.Payer.GetPaymentResult( + attempt.AttemptID, p.paymentHash, errorDecryptor, + ) + switch { + + // If this attempt ID is unknown to the Switch, it means it was never + // checkpointed and forwarded by the switch before a restart. In this + // case we can safely send a new payment attempt, and wait for its + // result to be available. + case err == htlcswitch.ErrPaymentIDNotFound: + log.Debugf("Payment ID %v for hash %x not found in "+ + "the Switch, retrying.", attempt.AttemptID, + p.paymentHash) + + cErr := p.failAttempt(attempt, err) + if cErr != nil { + return nil, cErr + } + + return &shardResult{ + err: err, + }, nil + + // A critical, unexpected error was encountered. + case err != nil: + log.Errorf("Failed getting result for attemptID %d "+ + "from switch: %v", attempt.AttemptID, err) + + return nil, err + } + + // The switch knows about this payment, we'll wait for a result to be + // available. + var ( + result *htlcswitch.PaymentResult + ok bool + ) + + select { + case result, ok = <-resultChan: + if !ok { + return nil, htlcswitch.ErrSwitchExiting + } + + case <-p.router.quit: + return nil, ErrRouterShuttingDown + + case <-p.quit: + return nil, fmt.Errorf("shard handler exiting") + } + + // In case of a payment failure, fail the attempt with the control + // tower and return. + if result.Error != nil { + err := p.failAttempt(attempt, result.Error) + if err != nil { + return nil, err + } + + return &shardResult{ + err: result.Error, + }, nil + } + + // We successfully got a payment result back from the switch. + log.Debugf("Payment %x succeeded with pid=%v", + p.paymentHash, attempt.AttemptID) + + // Report success to mission control. + err = p.router.cfg.MissionControl.ReportPaymentSuccess( + attempt.AttemptID, &attempt.Route, + ) + if err != nil { + log.Errorf("Error reporting payment success to mc: %v", + err) + } + + // In case of success we atomically store settle result to the DB move + // the shard to the settled state. + err = p.router.cfg.Control.SettleAttempt( + p.paymentHash, attempt.AttemptID, + &channeldb.HTLCSettleInfo{ + Preimage: result.Preimage, + SettleTime: p.router.cfg.Clock.Now(), + }, + ) + if err != nil { + log.Errorf("Unable to succeed payment attempt: %v", err) + return nil, err + } + + return &shardResult{ + preimage: result.Preimage, + }, nil +} + +// createNewPaymentAttempt creates a new payment attempt from the given route. +func (p *shardHandler) createNewPaymentAttempt(rt *route.Route) ( + lnwire.ShortChannelID, *lnwire.UpdateAddHTLC, + *channeldb.HTLCAttemptInfo, error) { + // Generate a new key to be used for this attempt. sessionKey, err := generateNewSessionKey() if err != nil { - return lnwire.ShortChannelID{}, nil, err + return lnwire.ShortChannelID{}, nil, nil, err } // Generate the raw encoded sphinx packet to be included along // with the htlcAdd message that we send directly to the // switch. - onionBlob, c, err := generateSphinxPacket( - rt, p.payment.PaymentHash[:], sessionKey, + onionBlob, _, err := generateSphinxPacket( + rt, p.paymentHash[:], sessionKey, ) - - // With SendToRoute, it can happen that the route exceeds protocol - // constraints. Mark the payment as failed with an internal error. - if err == route.ErrMaxRouteHopsExceeded || - err == sphinx.ErrMaxRoutingInfoSizeExceeded { - - log.Debugf("Invalid route provided for payment %x: %v", - p.payment.PaymentHash, err) - - controlErr := p.router.cfg.Control.Fail( - p.payment.PaymentHash, channeldb.FailureReasonError, - ) - if controlErr != nil { - return lnwire.ShortChannelID{}, nil, controlErr - } - } - - // In any case, don't continue if there is an error. if err != nil { - return lnwire.ShortChannelID{}, nil, err + return lnwire.ShortChannelID{}, nil, nil, err } - // Update our cached circuit with the newly generated - // one. - p.circuit = c - // Craft an HTLC packet to send to the layer 2 switch. The // metadata within this packet will be used to route the // payment through the network, starting with the first-hop. htlcAdd := &lnwire.UpdateAddHTLC{ Amount: rt.TotalAmount, Expiry: rt.TotalTimeLock, - PaymentHash: p.payment.PaymentHash, + PaymentHash: p.paymentHash, } copy(htlcAdd.OnionBlob[:], onionBlob) @@ -356,40 +617,30 @@ func (p *paymentLifecycle) createNewPaymentAttempt() (lnwire.ShortChannelID, // this HTLC. attemptID, err := p.router.cfg.NextPaymentID() if err != nil { - return lnwire.ShortChannelID{}, nil, err + return lnwire.ShortChannelID{}, nil, nil, err } // We now have all the information needed to populate // the current attempt information. - p.attempt = &channeldb.HTLCAttemptInfo{ + attempt := &channeldb.HTLCAttemptInfo{ AttemptID: attemptID, AttemptTime: p.router.cfg.Clock.Now(), SessionKey: sessionKey, Route: *rt, } - // Before sending this HTLC to the switch, we checkpoint the - // fresh attemptID and route to the DB. This lets us know on - // startup the ID of the payment that we attempted to send, - // such that we can query the Switch for its whereabouts. The - // route is needed to handle the result when it eventually - // comes back. - err = p.router.cfg.Control.RegisterAttempt(p.payment.PaymentHash, p.attempt) - if err != nil { - return lnwire.ShortChannelID{}, nil, err - } - - return firstHop, htlcAdd, nil + return firstHop, htlcAdd, attempt, nil } // sendPaymentAttempt attempts to send the current attempt to the switch. -func (p *paymentLifecycle) sendPaymentAttempt(firstHop lnwire.ShortChannelID, +func (p *shardHandler) sendPaymentAttempt( + attempt *channeldb.HTLCAttemptInfo, firstHop lnwire.ShortChannelID, htlcAdd *lnwire.UpdateAddHTLC) error { log.Tracef("Attempting to send payment %x (pid=%v), "+ - "using route: %v", p.payment.PaymentHash, p.attempt.AttemptID, + "using route: %v", p.paymentHash, attempt.AttemptID, newLogClosure(func() string { - return spew.Sdump(p.attempt.Route) + return spew.Sdump(attempt.Route) }), ) @@ -398,63 +649,60 @@ func (p *paymentLifecycle) sendPaymentAttempt(firstHop lnwire.ShortChannelID, // such that we can resume waiting for the result after a // restart. err := p.router.cfg.Payer.SendHTLC( - firstHop, p.attempt.AttemptID, htlcAdd, + firstHop, attempt.AttemptID, htlcAdd, ) if err != nil { log.Errorf("Failed sending attempt %d for payment "+ - "%x to switch: %v", p.attempt.AttemptID, - p.payment.PaymentHash, err) + "%x to switch: %v", attempt.AttemptID, + p.paymentHash, err) return err } log.Debugf("Payment %x (pid=%v) successfully sent to switch, route: %v", - p.payment.PaymentHash, p.attempt.AttemptID, &p.attempt.Route) + p.paymentHash, attempt.AttemptID, &attempt.Route) return nil } // handleSendError inspects the given error from the Switch and determines -// whether we should make another payment attempt. -func (p *paymentLifecycle) handleSendError(sendErr error) error { +// whether we should make another payment attempt, or if it should be +// considered a terminal error. Terminal errors will be recorded with the +// control tower. +func (p *shardHandler) handleSendError(attempt *channeldb.HTLCAttemptInfo, + sendErr error) error { reason := p.router.processSendError( - p.attempt.AttemptID, &p.attempt.Route, sendErr, + attempt.AttemptID, &attempt.Route, sendErr, ) if reason == nil { - // Save the forwarding error so it can be returned if - // this turns out to be the last attempt. - p.lastError = sendErr - return nil } log.Debugf("Payment %x failed: final_outcome=%v, raw_err=%v", - p.payment.PaymentHash, *reason, sendErr) + p.paymentHash, *reason, sendErr) - // Mark the payment failed with no route. - // - // TODO(halseth): make payment codes for the actual reason we don't - // continue path finding. - err := p.router.cfg.Control.Fail( - p.payment.PaymentHash, *reason, - ) + err := p.router.cfg.Control.Fail(p.paymentHash, *reason) if err != nil { return err } - // Terminal state, return the error we encountered. - return sendErr + return nil } // failAttempt calls control tower to fail the current payment attempt. -func (p *paymentLifecycle) failAttempt(sendError error) error { +func (p *shardHandler) failAttempt(attempt *channeldb.HTLCAttemptInfo, + sendError error) error { + + log.Warnf("Attempt %v for payment %v failed: %v", attempt.AttemptID, + p.paymentHash, sendError) + failInfo := marshallError( sendError, p.router.cfg.Clock.Now(), ) return p.router.cfg.Control.FailAttempt( - p.payment.PaymentHash, p.attempt.AttemptID, + p.paymentHash, attempt.AttemptID, failInfo, ) } diff --git a/routing/payment_lifecycle_test.go b/routing/payment_lifecycle_test.go new file mode 100644 index 000000000..e83ac17f4 --- /dev/null +++ b/routing/payment_lifecycle_test.go @@ -0,0 +1,898 @@ +package routing + +import ( + "crypto/rand" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/btcsuite/btcutil" + "github.com/go-errors/errors" + "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/htlcswitch" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" +) + +const stepTimeout = 5 * time.Second + +// createTestRoute builds a route a->b->c paying the given amt to c. +func createTestRoute(amt lnwire.MilliSatoshi, + aliasMap map[string]route.Vertex) (*route.Route, error) { + + hopFee := lnwire.NewMSatFromSatoshis(3) + hop1 := aliasMap["b"] + hop2 := aliasMap["c"] + hops := []*route.Hop{ + { + ChannelID: 1, + PubKeyBytes: hop1, + LegacyPayload: true, + AmtToForward: amt + hopFee, + }, + { + ChannelID: 2, + PubKeyBytes: hop2, + LegacyPayload: true, + AmtToForward: amt, + }, + } + + // We create a simple route that we will supply every time the router + // requests one. + return route.NewRouteFromHops( + amt+2*hopFee, 100, aliasMap["a"], hops, + ) +} + +// TestRouterPaymentStateMachine tests that the router interacts as expected +// with the ControlTower during a payment lifecycle, such that it payment +// attempts are not sent twice to the switch, and results are handled after a +// restart. +func TestRouterPaymentStateMachine(t *testing.T) { + t.Parallel() + + const startingBlockHeight = 101 + + // Setup two simple channels such that we can mock sending along this + // route. + chanCapSat := btcutil.Amount(100000) + testChannels := []*testChannel{ + symmetricTestChannel("a", "b", chanCapSat, &testChannelPolicy{ + Expiry: 144, + FeeRate: 400, + MinHTLC: 1, + MaxHTLC: lnwire.NewMSatFromSatoshis(chanCapSat), + }, 1), + symmetricTestChannel("b", "c", chanCapSat, &testChannelPolicy{ + Expiry: 144, + FeeRate: 400, + MinHTLC: 1, + MaxHTLC: lnwire.NewMSatFromSatoshis(chanCapSat), + }, 2), + } + + testGraph, err := createTestGraphFromChannels(testChannels, "a") + if err != nil { + t.Fatalf("unable to create graph: %v", err) + } + defer testGraph.cleanUp() + + paymentAmt := lnwire.NewMSatFromSatoshis(1000) + + // We create a simple route that we will supply every time the router + // requests one. + rt, err := createTestRoute(paymentAmt, testGraph.aliasMap) + if err != nil { + t.Fatalf("unable to create route: %v", err) + } + + shard, err := createTestRoute(paymentAmt/4, testGraph.aliasMap) + if err != nil { + t.Fatalf("unable to create route: %v", err) + } + + // A payment state machine test case consists of several ordered steps, + // that we use for driving the scenario. + type testCase struct { + // steps is a list of steps to perform during the testcase. + steps []string + + // routes is the sequence of routes we will provide to the + // router when it requests a new route. + routes []*route.Route + } + + const ( + // routerInitPayment is a test step where we expect the router + // to call the InitPayment method on the control tower. + routerInitPayment = "Router:init-payment" + + // routerRegisterAttempt is a test step where we expect the + // router to call the RegisterAttempt method on the control + // tower. + routerRegisterAttempt = "Router:register-attempt" + + // routerSettleAttempt is a test step where we expect the + // router to call the SettleAttempt method on the control + // tower. + routerSettleAttempt = "Router:settle-attempt" + + // routerFailAttempt is a test step where we expect the router + // to call the FailAttempt method on the control tower. + routerFailAttempt = "Router:fail-attempt" + + // routerFailPayment is a test step where we expect the router + // to call the Fail method on the control tower. + routerFailPayment = "Router:fail-payment" + + // sendToSwitchSuccess is a step where we expect the router to + // call send the payment attempt to the switch, and we will + // respond with a non-error, indicating that the payment + // attempt was successfully forwarded. + sendToSwitchSuccess = "SendToSwitch:success" + + // sendToSwitchResultFailure is a step where we expect the + // router to send the payment attempt to the switch, and we + // will respond with a forwarding error. This can happen when + // forwarding fail on our local links. + sendToSwitchResultFailure = "SendToSwitch:failure" + + // getPaymentResultSuccess is a test step where we expect the + // router to call the GetPaymentResult method, and we will + // respond with a successful payment result. + getPaymentResultSuccess = "GetPaymentResult:success" + + // getPaymentResultTempFailure is a test step where we expect the + // router to call the GetPaymentResult method, and we will + // respond with a forwarding error, expecting the router to retry. + getPaymentResultTempFailure = "GetPaymentResult:temp-failure" + + // getPaymentResultTerminalFailure is a test step where we + // expect the router to call the GetPaymentResult method, and + // we will respond with a terminal error, expecting the router + // to stop making payment attempts. + getPaymentResultTerminalFailure = "GetPaymentResult:terminal-failure" + + // resendPayment is a test step where we manually try to resend + // the same payment, making sure the router responds with an + // error indicating that it is already in flight. + resendPayment = "ResendPayment" + + // startRouter is a step where we manually start the router, + // used to test that it automatically will resume payments at + // startup. + startRouter = "StartRouter" + + // stopRouter is a test step where we manually make the router + // shut down. + stopRouter = "StopRouter" + + // paymentSuccess is a step where assert that we receive a + // successful result for the original payment made. + paymentSuccess = "PaymentSuccess" + + // paymentError is a step where assert that we receive an error + // for the original payment made. + paymentError = "PaymentError" + + // resentPaymentSuccess is a step where assert that we receive + // a successful result for a payment that was resent. + resentPaymentSuccess = "ResentPaymentSuccess" + + // resentPaymentError is a step where assert that we receive an + // error for a payment that was resent. + resentPaymentError = "ResentPaymentError" + ) + + tests := []testCase{ + { + // Tests a normal payment flow that succeeds. + steps: []string{ + routerInitPayment, + routerRegisterAttempt, + sendToSwitchSuccess, + getPaymentResultSuccess, + routerSettleAttempt, + paymentSuccess, + }, + routes: []*route.Route{rt}, + }, + { + // A payment flow with a failure on the first attempt, + // but that succeeds on the second attempt. + steps: []string{ + routerInitPayment, + routerRegisterAttempt, + sendToSwitchSuccess, + + // Make the first sent attempt fail. + getPaymentResultTempFailure, + routerFailAttempt, + + // The router should retry. + routerRegisterAttempt, + sendToSwitchSuccess, + + // Make the second sent attempt succeed. + getPaymentResultSuccess, + routerSettleAttempt, + paymentSuccess, + }, + routes: []*route.Route{rt, rt}, + }, + { + // A payment flow with a forwarding failure first time + // sending to the switch, but that succeeds on the + // second attempt. + steps: []string{ + routerInitPayment, + routerRegisterAttempt, + + // Make the first sent attempt fail. + sendToSwitchResultFailure, + routerFailAttempt, + + // The router should retry. + routerRegisterAttempt, + sendToSwitchSuccess, + + // Make the second sent attempt succeed. + getPaymentResultSuccess, + routerSettleAttempt, + paymentSuccess, + }, + routes: []*route.Route{rt, rt}, + }, + { + // A payment that fails on the first attempt, and has + // only one route available to try. It will therefore + // fail permanently. + steps: []string{ + routerInitPayment, + routerRegisterAttempt, + sendToSwitchSuccess, + + // Make the first sent attempt fail. + getPaymentResultTempFailure, + routerFailAttempt, + + // Since there are no more routes to try, the + // payment should fail. + routerFailPayment, + paymentError, + }, + routes: []*route.Route{rt}, + }, + { + // We expect the payment to fail immediately if we have + // no routes to try. + steps: []string{ + routerInitPayment, + routerFailPayment, + paymentError, + }, + routes: []*route.Route{}, + }, + { + // A normal payment flow, where we attempt to resend + // the same payment after each step. This ensures that + // the router don't attempt to resend a payment already + // in flight. + steps: []string{ + routerInitPayment, + routerRegisterAttempt, + + // Manually resend the payment, the router + // should attempt to init with the control + // tower, but fail since it is already in + // flight. + resendPayment, + routerInitPayment, + resentPaymentError, + + // The original payment should proceed as + // normal. + sendToSwitchSuccess, + + // Again resend the payment and assert it's not + // allowed. + resendPayment, + routerInitPayment, + resentPaymentError, + + // Notify about a success for the original + // payment. + getPaymentResultSuccess, + routerSettleAttempt, + + // Now that the original payment finished, + // resend it again to ensure this is not + // allowed. + resendPayment, + routerInitPayment, + resentPaymentError, + paymentSuccess, + }, + routes: []*route.Route{rt}, + }, + { + // Tests that the router is able to handle the + // receieved payment result after a restart. + steps: []string{ + routerInitPayment, + routerRegisterAttempt, + sendToSwitchSuccess, + + // Shut down the router. The original caller + // should get notified about this. + stopRouter, + paymentError, + + // Start the router again, and ensure the + // router registers the success with the + // control tower. + startRouter, + getPaymentResultSuccess, + routerSettleAttempt, + }, + routes: []*route.Route{rt}, + }, + { + // Tests that we are allowed to resend a payment after + // it has permanently failed. + steps: []string{ + routerInitPayment, + routerRegisterAttempt, + sendToSwitchSuccess, + + // Resending the payment at this stage should + // not be allowed. + resendPayment, + routerInitPayment, + resentPaymentError, + + // Make the first attempt fail. + getPaymentResultTempFailure, + routerFailAttempt, + + // Since we have no more routes to try, the + // original payment should fail. + routerFailPayment, + paymentError, + + // Now resend the payment again. This should be + // allowed, since the payment has failed. + resendPayment, + routerInitPayment, + routerRegisterAttempt, + sendToSwitchSuccess, + getPaymentResultSuccess, + routerSettleAttempt, + resentPaymentSuccess, + }, + routes: []*route.Route{rt}, + }, + + // ===================================== + // || MPP scenarios || + // ===================================== + { + // Tests a simple successful MP payment of 4 shards. + steps: []string{ + routerInitPayment, + + // shard 0 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 1 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 2 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 3 + routerRegisterAttempt, + sendToSwitchSuccess, + + // All shards succeed. + getPaymentResultSuccess, + getPaymentResultSuccess, + getPaymentResultSuccess, + getPaymentResultSuccess, + + // Router should settle them all. + routerSettleAttempt, + routerSettleAttempt, + routerSettleAttempt, + routerSettleAttempt, + + // And the final result is obviously + // successful. + paymentSuccess, + }, + routes: []*route.Route{shard, shard, shard, shard}, + }, + { + // An MP payment scenario where we need several extra + // attempts before the payment finally settle. + steps: []string{ + routerInitPayment, + + // shard 0 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 1 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 2 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 3 + routerRegisterAttempt, + sendToSwitchSuccess, + + // First two shards fail, two new ones are sent. + getPaymentResultTempFailure, + getPaymentResultTempFailure, + routerFailAttempt, + routerFailAttempt, + + routerRegisterAttempt, + sendToSwitchSuccess, + routerRegisterAttempt, + sendToSwitchSuccess, + + // The four shards settle. + getPaymentResultSuccess, + getPaymentResultSuccess, + getPaymentResultSuccess, + getPaymentResultSuccess, + routerSettleAttempt, + routerSettleAttempt, + routerSettleAttempt, + routerSettleAttempt, + + // Overall payment succeeds. + paymentSuccess, + }, + routes: []*route.Route{ + shard, shard, shard, shard, shard, shard, + }, + }, + { + // An MP payment scenario where 3 of the shards fail. + // However the last shard settle, which means we get + // the preimage and should consider the overall payment + // a success. + steps: []string{ + routerInitPayment, + + // shard 0 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 1 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 2 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 3 + routerRegisterAttempt, + sendToSwitchSuccess, + + // 3 shards fail, and should be failed by the + // router. + getPaymentResultTempFailure, + getPaymentResultTempFailure, + getPaymentResultTempFailure, + routerFailAttempt, + routerFailAttempt, + routerFailAttempt, + + // The fourth shard succeed against all odds, + // making the overall payment succeed. + getPaymentResultSuccess, + routerSettleAttempt, + paymentSuccess, + }, + routes: []*route.Route{shard, shard, shard, shard}, + }, + { + // An MP payment scenario a shard fail with a terminal + // error, causing the router to stop attempting. + steps: []string{ + routerInitPayment, + + // shard 0 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 1 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 2 + routerRegisterAttempt, + sendToSwitchSuccess, + + // shard 3 + routerRegisterAttempt, + sendToSwitchSuccess, + + // The first shard fail with a terminal error. + getPaymentResultTerminalFailure, + routerFailAttempt, + routerFailPayment, + + // Remaining 3 shards fail. + getPaymentResultTempFailure, + getPaymentResultTempFailure, + getPaymentResultTempFailure, + routerFailAttempt, + routerFailAttempt, + routerFailAttempt, + + // Payment fails. + paymentError, + }, + routes: []*route.Route{ + shard, shard, shard, shard, shard, shard, + }, + }, + } + + // Create a mock control tower with channels set up, that we use to + // synchronize and listen for events. + control := makeMockControlTower() + control.init = make(chan initArgs, 20) + control.registerAttempt = make(chan registerAttemptArgs, 20) + control.settleAttempt = make(chan settleAttemptArgs, 20) + control.failAttempt = make(chan failAttemptArgs, 20) + control.failPayment = make(chan failPaymentArgs, 20) + control.fetchInFlight = make(chan struct{}, 20) + + quit := make(chan struct{}) + defer close(quit) + + // setupRouter is a helper method that creates and starts the router in + // the desired configuration for this test. + setupRouter := func() (*ChannelRouter, chan error, + chan *htlcswitch.PaymentResult, chan error) { + + chain := newMockChain(startingBlockHeight) + chainView := newMockChainView(chain) + + // We set uo the use the following channels and a mock Payer to + // synchonize with the interaction to the Switch. + sendResult := make(chan error) + paymentResultErr := make(chan error) + paymentResult := make(chan *htlcswitch.PaymentResult) + + payer := &mockPayer{ + sendResult: sendResult, + paymentResult: paymentResult, + paymentResultErr: paymentResultErr, + } + + router, err := New(Config{ + Graph: testGraph.graph, + Chain: chain, + ChainView: chainView, + Control: control, + SessionSource: &mockPaymentSessionSource{}, + MissionControl: &mockMissionControl{}, + Payer: payer, + ChannelPruneExpiry: time.Hour * 24, + GraphPruneInterval: time.Hour * 2, + QueryBandwidth: func(e *channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi { + return lnwire.NewMSatFromSatoshis(e.Capacity) + }, + NextPaymentID: func() (uint64, error) { + next := atomic.AddUint64(&uniquePaymentID, 1) + return next, nil + }, + Clock: clock.NewTestClock(time.Unix(1, 0)), + }) + if err != nil { + t.Fatalf("unable to create router %v", err) + } + + // On startup, the router should fetch all pending payments + // from the ControlTower, so assert that here. + errCh := make(chan error) + go func() { + close(errCh) + select { + case <-control.fetchInFlight: + return + case <-time.After(1 * time.Second): + errCh <- errors.New("router did not fetch in flight " + + "payments") + } + }() + + if err := router.Start(); err != nil { + t.Fatalf("unable to start router: %v", err) + } + + select { + case err := <-errCh: + if err != nil { + t.Fatalf("error in anonymous goroutine: %s", err) + } + case <-time.After(1 * time.Second): + t.Fatalf("did not fetch in flight payments at startup") + } + + return router, sendResult, paymentResult, paymentResultErr + } + + router, sendResult, getPaymentResult, getPaymentResultErr := setupRouter() + defer func() { + if err := router.Stop(); err != nil { + t.Fatal(err) + } + }() + + for _, test := range tests { + // Craft a LightningPayment struct. + var preImage lntypes.Preimage + if _, err := rand.Read(preImage[:]); err != nil { + t.Fatalf("unable to generate preimage") + } + + payHash := preImage.Hash() + + payment := LightningPayment{ + Target: testGraph.aliasMap["c"], + Amount: paymentAmt, + FeeLimit: noFeeLimit, + PaymentHash: payHash, + } + + router.cfg.SessionSource = &mockPaymentSessionSource{ + routes: test.routes, + } + + router.cfg.MissionControl = &mockMissionControl{} + + // Send the payment. Since this is new payment hash, the + // information should be registered with the ControlTower. + paymentResult := make(chan error) + go func() { + _, _, err := router.SendPayment(&payment) + paymentResult <- err + }() + + var resendResult chan error + for _, step := range test.steps { + switch step { + + case routerInitPayment: + var args initArgs + select { + case args = <-control.init: + case <-time.After(stepTimeout): + t.Fatalf("no init payment with control") + } + + if args.c == nil { + t.Fatalf("expected non-nil CreationInfo") + } + + // In this step we expect the router to make a call to + // register a new attempt with the ControlTower. + case routerRegisterAttempt: + var args registerAttemptArgs + select { + case args = <-control.registerAttempt: + case <-time.After(stepTimeout): + t.Fatalf("attempt not registered " + + "with control") + } + + if args.a == nil { + t.Fatalf("expected non-nil AttemptInfo") + } + + // In this step we expect the router to call the + // ControlTower's SettleAttempt method with the preimage. + case routerSettleAttempt: + select { + case <-control.settleAttempt: + case <-time.After(stepTimeout): + t.Fatalf("attempt settle not " + + "registered with control") + } + + // In this step we expect the router to call the + // ControlTower's FailAttempt method with a HTLC fail + // info. + case routerFailAttempt: + select { + case <-control.failAttempt: + case <-time.After(stepTimeout): + t.Fatalf("attempt fail not " + + "registered with control") + } + + // In this step we expect the router to call the + // ControlTower's Fail method, to indicate that the + // payment failed. + case routerFailPayment: + select { + case <-control.failPayment: + case <-time.After(stepTimeout): + t.Fatalf("payment fail not " + + "registered with control") + } + + // In this step we expect the SendToSwitch method to be + // called, and we respond with a nil-error. + case sendToSwitchSuccess: + select { + case sendResult <- nil: + case <-time.After(stepTimeout): + t.Fatalf("unable to send result") + } + + // In this step we expect the SendToSwitch method to be + // called, and we respond with a forwarding error + case sendToSwitchResultFailure: + select { + case sendResult <- htlcswitch.NewForwardingError( + &lnwire.FailTemporaryChannelFailure{}, + 1, + ): + case <-time.After(stepTimeout): + t.Fatalf("unable to send result") + } + + // In this step we expect the GetPaymentResult method + // to be called, and we respond with the preimage to + // complete the payment. + case getPaymentResultSuccess: + select { + case getPaymentResult <- &htlcswitch.PaymentResult{ + Preimage: preImage, + }: + case <-time.After(stepTimeout): + t.Fatalf("unable to send result") + } + + // In this state we expect the GetPaymentResult method + // to be called, and we respond with a forwarding + // error, indicating that the router should retry. + case getPaymentResultTempFailure: + failure := htlcswitch.NewForwardingError( + &lnwire.FailTemporaryChannelFailure{}, + 1, + ) + + select { + case getPaymentResult <- &htlcswitch.PaymentResult{ + Error: failure, + }: + case <-time.After(stepTimeout): + t.Fatalf("unable to get result") + } + + // In this state we expect the router to call the + // GetPaymentResult method, and we will respond with a + // terminal error, indiating the router should stop + // making payment attempts. + case getPaymentResultTerminalFailure: + failure := htlcswitch.NewForwardingError( + &lnwire.FailIncorrectDetails{}, + 1, + ) + + select { + case getPaymentResult <- &htlcswitch.PaymentResult{ + Error: failure, + }: + case <-time.After(stepTimeout): + t.Fatalf("unable to get result") + } + + // In this step we manually try to resend the same + // payment, making sure the router responds with an + // error indicating that it is already in flight. + case resendPayment: + resendResult = make(chan error) + go func() { + _, _, err := router.SendPayment(&payment) + resendResult <- err + }() + + // In this step we manually stop the router. + case stopRouter: + select { + case getPaymentResultErr <- fmt.Errorf( + "shutting down"): + case <-time.After(stepTimeout): + t.Fatalf("unable to send payment " + + "result error") + } + + if err := router.Stop(); err != nil { + t.Fatalf("unable to restart: %v", err) + } + + // In this step we manually start the router. + case startRouter: + router, sendResult, getPaymentResult, + getPaymentResultErr = setupRouter() + + // In this state we expect to receive an error for the + // original payment made. + case paymentError: + select { + case err := <-paymentResult: + if err == nil { + t.Fatalf("expected error") + } + + case <-time.After(stepTimeout): + t.Fatalf("got no payment result") + } + + // In this state we expect the original payment to + // succeed. + case paymentSuccess: + select { + case err := <-paymentResult: + if err != nil { + t.Fatalf("did not expect "+ + "error %v", err) + } + + case <-time.After(stepTimeout): + t.Fatalf("got no payment result") + } + + // In this state we expect to receive an error for the + // resent payment made. + case resentPaymentError: + select { + case err := <-resendResult: + if err == nil { + t.Fatalf("expected error") + } + + case <-time.After(stepTimeout): + t.Fatalf("got no payment result") + } + + // In this state we expect the resent payment to + // succeed. + case resentPaymentSuccess: + select { + case err := <-resendResult: + if err != nil { + t.Fatalf("did not expect error %v", err) + } + + case <-time.After(stepTimeout): + t.Fatalf("got no payment result") + } + + default: + t.Fatalf("unknown step %v", step) + } + } + } +} diff --git a/routing/payment_session.go b/routing/payment_session.go index 47732e01a..6a597ff01 100644 --- a/routing/payment_session.go +++ b/routing/payment_session.go @@ -1,8 +1,6 @@ package routing import ( - "errors" - "github.com/lightningnetwork/lnd/channeldb" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/routing/route" @@ -12,20 +10,89 @@ import ( // to prevent an HTLC being failed if some blocks are mined while it's in-flight. const BlockPadding uint16 = 3 -var ( - // errPrebuiltRouteTried is returned when the single pre-built route - // failed and there is nothing more we can do. - errPrebuiltRouteTried = errors.New("pre-built route already tried") +// noRouteError encodes a non-critical error encountered during path finding. +type noRouteError uint8 + +const ( + // errNoTlvPayload is returned when the destination hop does not support + // a tlv payload. + errNoTlvPayload noRouteError = iota + + // errNoPaymentAddr is returned when the destination hop does not + // support payment addresses. + errNoPaymentAddr + + // errNoPathFound is returned when a path to the target destination does + // not exist in the graph. + errNoPathFound + + // errInsufficientLocalBalance is returned when none of the local + // channels have enough balance for the payment. + errInsufficientBalance + + // errEmptyPaySession is returned when the empty payment session is + // queried for a route. + errEmptyPaySession ) +// Error returns the string representation of the noRouteError +func (e noRouteError) Error() string { + switch e { + case errNoTlvPayload: + return "destination hop doesn't understand new TLV payloads" + + case errNoPaymentAddr: + return "destination hop doesn't understand payment addresses" + + case errNoPathFound: + return "unable to find a path to destination" + + case errEmptyPaySession: + return "empty payment session" + + case errInsufficientBalance: + return "insufficient local balance" + + default: + return "unknown no-route error" + } +} + +// FailureReason converts a path finding error into a payment-level failure. +func (e noRouteError) FailureReason() channeldb.FailureReason { + switch e { + case + errNoTlvPayload, + errNoPaymentAddr, + errNoPathFound, + errEmptyPaySession: + + return channeldb.FailureReasonNoRoute + + case errInsufficientBalance: + return channeldb.FailureReasonInsufficientBalance + + default: + return channeldb.FailureReasonError + } +} + // PaymentSession is used during SendPayment attempts to provide routes to // attempt. It also defines methods to give the PaymentSession additional // information learned during the previous attempts. type PaymentSession interface { // RequestRoute returns the next route to attempt for routing the - // specified HTLC payment to the target node. - RequestRoute(payment *LightningPayment, - height uint32, finalCltvDelta uint16) (*route.Route, error) + // specified HTLC payment to the target node. The returned route should + // carry at most maxAmt to the target node, and pay at most feeLimit in + // fees. It can carry less if the payment is MPP. The activeShards + // argument should be set to instruct the payment session about the + // number of in flight HTLCS for the payment, such that it can choose + // splitting strategy accordingly. + // + // A noRouteError is returned if a non-critical error is encountered + // during path finding. + RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi, + activeShards, height uint32) (*route.Route, error) } // paymentSession is used during an HTLC routings session to prune the local @@ -43,8 +110,9 @@ type paymentSession struct { sessionSource *SessionSource - preBuiltRoute *route.Route - preBuiltRouteTried bool + payment *LightningPayment + + empty bool pathFinder pathFinder } @@ -58,31 +126,22 @@ type paymentSession struct { // // NOTE: This function is safe for concurrent access. // NOTE: Part of the PaymentSession interface. -func (p *paymentSession) RequestRoute(payment *LightningPayment, - height uint32, finalCltvDelta uint16) (*route.Route, error) { +func (p *paymentSession) RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi, + activeShards, height uint32) (*route.Route, error) { - switch { - - // If we have a pre-built route, use that directly. - case p.preBuiltRoute != nil && !p.preBuiltRouteTried: - p.preBuiltRouteTried = true - - return p.preBuiltRoute, nil - - // If the pre-built route has been tried already, the payment session is - // over. - case p.preBuiltRoute != nil: - return nil, errPrebuiltRouteTried + if p.empty { + return nil, errEmptyPaySession } // Add BlockPadding to the finalCltvDelta so that the receiving node // does not reject the HTLC if some blocks are mined while it's in-flight. + finalCltvDelta := p.payment.FinalCLTVDelta finalCltvDelta += BlockPadding // We need to subtract the final delta before passing it into path // finding. The optimal path is independent of the final cltv delta and // the path finding algorithm is unaware of this value. - cltvLimit := payment.CltvLimit - uint32(finalCltvDelta) + cltvLimit := p.payment.CltvLimit - uint32(finalCltvDelta) // TODO(roasbeef): sync logic amongst dist sys @@ -93,13 +152,13 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment, restrictions := &RestrictParams{ ProbabilitySource: ss.MissionControl.GetProbability, - FeeLimit: payment.FeeLimit, - OutgoingChannelID: payment.OutgoingChannelID, - LastHop: payment.LastHop, + FeeLimit: feeLimit, + OutgoingChannelID: p.payment.OutgoingChannelID, + LastHop: p.payment.LastHop, CltvLimit: cltvLimit, - DestCustomRecords: payment.DestCustomRecords, - DestFeatures: payment.DestFeatures, - PaymentAddr: payment.PaymentAddr, + DestCustomRecords: p.payment.DestCustomRecords, + DestFeatures: p.payment.DestFeatures, + PaymentAddr: p.payment.PaymentAddr, } // We'll also obtain a set of bandwidthHints from the lower layer for @@ -122,8 +181,8 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment, bandwidthHints: bandwidthHints, }, restrictions, &ss.PathFindingConfig, - ss.SelfNode.PubKeyBytes, payment.Target, - payment.Amount, finalHtlcExpiry, + ss.SelfNode.PubKeyBytes, p.payment.Target, + maxAmt, finalHtlcExpiry, ) if err != nil { return nil, err @@ -135,10 +194,10 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment, route, err := newRoute( sourceVertex, path, height, finalHopParams{ - amt: payment.Amount, + amt: maxAmt, cltvDelta: finalCltvDelta, - records: payment.DestCustomRecords, - paymentAddr: payment.PaymentAddr, + records: p.payment.DestCustomRecords, + paymentAddr: p.payment.PaymentAddr, }, ) if err != nil { diff --git a/routing/payment_session_source.go b/routing/payment_session_source.go index 05295b58d..3d6cfedf2 100644 --- a/routing/payment_session_source.go +++ b/routing/payment_session_source.go @@ -47,10 +47,10 @@ type SessionSource struct { // view from Mission Control. An optional set of routing hints can be provided // in order to populate additional edges to explore when finding a path to the // payment's destination. -func (m *SessionSource) NewPaymentSession(routeHints [][]zpay32.HopHint, - target route.Vertex) (PaymentSession, error) { +func (m *SessionSource) NewPaymentSession(p *LightningPayment) ( + PaymentSession, error) { - edges, err := RouteHintsToEdges(routeHints, target) + edges, err := RouteHintsToEdges(p.RouteHints, p.Target) if err != nil { return nil, err } @@ -70,27 +70,18 @@ func (m *SessionSource) NewPaymentSession(routeHints [][]zpay32.HopHint, additionalEdges: edges, getBandwidthHints: getBandwidthHints, sessionSource: m, + payment: p, pathFinder: findPath, }, nil } -// NewPaymentSessionForRoute creates a new paymentSession instance that is just -// used for failure reporting to missioncontrol. -func (m *SessionSource) NewPaymentSessionForRoute(preBuiltRoute *route.Route) PaymentSession { - return &paymentSession{ - sessionSource: m, - preBuiltRoute: preBuiltRoute, - } -} - // NewPaymentSessionEmpty creates a new paymentSession instance that is empty, // and will be exhausted immediately. Used for failure reporting to // missioncontrol for resumed payment we don't want to make more attempts for. func (m *SessionSource) NewPaymentSessionEmpty() PaymentSession { return &paymentSession{ - sessionSource: m, - preBuiltRoute: &route.Route{}, - preBuiltRouteTried: true, + sessionSource: m, + empty: true, } } diff --git a/routing/payment_session_test.go b/routing/payment_session_test.go index 55549c448..a91658043 100644 --- a/routing/payment_session_test.go +++ b/routing/payment_session_test.go @@ -44,6 +44,16 @@ func TestRequestRoute(t *testing.T) { }, } + cltvLimit := uint32(30) + finalCltvDelta := uint16(8) + + payment := &LightningPayment{ + CltvLimit: cltvLimit, + FinalCLTVDelta: finalCltvDelta, + Amount: 1000, + FeeLimit: 1000, + } + session := &paymentSession{ getBandwidthHints: func() (map[uint64]lnwire.MilliSatoshi, error) { @@ -51,18 +61,13 @@ func TestRequestRoute(t *testing.T) { return nil, nil }, sessionSource: sessionSource, + payment: payment, pathFinder: findPath, } - cltvLimit := uint32(30) - finalCltvDelta := uint16(8) - - payment := &LightningPayment{ - CltvLimit: cltvLimit, - FinalCLTVDelta: finalCltvDelta, - } - - route, err := session.RequestRoute(payment, height, finalCltvDelta) + route, err := session.RequestRoute( + payment.Amount, payment.FeeLimit, 0, height, + ) if err != nil { t.Fatal(err) } diff --git a/routing/route/route.go b/routing/route/route.go index f4de728fc..63944af18 100644 --- a/routing/route/route.go +++ b/routing/route/route.go @@ -129,6 +129,23 @@ type Hop struct { LegacyPayload bool } +// Copy returns a deep copy of the Hop. +func (h *Hop) Copy() *Hop { + c := *h + + if h.MPP != nil { + m := *h.MPP + c.MPP = &m + } + + if h.AMP != nil { + a := *h.AMP + c.AMP = &a + } + + return &c +} + // PackHopPayload writes to the passed io.Writer, the series of byes that can // be placed directly into the per-hop payload (EOB) for this hop. This will // include the required routing fields, as well as serializing any of the @@ -287,6 +304,18 @@ type Route struct { Hops []*Hop } +// Copy returns a deep copy of the Route. +func (r *Route) Copy() *Route { + c := *r + + c.Hops = make([]*Hop, len(r.Hops)) + for i := range r.Hops { + c.Hops[i] = r.Hops[i].Copy() + } + + return &c +} + // HopFee returns the fee charged by the route hop indicated by hopIndex. func (r *Route) HopFee(hopIndex int) lnwire.MilliSatoshi { var incomingAmt lnwire.MilliSatoshi @@ -308,7 +337,25 @@ func (r *Route) TotalFees() lnwire.MilliSatoshi { return 0 } - return r.TotalAmount - r.Hops[len(r.Hops)-1].AmtToForward + return r.TotalAmount - r.ReceiverAmt() +} + +// ReceiverAmt is the amount received by the final hop of this route. +func (r *Route) ReceiverAmt() lnwire.MilliSatoshi { + if len(r.Hops) == 0 { + return 0 + } + + return r.Hops[len(r.Hops)-1].AmtToForward +} + +// FinalHop returns the last hop of the route, or nil if the route is empty. +func (r *Route) FinalHop() *Hop { + if len(r.Hops) == 0 { + return nil + } + + return r.Hops[len(r.Hops)-1] } // NewRouteFromHops creates a new Route structure from the minimally required diff --git a/routing/route/route_test.go b/routing/route/route_test.go index 2095430b8..991175f49 100644 --- a/routing/route/route_test.go +++ b/routing/route/route_test.go @@ -20,15 +20,24 @@ var ( func TestRouteTotalFees(t *testing.T) { t.Parallel() - // Make sure empty route returns a 0 fee. + // Make sure empty route returns a 0 fee, and zero amount. r := &Route{} if r.TotalFees() != 0 { t.Fatalf("expected 0 fees, got %v", r.TotalFees()) } + if r.ReceiverAmt() != 0 { + t.Fatalf("expected 0 amt, got %v", r.ReceiverAmt()) + } + + // Make sure empty route won't be allowed in the constructor. + amt := lnwire.MilliSatoshi(1000) + _, err := NewRouteFromHops(amt, 100, Vertex{}, []*Hop{}) + if err != ErrNoRouteHopsProvided { + t.Fatalf("expected ErrNoRouteHopsProvided, got %v", err) + } // For one-hop routes the fee should be 0, since the last node will // receive the full amount. - amt := lnwire.MilliSatoshi(1000) hops := []*Hop{ { PubKeyBytes: Vertex{}, @@ -37,7 +46,7 @@ func TestRouteTotalFees(t *testing.T) { AmtToForward: amt, }, } - r, err := NewRouteFromHops(amt, 100, Vertex{}, hops) + r, err = NewRouteFromHops(amt, 100, Vertex{}, hops) if err != nil { t.Fatal(err) } @@ -46,6 +55,10 @@ func TestRouteTotalFees(t *testing.T) { t.Fatalf("expected 0 fees, got %v", r.TotalFees()) } + if r.ReceiverAmt() != amt { + t.Fatalf("expected %v amt, got %v", amt, r.ReceiverAmt()) + } + // Append the route with a node, making the first one take a fee. fee := lnwire.MilliSatoshi(100) hops = append(hops, &Hop{ @@ -64,6 +77,10 @@ func TestRouteTotalFees(t *testing.T) { if r.TotalFees() != fee { t.Fatalf("expected %v fees, got %v", fee, r.TotalFees()) } + + if r.ReceiverAmt() != amt-fee { + t.Fatalf("expected %v amt, got %v", amt-fee, r.ReceiverAmt()) + } } var ( diff --git a/routing/router.go b/routing/router.go index ca7a20f80..cac3e69d4 100644 --- a/routing/router.go +++ b/routing/router.go @@ -159,13 +159,7 @@ type PaymentSessionSource interface { // routes to the given target. An optional set of routing hints can be // provided in order to populate additional edges to explore when // finding a path to the payment's destination. - NewPaymentSession(routeHints [][]zpay32.HopHint, - target route.Vertex) (PaymentSession, error) - - // NewPaymentSessionForRoute creates a new paymentSession instance that - // is just used for failure reporting to missioncontrol, and will only - // attempt the given route. - NewPaymentSessionForRoute(preBuiltRoute *route.Route) PaymentSession + NewPaymentSession(p *LightningPayment) (PaymentSession, error) // NewPaymentSessionEmpty creates a new paymentSession instance that is // empty, and will be exhausted immediately. Used for failure reporting @@ -532,23 +526,17 @@ func (r *ChannelRouter) Start() error { // We create a dummy, empty payment session such that // we won't make another payment attempt when the // result for the in-flight attempt is received. - // - // PayAttemptTime doesn't need to be set, as there is - // only a single attempt. paySession := r.cfg.SessionSource.NewPaymentSessionEmpty() - lPayment := &LightningPayment{ - PaymentHash: payment.Info.PaymentHash, - } - - // TODO(joostjager): For mpp, possibly relaunch multiple - // in-flight htlcs here. - var attempt *channeldb.HTLCAttemptInfo - if len(payment.Attempts) > 0 { - attempt = &payment.Attempts[0] - } - - _, _, err := r.sendPayment(attempt, lPayment, paySession) + // We pass in a zero timeout value, to indicate we + // don't need it to timeout. It will stop immediately + // after the existing attempt has finished anyway. We + // also set a zero fee limit, as no more routes should + // be tried. + _, _, err := r.sendPayment( + payment.Info.Value, 0, + payment.Info.PaymentHash, 0, paySession, + ) if err != nil { log.Errorf("Resuming payment with hash %v "+ "failed: %v.", payment.Info.PaymentHash, err) @@ -1640,9 +1628,15 @@ func (r *ChannelRouter) SendPayment(payment *LightningPayment) ([32]byte, return [32]byte{}, nil, err } + log.Tracef("Dispatching SendPayment for lightning payment: %v", + spewPayment(payment)) + // Since this is the first time this payment is being made, we pass nil // for the existing attempt. - return r.sendPayment(nil, payment, paySession) + return r.sendPayment( + payment.Amount, payment.FeeLimit, payment.PaymentHash, + payment.PayAttemptTimeout, paySession, + ) } // SendPaymentAsync is the non-blocking version of SendPayment. The payment @@ -1659,7 +1653,13 @@ func (r *ChannelRouter) SendPaymentAsync(payment *LightningPayment) error { go func() { defer r.wg.Done() - _, _, err := r.sendPayment(nil, payment, paySession) + log.Tracef("Dispatching SendPayment for lightning payment: %v", + spewPayment(payment)) + + _, _, err := r.sendPayment( + payment.Amount, payment.FeeLimit, payment.PaymentHash, + payment.PayAttemptTimeout, paySession, + ) if err != nil { log.Errorf("Payment with hash %x failed: %v", payment.PaymentHash, err) @@ -1669,6 +1669,28 @@ func (r *ChannelRouter) SendPaymentAsync(payment *LightningPayment) error { return nil } +// spewPayment returns a log closures that provides a spewed string +// representation of the passed payment. +func spewPayment(payment *LightningPayment) logClosure { + return newLogClosure(func() string { + // Make a copy of the payment with a nilled Curve + // before spewing. + var routeHints [][]zpay32.HopHint + for _, routeHint := range payment.RouteHints { + var hopHints []zpay32.HopHint + for _, hopHint := range routeHint { + h := hopHint.Copy() + h.NodeID.Curve = nil + hopHints = append(hopHints, h) + } + routeHints = append(routeHints, hopHints) + } + p := *payment + p.RouteHints = routeHints + return spew.Sdump(p) + }) +} + // preparePayment creates the payment session and registers the payment with the // control tower. func (r *ChannelRouter) preparePayment(payment *LightningPayment) ( @@ -1677,9 +1699,7 @@ func (r *ChannelRouter) preparePayment(payment *LightningPayment) ( // Before starting the HTLC routing attempt, we'll create a fresh // payment session which will report our errors back to mission // control. - paySession, err := r.cfg.SessionSource.NewPaymentSession( - payment.RouteHints, payment.Target, - ) + paySession, err := r.cfg.SessionSource.NewPaymentSession(payment) if err != nil { return nil, err } @@ -1706,14 +1726,19 @@ func (r *ChannelRouter) preparePayment(payment *LightningPayment) ( // SendToRoute attempts to send a payment with the given hash through the // provided route. This function is blocking and will return the obtained // preimage if the payment is successful or the full error in case of a failure. -func (r *ChannelRouter) SendToRoute(hash lntypes.Hash, route *route.Route) ( +func (r *ChannelRouter) SendToRoute(hash lntypes.Hash, rt *route.Route) ( lntypes.Preimage, error) { - // Create a payment session for just this route. - paySession := r.cfg.SessionSource.NewPaymentSessionForRoute(route) - // Calculate amount paid to receiver. - amt := route.TotalAmount - route.TotalFees() + amt := rt.ReceiverAmt() + + // If this is meant as a MP payment shard, we set the amount + // for the creating info to the total amount of the payment. + finalHop := rt.Hops[len(rt.Hops)-1] + mpp := finalHop.MPP + if mpp != nil { + amt = mpp.TotalMsat() + } // Record this payment hash with the ControlTower, ensuring it is not // already in-flight. @@ -1725,50 +1750,100 @@ func (r *ChannelRouter) SendToRoute(hash lntypes.Hash, route *route.Route) ( } err := r.cfg.Control.InitPayment(hash, info) - if err != nil { + switch { + // If this is an MPP attempt and the hash is already registered with + // the database, we can go on to launch the shard. + case err == channeldb.ErrPaymentInFlight && mpp != nil: + + // Any other error is not tolerated. + case err != nil: return [32]byte{}, err } - // Create a (mostly) dummy payment, as the created payment session is - // not going to do path finding. - // TODO(halseth): sendPayment doesn't really need LightningPayment, make - // it take just needed fields instead. - // - // PayAttemptTime doesn't need to be set, as there is only a single - // attempt. - payment := &LightningPayment{ - PaymentHash: hash, + log.Tracef("Dispatching SendToRoute for hash %v: %v", + hash, newLogClosure(func() string { + return spew.Sdump(rt) + }), + ) + + // Launch a shard along the given route. + sh := &shardHandler{ + router: r, + paymentHash: hash, } - // Since this is the first time this payment is being made, we pass nil - // for the existing attempt. - preimage, _, err := r.sendPayment(nil, payment, paySession) - if err != nil { - // SendToRoute should return a structured error. In case the - // provided route fails, payment lifecycle will return a - // noRouteError with the structured error embedded. - if noRouteError, ok := err.(errNoRoute); ok { - if noRouteError.lastError == nil { - return lntypes.Preimage{}, - errors.New("failure message missing") - } + var shardError error + attempt, outcome, err := sh.launchShard(rt) - return lntypes.Preimage{}, noRouteError.lastError + // With SendToRoute, it can happen that the route exceeds protocol + // constraints. Mark the payment as failed with an internal error. + if err == route.ErrMaxRouteHopsExceeded || + err == sphinx.ErrMaxRoutingInfoSizeExceeded { + + log.Debugf("Invalid route provided for payment %x: %v", + hash, err) + + controlErr := r.cfg.Control.Fail( + hash, channeldb.FailureReasonError, + ) + if controlErr != nil { + return [32]byte{}, controlErr } + } + // In any case, don't continue if there is an error. + if err != nil { return lntypes.Preimage{}, err } - return preimage, nil + switch { + // Failed to launch shard. + case outcome.err != nil: + shardError = outcome.err + + // Shard successfully launched, wait for the result to be available. + default: + result, err := sh.collectResult(attempt) + if err != nil { + return lntypes.Preimage{}, err + } + + // We got a successful result. + if result.err == nil { + return result.preimage, nil + } + + // The shard failed, break switch to handle it. + shardError = result.err + } + + // Since for SendToRoute we won't retry in case the shard fails, we'll + // mark the payment failed with the control tower immediately. Process + // the error to check if it maps into a terminal error code, if not use + // a generic NO_ROUTE error. + reason := r.processSendError( + attempt.AttemptID, &attempt.Route, shardError, + ) + if reason == nil { + r := channeldb.FailureReasonNoRoute + reason = &r + } + + err = r.cfg.Control.Fail(hash, *reason) + if err != nil { + return lntypes.Preimage{}, err + } + + return lntypes.Preimage{}, shardError } -// sendPayment attempts to send a payment as described within the passed -// LightningPayment. This function is blocking and will return either: when the -// payment is successful, or all candidates routes have been attempted and -// resulted in a failed payment. If the payment succeeds, then a non-nil Route -// will be returned which describes the path the successful payment traversed -// within the network to reach the destination. Additionally, the payment -// preimage will also be returned. +// sendPayment attempts to send a payment to the passed payment hash. This +// function is blocking and will return either: when the payment is successful, +// or all candidates routes have been attempted and resulted in a failed +// payment. If the payment succeeds, then a non-nil Route will be returned +// which describes the path the successful payment traversed within the network +// to reach the destination. Additionally, the payment preimage will also be +// returned. // // The existing attempt argument should be set to nil if this is a payment that // haven't had any payment attempt sent to the switch yet. If it has had an @@ -1779,29 +1854,9 @@ func (r *ChannelRouter) SendToRoute(hash lntypes.Hash, route *route.Route) ( // router will call this method for every payment still in-flight according to // the ControlTower. func (r *ChannelRouter) sendPayment( - existingAttempt *channeldb.HTLCAttemptInfo, - payment *LightningPayment, paySession PaymentSession) ( - [32]byte, *route.Route, error) { - - log.Tracef("Dispatching route for lightning payment: %v", - newLogClosure(func() string { - // Make a copy of the payment with a nilled Curve - // before spewing. - var routeHints [][]zpay32.HopHint - for _, routeHint := range payment.RouteHints { - var hopHints []zpay32.HopHint - for _, hopHint := range routeHint { - h := hopHint.Copy() - h.NodeID.Curve = nil - hopHints = append(hopHints, h) - } - routeHints = append(routeHints, hopHints) - } - p := *payment - p.RouteHints = routeHints - return spew.Sdump(p) - }), - ) + totalAmt, feeLimit lnwire.MilliSatoshi, paymentHash lntypes.Hash, + timeout time.Duration, + paySession PaymentSession) ([32]byte, *route.Route, error) { // We'll also fetch the current block height so we can properly // calculate the required HTLC time locks within the route. @@ -1813,21 +1868,19 @@ func (r *ChannelRouter) sendPayment( // Now set up a paymentLifecycle struct with these params, such that we // can resume the payment from the current state. p := &paymentLifecycle{ - router: r, - payment: payment, - paySession: paySession, - currentHeight: currentHeight, - finalCLTVDelta: uint16(payment.FinalCLTVDelta), - attempt: existingAttempt, - circuit: nil, - lastError: nil, + router: r, + totalAmount: totalAmt, + feeLimit: feeLimit, + paymentHash: paymentHash, + paySession: paySession, + currentHeight: currentHeight, } // If a timeout is specified, create a timeout channel. If no timeout is // specified, the channel is left nil and will never abort the payment // loop. - if payment.PayAttemptTimeout != 0 { - p.timeoutChan = time.After(payment.PayAttemptTimeout) + if timeout != 0 { + p.timeoutChan = time.After(timeout) } return p.resumePayment() diff --git a/routing/router_test.go b/routing/router_test.go index 42020b7ea..7ac7527c5 100644 --- a/routing/router_test.go +++ b/routing/router_test.go @@ -2,12 +2,10 @@ package routing import ( "bytes" - "errors" "fmt" "image/color" "math" "math/rand" - "strings" "sync/atomic" "testing" "time" @@ -23,6 +21,7 @@ import ( "github.com/lightningnetwork/lnd/htlcswitch" "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/record" "github.com/lightningnetwork/lnd/routing/route" "github.com/lightningnetwork/lnd/zpay32" ) @@ -792,8 +791,30 @@ func TestSendPaymentErrorPathPruning(t *testing.T) { // The final error returned should also indicate that the peer wasn't // online (the last error we returned). - if !strings.Contains(err.Error(), "UnknownNextPeer") { - t.Fatalf("expected UnknownNextPeer instead got: %v", err) + if err != channeldb.FailureReasonNoRoute { + t.Fatalf("expected no route instead got: %v", err) + } + + // Inspect the two attempts that were made before the payment failed. + p, err := ctx.router.cfg.Control.FetchPayment(payHash) + if err != nil { + t.Fatal(err) + } + + if len(p.HTLCs) != 2 { + t.Fatalf("expected two attempts got %v", len(p.HTLCs)) + } + + // We expect the first attempt to have failed with a + // TemporaryChannelFailure, the second with UnknownNextPeer. + msg := p.HTLCs[0].Failure.Message + if _, ok := msg.(*lnwire.FailTemporaryChannelFailure); !ok { + t.Fatalf("unexpected fail message: %T", msg) + } + + msg = p.HTLCs[1].Failure.Message + if _, ok := msg.(*lnwire.FailUnknownNextPeer); !ok { + t.Fatalf("unexpected fail message: %T", msg) } ctx.router.cfg.MissionControl.(*MissionControl).ResetHistory() @@ -2595,639 +2616,6 @@ func assertChannelsPruned(t *testing.T, graph *channeldb.ChannelGraph, } } -// TestRouterPaymentStateMachine tests that the router interacts as expected -// with the ControlTower during a payment lifecycle, such that it payment -// attempts are not sent twice to the switch, and results are handled after a -// restart. -func TestRouterPaymentStateMachine(t *testing.T) { - t.Parallel() - - const startingBlockHeight = 101 - - // Setup two simple channels such that we can mock sending along this - // route. - chanCapSat := btcutil.Amount(100000) - testChannels := []*testChannel{ - symmetricTestChannel("a", "b", chanCapSat, &testChannelPolicy{ - Expiry: 144, - FeeRate: 400, - MinHTLC: 1, - MaxHTLC: lnwire.NewMSatFromSatoshis(chanCapSat), - }, 1), - symmetricTestChannel("b", "c", chanCapSat, &testChannelPolicy{ - Expiry: 144, - FeeRate: 400, - MinHTLC: 1, - MaxHTLC: lnwire.NewMSatFromSatoshis(chanCapSat), - }, 2), - } - - testGraph, err := createTestGraphFromChannels(testChannels, "a") - if err != nil { - t.Fatalf("unable to create graph: %v", err) - } - defer testGraph.cleanUp() - - hop1 := testGraph.aliasMap["b"] - hop2 := testGraph.aliasMap["c"] - hops := []*route.Hop{ - { - ChannelID: 1, - PubKeyBytes: hop1, - LegacyPayload: true, - }, - { - ChannelID: 2, - PubKeyBytes: hop2, - LegacyPayload: true, - }, - } - - // We create a simple route that we will supply every time the router - // requests one. - rt, err := route.NewRouteFromHops( - lnwire.MilliSatoshi(10000), 100, testGraph.aliasMap["a"], hops, - ) - if err != nil { - t.Fatalf("unable to create route: %v", err) - } - - // A payment state machine test case consists of several ordered steps, - // that we use for driving the scenario. - type testCase struct { - // steps is a list of steps to perform during the testcase. - steps []string - - // routes is the sequence of routes we will provide to the - // router when it requests a new route. - routes []*route.Route - } - - const ( - // routerInitPayment is a test step where we expect the router - // to call the InitPayment method on the control tower. - routerInitPayment = "Router:init-payment" - - // routerRegisterAttempt is a test step where we expect the - // router to call the RegisterAttempt method on the control - // tower. - routerRegisterAttempt = "Router:register-attempt" - - // routerSuccess is a test step where we expect the router to - // call the Success method on the control tower. - routerSuccess = "Router:success" - - // routerFail is a test step where we expect the router to call - // the Fail method on the control tower. - routerFail = "Router:fail" - - // sendToSwitchSuccess is a step where we expect the router to - // call send the payment attempt to the switch, and we will - // respond with a non-error, indicating that the payment - // attempt was successfully forwarded. - sendToSwitchSuccess = "SendToSwitch:success" - - // sendToSwitchResultFailure is a step where we expect the - // router to send the payment attempt to the switch, and we - // will respond with a forwarding error. This can happen when - // forwarding fail on our local links. - sendToSwitchResultFailure = "SendToSwitch:failure" - - // getPaymentResultSuccess is a test step where we expect the - // router to call the GetPaymentResult method, and we will - // respond with a successful payment result. - getPaymentResultSuccess = "GetPaymentResult:success" - - // getPaymentResultFailure is a test step where we expect the - // router to call the GetPaymentResult method, and we will - // respond with a forwarding error. - getPaymentResultFailure = "GetPaymentResult:failure" - - // resendPayment is a test step where we manually try to resend - // the same payment, making sure the router responds with an - // error indicating that it is alreayd in flight. - resendPayment = "ResendPayment" - - // startRouter is a step where we manually start the router, - // used to test that it automatically will resume payments at - // startup. - startRouter = "StartRouter" - - // stopRouter is a test step where we manually make the router - // shut down. - stopRouter = "StopRouter" - - // paymentSuccess is a step where assert that we receive a - // successful result for the original payment made. - paymentSuccess = "PaymentSuccess" - - // paymentError is a step where assert that we receive an error - // for the original payment made. - paymentError = "PaymentError" - - // resentPaymentSuccess is a step where assert that we receive - // a successful result for a payment that was resent. - resentPaymentSuccess = "ResentPaymentSuccess" - - // resentPaymentError is a step where assert that we receive an - // error for a payment that was resent. - resentPaymentError = "ResentPaymentError" - ) - - tests := []testCase{ - { - // Tests a normal payment flow that succeeds. - steps: []string{ - routerInitPayment, - routerRegisterAttempt, - sendToSwitchSuccess, - getPaymentResultSuccess, - routerSuccess, - paymentSuccess, - }, - routes: []*route.Route{rt}, - }, - { - // A payment flow with a failure on the first attempt, - // but that succeeds on the second attempt. - steps: []string{ - routerInitPayment, - routerRegisterAttempt, - sendToSwitchSuccess, - - // Make the first sent attempt fail. - getPaymentResultFailure, - - // The router should retry. - routerRegisterAttempt, - sendToSwitchSuccess, - - // Make the second sent attempt succeed. - getPaymentResultSuccess, - routerSuccess, - paymentSuccess, - }, - routes: []*route.Route{rt, rt}, - }, - { - // A payment flow with a forwarding failure first time - // sending to the switch, but that succeeds on the - // second attempt. - steps: []string{ - routerInitPayment, - routerRegisterAttempt, - - // Make the first sent attempt fail. - sendToSwitchResultFailure, - - // The router should retry. - routerRegisterAttempt, - sendToSwitchSuccess, - - // Make the second sent attempt succeed. - getPaymentResultSuccess, - routerSuccess, - paymentSuccess, - }, - routes: []*route.Route{rt, rt}, - }, - { - // A payment that fails on the first attempt, and has - // only one route available to try. It will therefore - // fail permanently. - steps: []string{ - routerInitPayment, - routerRegisterAttempt, - sendToSwitchSuccess, - - // Make the first sent attempt fail. - getPaymentResultFailure, - - // Since there are no more routes to try, the - // payment should fail. - routerFail, - paymentError, - }, - routes: []*route.Route{rt}, - }, - { - // We expect the payment to fail immediately if we have - // no routes to try. - steps: []string{ - routerInitPayment, - routerFail, - paymentError, - }, - routes: []*route.Route{}, - }, - { - // A normal payment flow, where we attempt to resend - // the same payment after each step. This ensures that - // the router don't attempt to resend a payment already - // in flight. - steps: []string{ - routerInitPayment, - routerRegisterAttempt, - - // Manually resend the payment, the router - // should attempt to init with the control - // tower, but fail since it is already in - // flight. - resendPayment, - routerInitPayment, - resentPaymentError, - - // The original payment should proceed as - // normal. - sendToSwitchSuccess, - - // Again resend the payment and assert it's not - // allowed. - resendPayment, - routerInitPayment, - resentPaymentError, - - // Notify about a success for the original - // payment. - getPaymentResultSuccess, - routerSuccess, - - // Now that the original payment finished, - // resend it again to ensure this is not - // allowed. - resendPayment, - routerInitPayment, - resentPaymentError, - paymentSuccess, - }, - routes: []*route.Route{rt}, - }, - { - // Tests that the router is able to handle the - // receieved payment result after a restart. - steps: []string{ - routerInitPayment, - routerRegisterAttempt, - sendToSwitchSuccess, - - // Shut down the router. The original caller - // should get notified about this. - stopRouter, - paymentError, - - // Start the router again, and ensure the - // router registers the success with the - // control tower. - startRouter, - getPaymentResultSuccess, - routerSuccess, - }, - routes: []*route.Route{rt}, - }, - { - // Tests that we are allowed to resend a payment after - // it has permanently failed. - steps: []string{ - routerInitPayment, - routerRegisterAttempt, - sendToSwitchSuccess, - - // Resending the payment at this stage should - // not be allowed. - resendPayment, - routerInitPayment, - resentPaymentError, - - // Make the first attempt fail. - getPaymentResultFailure, - routerFail, - - // Since we have no more routes to try, the - // original payment should fail. - paymentError, - - // Now resend the payment again. This should be - // allowed, since the payment has failed. - resendPayment, - routerInitPayment, - routerRegisterAttempt, - sendToSwitchSuccess, - getPaymentResultSuccess, - routerSuccess, - resentPaymentSuccess, - }, - routes: []*route.Route{rt}, - }, - } - - // Create a mock control tower with channels set up, that we use to - // synchronize and listen for events. - control := makeMockControlTower() - control.init = make(chan initArgs) - control.register = make(chan registerArgs) - control.success = make(chan successArgs) - control.fail = make(chan failArgs) - control.fetchInFlight = make(chan struct{}) - - quit := make(chan struct{}) - defer close(quit) - - // setupRouter is a helper method that creates and starts the router in - // the desired configuration for this test. - setupRouter := func() (*ChannelRouter, chan error, - chan *htlcswitch.PaymentResult, chan error) { - - chain := newMockChain(startingBlockHeight) - chainView := newMockChainView(chain) - - // We set uo the use the following channels and a mock Payer to - // synchonize with the interaction to the Switch. - sendResult := make(chan error) - paymentResultErr := make(chan error) - paymentResult := make(chan *htlcswitch.PaymentResult) - - payer := &mockPayer{ - sendResult: sendResult, - paymentResult: paymentResult, - paymentResultErr: paymentResultErr, - } - - router, err := New(Config{ - Graph: testGraph.graph, - Chain: chain, - ChainView: chainView, - Control: control, - SessionSource: &mockPaymentSessionSource{}, - MissionControl: &mockMissionControl{}, - Payer: payer, - ChannelPruneExpiry: time.Hour * 24, - GraphPruneInterval: time.Hour * 2, - QueryBandwidth: func(e *channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi { - return lnwire.NewMSatFromSatoshis(e.Capacity) - }, - NextPaymentID: func() (uint64, error) { - next := atomic.AddUint64(&uniquePaymentID, 1) - return next, nil - }, - Clock: clock.NewTestClock(time.Unix(1, 0)), - }) - if err != nil { - t.Fatalf("unable to create router %v", err) - } - - // On startup, the router should fetch all pending payments - // from the ControlTower, so assert that here. - errCh := make(chan error) - go func() { - close(errCh) - select { - case <-control.fetchInFlight: - return - case <-time.After(1 * time.Second): - errCh <- errors.New("router did not fetch in flight " + - "payments") - } - }() - - if err := router.Start(); err != nil { - t.Fatalf("unable to start router: %v", err) - } - - select { - case err := <-errCh: - if err != nil { - t.Fatalf("error in anonymous goroutine: %s", err) - } - case <-time.After(1 * time.Second): - t.Fatalf("did not fetch in flight payments at startup") - } - - return router, sendResult, paymentResult, paymentResultErr - } - - router, sendResult, getPaymentResult, getPaymentResultErr := setupRouter() - defer router.Stop() - - for _, test := range tests { - // Craft a LightningPayment struct. - var preImage lntypes.Preimage - if _, err := rand.Read(preImage[:]); err != nil { - t.Fatalf("unable to generate preimage") - } - - payHash := preImage.Hash() - - paymentAmt := lnwire.NewMSatFromSatoshis(1000) - payment := LightningPayment{ - Target: testGraph.aliasMap["c"], - Amount: paymentAmt, - FeeLimit: noFeeLimit, - PaymentHash: payHash, - } - - copy(preImage[:], bytes.Repeat([]byte{9}, 32)) - - router.cfg.SessionSource = &mockPaymentSessionSource{ - routes: test.routes, - } - - router.cfg.MissionControl = &mockMissionControl{} - - // Send the payment. Since this is new payment hash, the - // information should be registered with the ControlTower. - paymentResult := make(chan error) - go func() { - _, _, err := router.SendPayment(&payment) - paymentResult <- err - }() - - var resendResult chan error - for _, step := range test.steps { - switch step { - - case routerInitPayment: - var args initArgs - select { - case args = <-control.init: - case <-time.After(1 * time.Second): - t.Fatalf("no init payment with control") - } - - if args.c == nil { - t.Fatalf("expected non-nil CreationInfo") - } - - // In this step we expect the router to make a call to - // register a new attempt with the ControlTower. - case routerRegisterAttempt: - var args registerArgs - select { - case args = <-control.register: - case <-time.After(1 * time.Second): - t.Fatalf("not registered with control") - } - - if args.a == nil { - t.Fatalf("expected non-nil AttemptInfo") - } - - // In this step we expect the router to call the - // ControlTower's Succcess method with the preimage. - case routerSuccess: - select { - case _ = <-control.success: - case <-time.After(1 * time.Second): - t.Fatalf("not registered with control") - } - - // In this step we expect the router to call the - // ControlTower's Fail method, to indicate that the - // payment failed. - case routerFail: - select { - case _ = <-control.fail: - case <-time.After(1 * time.Second): - t.Fatalf("not registered with control") - } - - // In this step we expect the SendToSwitch method to be - // called, and we respond with a nil-error. - case sendToSwitchSuccess: - select { - case sendResult <- nil: - case <-time.After(1 * time.Second): - t.Fatalf("unable to send result") - } - - // In this step we expect the SendToSwitch method to be - // called, and we respond with a forwarding error - case sendToSwitchResultFailure: - select { - case sendResult <- htlcswitch.NewForwardingError( - &lnwire.FailTemporaryChannelFailure{}, - 1, - ): - case <-time.After(1 * time.Second): - t.Fatalf("unable to send result") - } - - // In this step we expect the GetPaymentResult method - // to be called, and we respond with the preimage to - // complete the payment. - case getPaymentResultSuccess: - select { - case getPaymentResult <- &htlcswitch.PaymentResult{ - Preimage: preImage, - }: - case <-time.After(1 * time.Second): - t.Fatalf("unable to send result") - } - - // In this state we expect the GetPaymentResult method - // to be called, and we respond with a forwarding - // error, indicating that the router should retry. - case getPaymentResultFailure: - failure := htlcswitch.NewForwardingError( - &lnwire.FailTemporaryChannelFailure{}, - 1, - ) - - select { - case getPaymentResult <- &htlcswitch.PaymentResult{ - Error: failure, - }: - case <-time.After(1 * time.Second): - t.Fatalf("unable to get result") - } - - // In this step we manually try to resend the same - // payment, making sure the router responds with an - // error indicating that it is alreayd in flight. - case resendPayment: - resendResult = make(chan error) - go func() { - _, _, err := router.SendPayment(&payment) - resendResult <- err - }() - - // In this step we manually stop the router. - case stopRouter: - select { - case getPaymentResultErr <- fmt.Errorf( - "shutting down"): - case <-time.After(1 * time.Second): - t.Fatalf("unable to send payment " + - "result error") - } - - if err := router.Stop(); err != nil { - t.Fatalf("unable to restart: %v", err) - } - - // In this step we manually start the router. - case startRouter: - router, sendResult, getPaymentResult, - getPaymentResultErr = setupRouter() - - // In this state we expect to receive an error for the - // original payment made. - case paymentError: - select { - case err := <-paymentResult: - if err == nil { - t.Fatalf("expected error") - } - - case <-time.After(1 * time.Second): - t.Fatalf("got no payment result") - } - - // In this state we expect the original payment to - // succeed. - case paymentSuccess: - select { - case err := <-paymentResult: - if err != nil { - t.Fatalf("did not expecte error %v", err) - } - - case <-time.After(1 * time.Second): - t.Fatalf("got no payment result") - } - - // In this state we expect to receive an error for the - // resent payment made. - case resentPaymentError: - select { - case err := <-resendResult: - if err == nil { - t.Fatalf("expected error") - } - - case <-time.After(1 * time.Second): - t.Fatalf("got no payment result") - } - - // In this state we expect the resent payment to - // succeed. - case resentPaymentSuccess: - select { - case err := <-resendResult: - if err != nil { - t.Fatalf("did not expect error %v", err) - } - - case <-time.After(1 * time.Second): - t.Fatalf("got no payment result") - } - - default: - t.Fatalf("unknown step %v", step) - } - } - } -} - // TestSendToRouteStructuredError asserts that SendToRoute returns a structured // error. func TestSendToRouteStructuredError(t *testing.T) { @@ -3338,6 +2726,138 @@ func TestSendToRouteStructuredError(t *testing.T) { } } +// TestSendToRouteMultiShardSend checks that a 3-shard payment can be executed +// using SendToRoute. +func TestSendToRouteMultiShardSend(t *testing.T) { + t.Parallel() + + ctx, cleanup, err := createTestCtxSingleNode(0) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + const numShards = 3 + const payAmt = lnwire.MilliSatoshi(numShards * 10000) + node, err := createTestNode() + if err != nil { + t.Fatal(err) + } + + // Create a simple 1-hop route that we will use for all three shards. + hops := []*route.Hop{ + { + ChannelID: 1, + PubKeyBytes: node.PubKeyBytes, + AmtToForward: payAmt / numShards, + MPP: record.NewMPP(payAmt, [32]byte{}), + }, + } + + sourceNode, err := ctx.graph.SourceNode() + if err != nil { + t.Fatal(err) + } + + rt, err := route.NewRouteFromHops( + payAmt, 100, sourceNode.PubKeyBytes, hops, + ) + if err != nil { + t.Fatalf("unable to create route: %v", err) + } + + // The first shard we send we'll fail immediately, to check that we are + // still allowed to retry with other shards after a failed one. + ctx.router.cfg.Payer.(*mockPaymentAttemptDispatcher).setPaymentResult( + func(firstHop lnwire.ShortChannelID) ([32]byte, error) { + return [32]byte{}, htlcswitch.NewForwardingError( + &lnwire.FailFeeInsufficient{ + Update: lnwire.ChannelUpdate{}, + }, 1, + ) + }) + + // The payment parameter is mostly redundant in SendToRoute. Can be left + // empty for this test. + var payment lntypes.Hash + + // Send the shard using the created route, and expect an error to be + // returned. + _, err = ctx.router.SendToRoute(payment, rt) + if err == nil { + t.Fatalf("expected forwarding error") + } + + // Now we'll modify the SendToSwitch method again to wait until all + // three shards are initiated before returning a result. We do this by + // signalling when the method has been called, and then stop to wait + // for the test to deliver the final result on the channel below. + waitForResultSignal := make(chan struct{}, numShards) + results := make(chan lntypes.Preimage, numShards) + + ctx.router.cfg.Payer.(*mockPaymentAttemptDispatcher).setPaymentResult( + func(firstHop lnwire.ShortChannelID) ([32]byte, error) { + + // Signal that the shard has been initiated and is + // waiting for a result. + waitForResultSignal <- struct{}{} + + // Wait for a result before returning it. + res, ok := <-results + if !ok { + return [32]byte{}, fmt.Errorf("failure") + } + return res, nil + }) + + // Launch three shards by calling SendToRoute in three goroutines, + // returning their final error on the channel. + errChan := make(chan error) + successes := make(chan lntypes.Preimage) + + for i := 0; i < numShards; i++ { + go func() { + preimg, err := ctx.router.SendToRoute(payment, rt) + if err != nil { + errChan <- err + return + } + + successes <- preimg + }() + } + + // Wait for all shards to signal they have been initiated. + for i := 0; i < numShards; i++ { + select { + case <-waitForResultSignal: + case <-time.After(5 * time.Second): + t.Fatalf("not waiting for results") + } + } + + // Deliver a dummy preimage to all the shard handlers. + preimage := lntypes.Preimage{} + preimage[4] = 42 + for i := 0; i < numShards; i++ { + results <- preimage + } + + // Finally expect all shards to return with the above preimage. + for i := 0; i < numShards; i++ { + select { + case p := <-successes: + if p != preimage { + t.Fatalf("preimage mismatch") + } + case err := <-errChan: + t.Fatalf("unexpected error from SendToRoute: %v", err) + case <-time.After(5 * time.Second): + t.Fatalf("result not received") + } + } +} + // TestSendToRouteMaxHops asserts that SendToRoute fails when using a route that // exceeds the maximum number of hops. func TestSendToRouteMaxHops(t *testing.T) {