From f5ffe82280696575fb6230c9829833fc27c94070 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Wed, 21 May 2025 11:23:42 +0800 Subject: [PATCH 1/4] lnwallet+channeldb: use `BigSizeT` to save space --- channeldb/revocation_log.go | 10 +++++---- channeldb/revocation_log_test.go | 16 +++++++------- lnwallet/channel.go | 38 ++++++++++++++++++-------------- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/channeldb/revocation_log.go b/channeldb/revocation_log.go index ea6eaf13f..840d5c502 100644 --- a/channeldb/revocation_log.go +++ b/channeldb/revocation_log.go @@ -165,7 +165,7 @@ type HTLCEntry struct { CustomBlob tlv.OptionalRecordT[tlv.TlvType5, tlv.Blob] // HtlcIndex is the index of the HTLC in the channel. - HtlcIndex tlv.OptionalRecordT[tlv.TlvType6, uint16] + HtlcIndex tlv.OptionalRecordT[tlv.TlvType6, tlv.BigSizeT[uint64]] } // toTlvStream converts an HTLCEntry record into a tlv representation. @@ -182,7 +182,9 @@ func (h *HTLCEntry) toTlvStream() (*tlv.Stream, error) { records = append(records, r.Record()) }) - h.HtlcIndex.WhenSome(func(r tlv.RecordT[tlv.TlvType6, uint16]) { + h.HtlcIndex.WhenSome(func(r tlv.RecordT[tlv.TlvType6, + tlv.BigSizeT[uint64]]) { + records = append(records, r.Record()) }) @@ -207,8 +209,8 @@ func NewHTLCEntryFromHTLC(htlc HTLC) (*HTLCEntry, error) { Amt: tlv.NewRecordT[tlv.TlvType4]( tlv.NewBigSizeT(htlc.Amt.ToSatoshis()), ), - HtlcIndex: tlv.SomeRecordT(tlv.NewPrimitiveRecord[tlv.TlvType6]( - uint16(htlc.HtlcIndex), + HtlcIndex: tlv.SomeRecordT(tlv.NewRecordT[tlv.TlvType6]( + tlv.NewBigSizeT(htlc.HtlcIndex), )), } diff --git a/channeldb/revocation_log_test.go b/channeldb/revocation_log_test.go index dcce16a03..626d23b34 100644 --- a/channeldb/revocation_log_test.go +++ b/channeldb/revocation_log_test.go @@ -59,13 +59,13 @@ var ( CustomBlob: tlv.SomeRecordT( tlv.NewPrimitiveRecord[tlv.TlvType5](blobBytes), ), - HtlcIndex: tlv.SomeRecordT( - tlv.NewPrimitiveRecord[tlv.TlvType6, uint16](0x33), - ), + HtlcIndex: tlv.SomeRecordT(tlv.NewRecordT[tlv.TlvType6]( + tlv.NewBigSizeT(uint64(0x33)), + )), } testHTLCEntryBytes = []byte{ - // Body length 45. - 0x2d, + // Body length 44. + 0x2c, // Rhash tlv. 0x0, 0x0, // RefundTimeout tlv. @@ -80,7 +80,7 @@ var ( 0x5, 0x11, 0xfe, 0x00, 0x01, 0x00, 0x01, 0x0b, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x20, 0x64, 0x61, 0x74, 0x61, // HLTC index tlv. - 0x6, 0x2, 0x0, 0x33, + 0x6, 0x1, 0x33, } testHTLCEntryHash = HTLCEntry{ @@ -133,7 +133,7 @@ var ( OutputIndex: int32(testHTLCEntry.OutputIndex.Val), HtlcIndex: uint64( testHTLCEntry.HtlcIndex.ValOpt(). - UnsafeFromSome(), + UnsafeFromSome().Int(), ), Incoming: testHTLCEntry.Incoming.Val, Amt: lnwire.NewMSatFromSatoshis( @@ -303,7 +303,7 @@ func TestSerializeHTLCEntries(t *testing.T) { partialBytes := testHTLCEntryBytes[3:] // Write the total length and RHash tlv. - expectedBytes := []byte{0x4d, 0x0, 0x20} + expectedBytes := []byte{0x4c, 0x0, 0x20} expectedBytes = append(expectedBytes, rHashBytes...) // Append the rest. diff --git a/lnwallet/channel.go b/lnwallet/channel.go index 545c4ccb8..81a201d1c 100644 --- a/lnwallet/channel.go +++ b/lnwallet/channel.go @@ -2340,18 +2340,21 @@ func createHtlcRetribution(chanState *channeldb.OpenChannel, // We'll generate the original second level witness script now, as // we'll need it if we're revoking an HTLC output on the remote // commitment transaction, and *they* go to the second level. + //nolint:ll secondLevelAuxLeaf := fn.FlatMapOption( func(l CommitAuxLeaves) fn.Option[input.AuxTapLeaf] { - return fn.MapOption(func(val uint16) input.AuxTapLeaf { - idx := input.HtlcIndex(val) + return fn.MapOption( + func(val tlv.BigSizeT[uint64]) input.AuxTapLeaf { + idx := val.Int() - if htlc.Incoming.Val { - leaves := l.IncomingHtlcLeaves[idx] - return leaves.SecondLevelLeaf - } + if htlc.Incoming.Val { + leaves := l.IncomingHtlcLeaves[idx] + return leaves.SecondLevelLeaf + } - return l.OutgoingHtlcLeaves[idx].SecondLevelLeaf - })(htlc.HtlcIndex.ValOpt()) + return l.OutgoingHtlcLeaves[idx].SecondLevelLeaf + }, + )(htlc.HtlcIndex.ValOpt()) }, )(auxLeaves) secondLevelScript, err := SecondLevelHtlcScript( @@ -2368,18 +2371,21 @@ func createHtlcRetribution(chanState *channeldb.OpenChannel, // HTLC script. Otherwise, is this was an outgoing HTLC that we sent, // then from the PoV of the remote commitment state, they're the // receiver of this HTLC. + //nolint:ll htlcLeaf := fn.FlatMapOption( func(l CommitAuxLeaves) fn.Option[input.AuxTapLeaf] { - return fn.MapOption(func(val uint16) input.AuxTapLeaf { - idx := input.HtlcIndex(val) + return fn.MapOption( + func(val tlv.BigSizeT[uint64]) input.AuxTapLeaf { + idx := val.Int() - if htlc.Incoming.Val { - leaves := l.IncomingHtlcLeaves[idx] - return leaves.AuxTapLeaf - } + if htlc.Incoming.Val { + leaves := l.IncomingHtlcLeaves[idx] + return leaves.AuxTapLeaf + } - return l.OutgoingHtlcLeaves[idx].AuxTapLeaf - })(htlc.HtlcIndex.ValOpt()) + return l.OutgoingHtlcLeaves[idx].AuxTapLeaf + }, + )(htlc.HtlcIndex.ValOpt()) }, )(auxLeaves) scriptInfo, err := genHtlcScript( From 4efd99774e67056d8aa7736d3b646f09c5d0e394 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 23 May 2025 22:37:33 +0800 Subject: [PATCH 2/4] channeldb: add customized encoding for `HtlcIndex` --- channeldb/revocation_log.go | 72 ++++++++++++++++++++++++++++++-- channeldb/revocation_log_test.go | 61 +++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 3 deletions(-) diff --git a/channeldb/revocation_log.go b/channeldb/revocation_log.go index 840d5c502..5a7f7a76b 100644 --- a/channeldb/revocation_log.go +++ b/channeldb/revocation_log.go @@ -2,6 +2,7 @@ package channeldb import ( "bytes" + "encoding/binary" "errors" "io" "math" @@ -512,13 +513,22 @@ func deserializeRevocationLog(r io.Reader) (RevocationLog, error) { // deserializeHTLCEntries deserializes a list of HTLC entries based on tlv // format. func deserializeHTLCEntries(r io.Reader) ([]*HTLCEntry, error) { - var htlcs []*HTLCEntry + var ( + htlcs []*HTLCEntry + + // htlcIndexBlob defines the tlv record type to be used when + // decoding from the disk. We use it instead of the one defined + // in `HTLCEntry.HtlcIndex` as previously this field was encoded + // using `uint16`, thus we will read it as raw bytes and + // deserialize it further below. + htlcIndexBlob tlv.OptionalRecordT[tlv.TlvType6, tlv.Blob] + ) for { var htlc HTLCEntry customBlob := htlc.CustomBlob.Zero() - htlcIndex := htlc.HtlcIndex.Zero() + htlcIndex := htlcIndexBlob.Zero() // Create the tlv stream. records := []tlv.Record{ @@ -551,7 +561,14 @@ func deserializeHTLCEntries(r io.Reader) ([]*HTLCEntry, error) { } if t, ok := parsedTypes[htlcIndex.TlvType()]; ok && t == nil { - htlc.HtlcIndex = tlv.SomeRecordT(htlcIndex) + record, err := deserializeHtlcIndexCompatible( + htlcIndex.Val, + ) + if err != nil { + return nil, err + } + + htlc.HtlcIndex = record } // Append the entry. @@ -561,6 +578,55 @@ func deserializeHTLCEntries(r io.Reader) ([]*HTLCEntry, error) { return htlcs, nil } +// deserializeHtlcIndexCompatible takes raw bytes and decodes it into an +// optional record that's assigned to the entry's HtlcIndex. +// +// NOTE: previously this `HtlcIndex` was a tlv record that used `uint16` to +// encode its value. Given now its value is encoded using BigSizeT, and for any +// BigSizeT, its possible length values are 1, 3, 5, and 8. This means if the +// tlv record has a length of 2, we know for sure it must be an old record +// whose value was encoded using uint16. +func deserializeHtlcIndexCompatible(rawBytes []byte) ( + tlv.OptionalRecordT[tlv.TlvType6, tlv.BigSizeT[uint64]], error) { + + var ( + // record defines the record that's used by the HtlcIndex in the + // entry. + record tlv.OptionalRecordT[ + tlv.TlvType6, tlv.BigSizeT[uint64], + ] + + // htlcIndexVal is the decoded uint64 value. + htlcIndexVal uint64 + ) + + // If the length of the tlv record is 2, it must be encoded using uint16 + // as the BigSizeT encoding cannot have this length. + if len(rawBytes) == 2 { + // Decode the raw bytes into uint16 and convert it into uint64. + htlcIndexVal = uint64(binary.BigEndian.Uint16(rawBytes)) + } else { + // This value is encoded using BigSizeT, we now use the decoder + // to deserialize the raw bytes. + r := bytes.NewBuffer(rawBytes) + + // Create a buffer to be used in the decoding process. + buf := [8]byte{} + + // Use the BigSizeT's decoder. + err := tlv.DBigSize(r, &htlcIndexVal, &buf, 8) + if err != nil { + return record, err + } + } + + record = tlv.SomeRecordT(tlv.NewRecordT[tlv.TlvType6]( + tlv.NewBigSizeT(htlcIndexVal), + )) + + return record, nil +} + // writeTlvStream is a helper function that encodes the tlv stream into the // writer. func writeTlvStream(w io.Writer, s *tlv.Stream) error { diff --git a/channeldb/revocation_log_test.go b/channeldb/revocation_log_test.go index 626d23b34..ceec9a265 100644 --- a/channeldb/revocation_log_test.go +++ b/channeldb/revocation_log_test.go @@ -2,6 +2,7 @@ package channeldb import ( "bytes" + "encoding/binary" "io" "math" "math/rand" @@ -785,3 +786,63 @@ func createTestRevocationLogBuckets(tx kvdb.RwTx) (kvdb.RwBucket, return chanBucket, logBucket, nil } + +// TestDeserializeHTLCEntriesLegacy checks that the legacy encoding of the +// HtlcIndex can be correctly read as BigSizeT. The field `HtlcIndex` was +// encoded using `uint16` and should now be deserialized into BigSizeT +// on-the-fly. +func TestDeserializeHTLCEntriesLegacy(t *testing.T) { + t.Parallel() + + // rawBytes defines the bytes read from the disk for the testing HTLC + // entry. + rawBytes := []byte{ + // Body length 45. + 0x2d, + // Rhash tlv. + 0x0, 0x0, + // RefundTimeout tlv. + 0x1, 0x4, 0x0, 0xb, 0x4a, 0xa0, + // OutputIndex tlv. + 0x2, 0x2, 0x0, 0xa, + // Incoming tlv. + 0x3, 0x1, 0x1, + // Amt tlv. + 0x4, 0x5, 0xfe, 0x0, 0xf, 0x42, 0x40, + // Custom blob tlv. + 0x5, 0x11, 0xfe, 0x00, 0x01, 0x00, 0x01, 0x0b, 0x63, 0x75, 0x73, + 0x74, 0x6f, 0x6d, 0x20, 0x64, 0x61, 0x74, 0x61, + + // HTLC index tlv. + // + // NOTE: We are missing two bytes in the end, which is appended + // below. + 0x6, 0x2, + } + + // Iterate through all possible values encoded using uint16. They should + // be correctly read as uint16 and converted to BigSizeT. + for i := range math.MaxUint16 + 1 { + // Encode the index using two bytes. + rawHtlcIndex := make([]byte, 2) + binary.BigEndian.PutUint16(rawHtlcIndex, uint16(i)) + + // Copy the raw bytes and append the htlc index. + rawEntry := bytes.Clone(rawBytes) + rawEntry = append(rawEntry, rawHtlcIndex...) + + // Read the tlv stream. + buf := bytes.NewBuffer(rawEntry) + htlcs, err := deserializeHTLCEntries(buf) + require.NoError(t, err) + + // Check the bytes are read as expected. + require.Len(t, htlcs, 1) + + // Assert that the uint16 is converted to BigSizeT. + record := tlv.SomeRecordT(tlv.NewRecordT[tlv.TlvType6]( + tlv.NewBigSizeT(uint64(i)), + )) + require.Equal(t, record, htlcs[0].HtlcIndex) + } +} From 2cd97fcd6df7e8c5500dbd7d025daf7589c90e00 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Wed, 4 Jun 2025 00:46:42 +0800 Subject: [PATCH 3/4] docs: add release note entry --- docs/release-notes/release-notes-0.19.2.md | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 docs/release-notes/release-notes-0.19.2.md diff --git a/docs/release-notes/release-notes-0.19.2.md b/docs/release-notes/release-notes-0.19.2.md new file mode 100644 index 000000000..56e5836dd --- /dev/null +++ b/docs/release-notes/release-notes-0.19.2.md @@ -0,0 +1,64 @@ +# Release Notes +- [Bug Fixes](#bug-fixes) +- [New Features](#new-features) + - [Functional Enhancements](#functional-enhancements) + - [RPC Additions](#rpc-additions) + - [lncli Additions](#lncli-additions) +- [Improvements](#improvements) + - [Functional Updates](#functional-updates) + - [RPC Updates](#rpc-updates) + - [lncli Updates](#lncli-updates) + - [Breaking Changes](#breaking-changes) + - [Performance Improvements](#performance-improvements) + - [Deprecations](#deprecations) +- [Technical and Architectural Updates](#technical-and-architectural-updates) + - [BOLT Spec Updates](#bolt-spec-updates) + - [Testing](#testing) + - [Database](#database) + - [Code Health](#code-health) + - [Tooling and Documentation](#tooling-and-documentation) + +# Bug Fixes + +- [Use](https://github.com/lightningnetwork/lnd/pull/9889) `BigSizeT` instead of + `uint16` for the htlc index that's used in the revocation log. + +# New Features + +## Functional Enhancements + +## RPC Additions + +## lncli Additions + +# Improvements + +## Functional Updates + +## RPC Updates + +## lncli Updates + +## Code Health + +## Breaking Changes + +## Performance Improvements + +## Deprecations + +# Technical and Architectural Updates + +## BOLT Spec Updates + +## Testing + +## Database + +## Code Health + +## Tooling and Documentation + +# Contributors (Alphabetical Order) + +* Yong Yu From 870e16bdd7c986185b4b4f886655b4a51d421618 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 6 Jun 2025 15:05:41 +0800 Subject: [PATCH 4/4] channeldb+lnwallet: fix typo --- channeldb/revocation_log_test.go | 2 +- lnwallet/channel.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/channeldb/revocation_log_test.go b/channeldb/revocation_log_test.go index ceec9a265..6e7afb9a3 100644 --- a/channeldb/revocation_log_test.go +++ b/channeldb/revocation_log_test.go @@ -80,7 +80,7 @@ var ( // Custom blob tlv. 0x5, 0x11, 0xfe, 0x00, 0x01, 0x00, 0x01, 0x0b, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x20, 0x64, 0x61, 0x74, 0x61, - // HLTC index tlv. + // HTLC index tlv. 0x6, 0x1, 0x33, } diff --git a/lnwallet/channel.go b/lnwallet/channel.go index 81a201d1c..ee45bf943 100644 --- a/lnwallet/channel.go +++ b/lnwallet/channel.go @@ -2368,7 +2368,7 @@ func createHtlcRetribution(chanState *channeldb.OpenChannel, // If this is an incoming HTLC, then this means that they were the // sender of the HTLC (relative to us). So we'll re-generate the sender - // HTLC script. Otherwise, is this was an outgoing HTLC that we sent, + // HTLC script. Otherwise, if this was an outgoing HTLC that we sent, // then from the PoV of the remote commitment state, they're the // receiver of this HTLC. //nolint:ll