From 2e737398372e1543bef8a05f07158626ce53e8d1 Mon Sep 17 00:00:00 2001 From: Murch Date: Wed, 26 Mar 2025 15:32:43 -0700 Subject: [PATCH 1/7] coinselection: Track BnB iteration count in result The expected iteration count demonstrates how the following improvements reduce iterations will help catch any regressions in the future. --- src/wallet/coinselection.cpp | 1 + src/wallet/test/coinselection_tests.cpp | 29 +++++++++++++------------ src/wallet/test/coinselector_tests.cpp | 9 ++++++++ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index d6ea6851e64..0d7b6603d72 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -122,6 +122,7 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool // Depth First search loop for choosing the UTXOs for (size_t curr_try = 0, utxo_pool_index = 0; curr_try < TOTAL_TRIES; ++curr_try, ++utxo_pool_index) { + result.SetSelectionsEvaluated(curr_try); // Conditions for starting a backtrack bool backtrack = false; if (curr_value + curr_available_value < selection_target || // Cannot possibly reach target with the amount remaining in the curr_available_value. diff --git a/src/wallet/test/coinselection_tests.cpp b/src/wallet/test/coinselection_tests.cpp index 6e60af1c0e3..2f982b74672 100644 --- a/src/wallet/test/coinselection_tests.cpp +++ b/src/wallet/test/coinselection_tests.cpp @@ -112,7 +112,7 @@ static std::string InputAmountsToString(const SelectionResult& selection) return "[" + util::Join(selection.GetInputSet(), " ", [](const auto& input){ return util::ToString(input->txout.nValue);}) + "]"; } -static void TestBnBSuccess(std::string test_title, std::vector& utxo_pool, const CAmount& selection_target, const std::vector& expected_input_amounts, const CoinSelectionParams& cs_params = default_cs_params, const int custom_spending_vsize = P2WPKH_INPUT_VSIZE, const int max_selection_weight = MAX_STANDARD_TX_WEIGHT) +static void TestBnBSuccess(std::string test_title, std::vector& utxo_pool, const CAmount& selection_target, const std::vector& expected_input_amounts, size_t expected_attempts, const CoinSelectionParams& cs_params = default_cs_params, const int custom_spending_vsize = P2WPKH_INPUT_VSIZE, const int max_selection_weight = MAX_STANDARD_TX_WEIGHT) { SelectionResult expected_result(CAmount(0), SelectionAlgorithm::BNB); CAmount expected_amount = 0; @@ -127,6 +127,7 @@ static void TestBnBSuccess(std::string test_title, std::vector& utx BOOST_CHECK_MESSAGE(HaveEquivalentValues(expected_result, *result), strprintf("Result mismatch in BnB-Success: %s. Expected %s, but got %s", test_title, InputAmountsToString(expected_result), InputAmountsToString(*result))); BOOST_CHECK_MESSAGE(result->GetSelectedValue() == expected_amount, strprintf("Selected amount mismatch in BnB-Success: %s. Expected %d, but got %d", test_title, expected_amount, result->GetSelectedValue())); BOOST_CHECK_MESSAGE(result->GetWeight() <= max_selection_weight, strprintf("Selected weight is higher than permitted in BnB-Success: %s. Expected %d, but got %d", test_title, max_selection_weight, result->GetWeight())); + BOOST_CHECK_MESSAGE(result->GetSelectionsEvaluated() == expected_attempts, strprintf("Unexpected number of attempts in BnB-Success: %s. Expected %i attempts, but got %i", test_title, expected_attempts, result->GetSelectionsEvaluated())); } static void TestBnBFail(std::string test_title, std::vector& utxo_pool, const CAmount& selection_target, const CoinSelectionParams& cs_params = default_cs_params, int max_selection_weight = MAX_STANDARD_TX_WEIGHT, const bool expect_max_weight_exceeded = false) @@ -149,19 +150,19 @@ BOOST_AUTO_TEST_CASE(bnb_test) AddCoins(utxo_pool, {1 * CENT, 3 * CENT, 5 * CENT}, cs_params); // Simple success cases - TestBnBSuccess("Select smallest UTXO", utxo_pool, /*selection_target=*/1 * CENT, /*expected_input_amounts=*/{1 * CENT}, cs_params); - TestBnBSuccess("Select middle UTXO", utxo_pool, /*selection_target=*/3 * CENT, /*expected_input_amounts=*/{3 * CENT}, cs_params); - TestBnBSuccess("Select biggest UTXO", utxo_pool, /*selection_target=*/5 * CENT, /*expected_input_amounts=*/{5 * CENT}, cs_params); - TestBnBSuccess("Select two UTXOs", utxo_pool, /*selection_target=*/4 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, cs_params); - TestBnBSuccess("Select all UTXOs", utxo_pool, /*selection_target=*/9 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT, 5 * CENT}, cs_params); + TestBnBSuccess("Select smallest UTXO", utxo_pool, /*selection_target=*/1 * CENT, /*expected_input_amounts=*/{1 * CENT}, /*expected_attempts=*/6, cs_params); + TestBnBSuccess("Select middle UTXO", utxo_pool, /*selection_target=*/3 * CENT, /*expected_input_amounts=*/{3 * CENT}, /*expected_attempts=*/4, cs_params); + TestBnBSuccess("Select biggest UTXO", utxo_pool, /*selection_target=*/5 * CENT, /*expected_input_amounts=*/{5 * CENT}, /*expected_attempts=*/2, cs_params); + TestBnBSuccess("Select two UTXOs", utxo_pool, /*selection_target=*/4 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/6, cs_params); + TestBnBSuccess("Select all UTXOs", utxo_pool, /*selection_target=*/9 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT, 5 * CENT}, /*expected_attempts=*/6, cs_params); // BnB finds changeless solution while overshooting by up to cost_of_change - TestBnBSuccess("Select upper bound", utxo_pool, /*selection_target=*/4 * CENT - cs_params.m_cost_of_change, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, cs_params); + TestBnBSuccess("Select upper bound", utxo_pool, /*selection_target=*/4 * CENT - cs_params.m_cost_of_change, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/6, cs_params); // BnB fails to find changeless solution when overshooting by cost_of_change + 1 sat TestBnBFail("Overshoot upper bound", utxo_pool, /*selection_target=*/4 * CENT - cs_params.m_cost_of_change - 1, cs_params); - TestBnBSuccess("Select max weight", utxo_pool, /*selection_target=*/4 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, cs_params, /*custom_spending_vsize=*/P2WPKH_INPUT_VSIZE, /*max_selection_weight=*/4 * 2 * P2WPKH_INPUT_VSIZE); + TestBnBSuccess("Select max weight", utxo_pool, /*selection_target=*/4 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/6, cs_params, /*custom_spending_vsize=*/P2WPKH_INPUT_VSIZE, /*max_selection_weight=*/4 * 2 * P2WPKH_INPUT_VSIZE); TestBnBFail("Exceed max weight", utxo_pool, /*selection_target=*/4 * CENT, cs_params, /*max_selection_weight=*/4 * 2 * P2WPKH_INPUT_VSIZE - 1, /*expect_max_weight_exceeded=*/true); @@ -174,7 +175,7 @@ BOOST_AUTO_TEST_CASE(bnb_test) std::vector clone_pool; AddCoins(clone_pool, {2 * CENT, 7 * CENT, 7 * CENT}, cs_params); AddDuplicateCoins(clone_pool, /*count=*/50'000, /*amount=*/5 * CENT, cs_params); - TestBnBSuccess("Skip equivalent input sets", clone_pool, /*selection_target=*/16 * CENT, /*expected_input_amounts=*/{2 * CENT, 7 * CENT, 7 * CENT}, cs_params); + TestBnBSuccess("Skip equivalent input sets", clone_pool, /*selection_target=*/16 * CENT, /*expected_input_amounts=*/{2 * CENT, 7 * CENT, 7 * CENT}, /*expected_attempts=*/99'999, cs_params); /* Test BnB attempt limit (`TOTAL_TRIES`) * @@ -207,7 +208,7 @@ BOOST_AUTO_TEST_CASE(bnb_test) } AddCoins(doppelganger_pool, doppelgangers, cs_params); // Among up to 17 unique UTXOs of similar effective value we will find a solution composed of the eight smallest UTXOs - TestBnBSuccess("Combine smallest 8 of 17 unique UTXOs", doppelganger_pool, /*selection_target=*/8 * CENT, /*expected_input_amounts=*/expected_inputs, cs_params); + TestBnBSuccess("Combine smallest 8 of 17 unique UTXOs", doppelganger_pool, /*selection_target=*/8 * CENT, /*expected_input_amounts=*/expected_inputs, /*expected_attempts=*/87'514, cs_params); // Starting with 18 unique UTXOs of similar effective value we will not find the solution due to exceeding the attempt limit AddCoins(doppelganger_pool, {1 * CENT + cs_params.m_cost_of_change + 17}, cs_params); @@ -220,21 +221,21 @@ BOOST_AUTO_TEST_CASE(bnb_feerate_sensitivity_test) // Create sets of UTXOs with the same effective amounts at different feerates (but different absolute amounts) std::vector low_feerate_pool; // 5 sat/vB (default, and lower than long_term_feerate of 10 sat/vB) AddCoins(low_feerate_pool, {2 * CENT, 3 * CENT, 5 * CENT, 10 * CENT}); - TestBnBSuccess("Select many inputs at low feerates", low_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{2 * CENT, 3 * CENT, 5 * CENT}); + TestBnBSuccess("Select many inputs at low feerates", low_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{2 * CENT, 3 * CENT, 5 * CENT}, /*expected_attempts=*/8); const CoinSelectionParams high_feerate_params = init_cs_params(/*eff_feerate=*/25'000); std::vector high_feerate_pool; // 25 sat/vB (greater than long_term_feerate of 10 sat/vB) AddCoins(high_feerate_pool, {2 * CENT, 3 * CENT, 5 * CENT, 10 * CENT}, high_feerate_params); - TestBnBSuccess("Select one input at high feerates", high_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{10 * CENT}, high_feerate_params); + TestBnBSuccess("Select one input at high feerates", high_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{10 * CENT}, /*expected_attempts=*/6, high_feerate_params); // Add heavy inputs {6, 7} to existing {2, 3, 5, 10} low_feerate_pool.push_back(MakeCoin(6 * CENT, true, default_cs_params, /*custom_spending_vsize=*/500)); low_feerate_pool.push_back(MakeCoin(7 * CENT, true, default_cs_params, /*custom_spending_vsize=*/500)); - TestBnBSuccess("Prefer two heavy inputs over two light inputs at low feerates", low_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{6 * CENT, 7 * CENT}, default_cs_params, /*custom_spending_vsize=*/500); + TestBnBSuccess("Prefer two heavy inputs over two light inputs at low feerates", low_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{6 * CENT, 7 * CENT}, /*expected_attempts=*/28, default_cs_params, /*custom_spending_vsize=*/500); high_feerate_pool.push_back(MakeCoin(6 * CENT, true, high_feerate_params, /*custom_spending_vsize=*/500)); high_feerate_pool.push_back(MakeCoin(7 * CENT, true, high_feerate_params, /*custom_spending_vsize=*/500)); - TestBnBSuccess("Prefer two light inputs over two heavy inputs at high feerates", high_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{3 * CENT, 10 * CENT}, high_feerate_params); + TestBnBSuccess("Prefer two light inputs over two heavy inputs at high feerates", high_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{3 * CENT, 10 * CENT}, /*expected_attempts=*/14, high_feerate_params); } static void TestSRDSuccess(std::string test_title, std::vector& utxo_pool, const CAmount& selection_target, const CoinSelectionParams& cs_params = default_cs_params, const int max_selection_weight = MAX_STANDARD_TX_WEIGHT) diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index ee6a639a43d..2b8150c6ee0 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -171,6 +171,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) // Setup std::vector utxo_pool; SelectionResult expected_result(CAmount(0), SelectionAlgorithm::BNB); + size_t expected_attempts; //////////////////// // Behavior tests // @@ -208,6 +209,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) const auto result9 = SelectCoinsBnB(GroupCoins(available_coins.All()), 1 * CENT, coin_selection_params_bnb.m_cost_of_change); BOOST_CHECK(result9); BOOST_CHECK_EQUAL(result9->GetSelectedValue(), 1 * CENT); + expected_attempts = 2; + BOOST_CHECK_MESSAGE(result9->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, result9->GetSelectionsEvaluated())); } { @@ -230,6 +233,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) LOCK(wallet->cs_wallet); const auto result10 = SelectCoins(*wallet, available_coins, selected_input, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(result10); + expected_attempts = 4; + BOOST_CHECK_MESSAGE(result10->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, result10->GetSelectionsEvaluated())); } { std::unique_ptr wallet = NewWallet(m_node); @@ -259,6 +264,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) available_coins.Erase({(++available_coins.coins[OutputType::BECH32].begin())->outpoint}); const auto result13 = SelectCoins(*wallet, available_coins, selected_input, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result13)); + expected_attempts = 4; + BOOST_CHECK_MESSAGE(result13->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, result13->GetSelectionsEvaluated())); } { @@ -290,6 +297,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) add_coin(5 * CENT, 2, expected_result); add_coin(3 * CENT, 2, expected_result); BOOST_CHECK(EquivalentResult(expected_result, *res)); + expected_attempts = 38; + BOOST_CHECK_MESSAGE(res->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, res->GetSelectionsEvaluated())); } } From 3ca0f36164b100f252cb61bba3d73bdc8c39c43e Mon Sep 17 00:00:00 2001 From: Murch Date: Wed, 26 Mar 2025 16:18:01 -0700 Subject: [PATCH 2/7] coinselection: rewrite BnB in CoinGrinder-style In the original implementation of BnB, the state of the search is backtracked by explicitly walking back to the omission branch and then testing again. This retests an equivalent candidate set as before, e.g., after backtracking from {ABC}, it would evaluate {AB_}, before trying {AB_D}, but {AB_} is equivalent to {AB} which was tested before. CoinGrinder tracks the state of the search instead by remembering which UTXO was last added and explicitly shifting from that UTXO directly to the next, so after {ABC}, it will immediately move on to {AB_D}. We replicate this approach here. The description of the two optimizations is removed from the documentation as they will only be implented in a later commit. --- src/wallet/coinselection.cpp | 173 ++++++++++++------------ src/wallet/test/coinselection_tests.cpp | 22 +-- src/wallet/test/coinselector_tests.cpp | 8 +- 3 files changed, 105 insertions(+), 98 deletions(-) diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 0d7b6603d72..6755376ff51 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -55,25 +55,19 @@ struct { * cost of creating and spending a change output. The algorithm uses a depth-first search on a binary * tree. In the binary tree, each node corresponds to the inclusion or the omission of a UTXO. UTXOs * are sorted by their effective values and the tree is explored deterministically per the inclusion - * branch first. At each node, the algorithm checks whether the selection is within the target range. + * branch first. For each new input set candidate, the algorithm checks whether the selection is within the target range. * While the selection has not reached the target range, more UTXOs are included. When a selection's * value exceeds the target range, the complete subtree deriving from this selection can be omitted. * At that point, the last included UTXO is deselected and the corresponding omission branch explored - * instead. The search ends after the complete tree has been searched or after a limited number of tries. + * instead starting by adding the subsequent UTXO. The search ends after the complete tree has been searched or after a limited number of tries. * - * The search continues to search for better solutions after one solution has been found. The best + * The algorithm continues to search for better solutions after one solution has been found. The best * solution is chosen by minimizing the waste metric. The waste metric is defined as the cost to * spend the current inputs at the given fee rate minus the long term expected cost to spend the * inputs, plus the amount by which the selection exceeds the spending target: * * waste = selectionTotal - target + inputs × (currentFeeRate - longTermFeeRate) * - * The algorithm uses two additional optimizations. A lookahead keeps track of the total value of - * the unexplored UTXOs. A subtree is not explored if the lookahead indicates that the target range - * cannot be reached. Further, it is unnecessary to test equivalent combinations. This allows us - * to skip testing the inclusion of UTXOs that match the effective value and waste of an omitted - * predecessor. - * * The Branch and Bound algorithm is described in detail in Murch's Master Thesis: * https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf * @@ -93,114 +87,127 @@ static const size_t TOTAL_TRIES = 100000; util::Result SelectCoinsBnB(std::vector& utxo_pool, const CAmount& selection_target, const CAmount& cost_of_change, int max_selection_weight) { - SelectionResult result(selection_target, SelectionAlgorithm::BNB); - CAmount curr_value = 0; - std::vector curr_selection; // selected utxo indexes - int curr_selection_weight = 0; // sum of selected utxo weight - - // Calculate curr_available_value - CAmount curr_available_value = 0; + // Check that there are sufficient funds + CAmount total_available = 0; for (const OutputGroup& utxo : utxo_pool) { - // Assert that this utxo is not negative. It should never be negative, - // effective value calculation should have removed it - assert(utxo.GetSelectionAmount() > 0); - curr_available_value += utxo.GetSelectionAmount(); + // Assert UTXOs with non-positive effective value have been filtered + Assume(utxo.GetSelectionAmount() > 0); + total_available += utxo.GetSelectionAmount(); } - if (curr_available_value < selection_target) { + + if (total_available < selection_target) { + // Insufficient funds return util::Error(); } - // Sort the utxo_pool std::sort(utxo_pool.begin(), utxo_pool.end(), descending); - CAmount curr_waste = 0; + // The current selection and the best input set found so far, stored as the utxo_pool indices of the UTXOs forming them + std::vector curr_selection; std::vector best_selection; + + // The currently selected effective amount + CAmount curr_amount = 0; + + // The waste score of the current selection, and the best waste score so far + CAmount curr_selection_waste = 0; CAmount best_waste = MAX_MONEY; - bool is_feerate_high = utxo_pool.at(0).fee > utxo_pool.at(0).long_term_fee; + // The weight of the currently selected input set + int curr_weight = 0; + + // Whether the input sets generated during this search have exceeded the maximum transaction weight at any point bool max_tx_weight_exceeded = false; - // Depth First search loop for choosing the UTXOs - for (size_t curr_try = 0, utxo_pool_index = 0; curr_try < TOTAL_TRIES; ++curr_try, ++utxo_pool_index) { - result.SetSelectionsEvaluated(curr_try); - // Conditions for starting a backtrack - bool backtrack = false; - if (curr_value + curr_available_value < selection_target || // Cannot possibly reach target with the amount remaining in the curr_available_value. - curr_value > selection_target + cost_of_change || // Selected value is out of range, go back and try other branch - (curr_waste > best_waste && is_feerate_high)) { // Don't select things which we know will be more wasteful if the waste is increasing - backtrack = true; - } else if (curr_selection_weight > max_selection_weight) { // Selected UTXOs weight exceeds the maximum weight allowed, cannot find more solutions by adding more inputs - max_tx_weight_exceeded = true; // at least one selection attempt exceeded the max weight - backtrack = true; - } else if (curr_value >= selection_target) { // Selected value is within range - curr_waste += (curr_value - selection_target); // This is the excess value which is added to the waste for the below comparison - // Adding another UTXO after this check could bring the waste down if the long term fee is higher than the current fee. - // However we are not going to explore that because this optimization for the waste is only done when we have hit our target - // value. Adding any more UTXOs will be just burning the UTXO; it will go entirely to fees. Thus we aren't going to - // explore any more UTXOs to avoid burning money like that. + // Index of the next UTXO to consider in utxo_pool + size_t next_utxo = 0; + + auto deselect_last = [&]() { + OutputGroup& utxo = utxo_pool[curr_selection.back()]; + curr_amount -= utxo.GetSelectionAmount(); + curr_weight -= utxo.m_weight; + curr_selection_waste -= utxo.fee - utxo.long_term_fee; + curr_selection.pop_back(); + }; + + size_t curr_try = 0; + while (true) { + bool should_shift{false}, should_cut{false}; + // Select `next_utxo` + OutputGroup& utxo = utxo_pool[next_utxo]; + curr_amount += utxo.GetSelectionAmount(); + curr_weight += utxo.m_weight; + curr_selection_waste += utxo.fee - utxo.long_term_fee; + curr_selection.push_back(next_utxo); + ++next_utxo; + ++curr_try; + + // EVALUATE current selection: check for solutions and see whether we can CUT or SHIFT before EXPLORING further + if (curr_weight > max_selection_weight) { + // max_weight exceeded: SHIFT + max_tx_weight_exceeded = true; + should_shift = true; + } else if (curr_amount > selection_target + cost_of_change) { + // Overshot target range: SHIFT + should_shift = true; + } else if (curr_amount >= selection_target) { + // Selection is within target window: potential solution + // Adding more UTXOs only increases fees and cannot be better: SHIFT + should_shift = true; + // The amount exceeding the selection_target (the "excess"), would be dropped to the fees: it is waste. + CAmount curr_excess = curr_amount - selection_target; + CAmount curr_waste = curr_selection_waste + curr_excess; if (curr_waste <= best_waste) { + // New best solution best_selection = curr_selection; best_waste = curr_waste; } - curr_waste -= (curr_value - selection_target); // Remove the excess value as we will be selecting different coins now - backtrack = true; } - if (backtrack) { // Backtracking, moving backwards - if (curr_selection.empty()) { // We have walked back to the first utxo and no branch is untraversed. All solutions searched + if (curr_try >= TOTAL_TRIES) { + // Solution is not guaranteed to be optimal if `curr_try` hit TOTAL_TRIES + break; + } + + if (next_utxo == utxo_pool.size()) { + // Last added UTXO was end of UTXO pool, nothing left to add on inclusion or omission branch: CUT + should_cut = true; + } + + if (should_cut) { + // Neither adding to the current selection nor exploring the omission branch of the last selected UTXO can + // find any solutions. Redirect to exploring the Omission branch of the penultimate selected UTXO (i.e. + // set `next_utxo` to one after the penultimate selected, then deselect the last two selected UTXOs) + deselect_last(); + should_shift = true; + } + + if (should_shift) { + if (curr_selection.empty()) { + // Exhausted search space before running into attempt limit break; } - - // Add omitted UTXOs back to lookahead before traversing the omission branch of last included UTXO. - for (--utxo_pool_index; utxo_pool_index > curr_selection.back(); --utxo_pool_index) { - curr_available_value += utxo_pool.at(utxo_pool_index).GetSelectionAmount(); - } - - // Output was included on previous iterations, try excluding now. - assert(utxo_pool_index == curr_selection.back()); - OutputGroup& utxo = utxo_pool.at(utxo_pool_index); - curr_value -= utxo.GetSelectionAmount(); - curr_waste -= utxo.fee - utxo.long_term_fee; - curr_selection_weight -= utxo.m_weight; - curr_selection.pop_back(); - } else { // Moving forwards, continuing down this branch - OutputGroup& utxo = utxo_pool.at(utxo_pool_index); - - // Remove this utxo from the curr_available_value utxo amount - curr_available_value -= utxo.GetSelectionAmount(); - - if (curr_selection.empty() || - // The previous index is included and therefore not relevant for exclusion shortcut - (utxo_pool_index - 1) == curr_selection.back() || - // Avoid searching a branch if the previous UTXO has the same value and same waste and was excluded. - // Since the ratio of fee to long term fee is the same, we only need to check if one of those values match in order to know that the waste is the same. - utxo.GetSelectionAmount() != utxo_pool.at(utxo_pool_index - 1).GetSelectionAmount() || - utxo.fee != utxo_pool.at(utxo_pool_index - 1).fee) - { - // Inclusion branch first (Largest First Exploration) - curr_selection.push_back(utxo_pool_index); - curr_value += utxo.GetSelectionAmount(); - curr_waste += utxo.fee - utxo.long_term_fee; - curr_selection_weight += utxo.m_weight; - } + // Set `next_utxo` to one after last selected, then deselect last selected UTXO + next_utxo = curr_selection.back() + 1; + deselect_last(); } } - // Check for solution + SelectionResult result(selection_target, SelectionAlgorithm::BNB); + result.SetSelectionsEvaluated(curr_try); + if (best_selection.empty()) { return max_tx_weight_exceeded ? ErrorMaxWeightExceeded() : util::Error(); } - // Set output set for (const size_t& i : best_selection) { result.AddInput(utxo_pool.at(i)); } - result.RecalculateWaste(cost_of_change, cost_of_change, CAmount{0}); - assert(best_waste == result.GetWaste()); return result; } + /* * TL;DR: Coin Grinder is a DFS-based algorithm that deterministically searches for the minimum-weight input set to fund * the transaction. The algorithm is similar to the Branch and Bound algorithm, but will produce a transaction _with_ a diff --git a/src/wallet/test/coinselection_tests.cpp b/src/wallet/test/coinselection_tests.cpp index 2f982b74672..05e36167a09 100644 --- a/src/wallet/test/coinselection_tests.cpp +++ b/src/wallet/test/coinselection_tests.cpp @@ -150,19 +150,19 @@ BOOST_AUTO_TEST_CASE(bnb_test) AddCoins(utxo_pool, {1 * CENT, 3 * CENT, 5 * CENT}, cs_params); // Simple success cases - TestBnBSuccess("Select smallest UTXO", utxo_pool, /*selection_target=*/1 * CENT, /*expected_input_amounts=*/{1 * CENT}, /*expected_attempts=*/6, cs_params); - TestBnBSuccess("Select middle UTXO", utxo_pool, /*selection_target=*/3 * CENT, /*expected_input_amounts=*/{3 * CENT}, /*expected_attempts=*/4, cs_params); - TestBnBSuccess("Select biggest UTXO", utxo_pool, /*selection_target=*/5 * CENT, /*expected_input_amounts=*/{5 * CENT}, /*expected_attempts=*/2, cs_params); - TestBnBSuccess("Select two UTXOs", utxo_pool, /*selection_target=*/4 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/6, cs_params); - TestBnBSuccess("Select all UTXOs", utxo_pool, /*selection_target=*/9 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT, 5 * CENT}, /*expected_attempts=*/6, cs_params); + TestBnBSuccess("Select smallest UTXO", utxo_pool, /*selection_target=*/1 * CENT, /*expected_input_amounts=*/{1 * CENT}, /*expected_attempts=*/3, cs_params); + TestBnBSuccess("Select middle UTXO", utxo_pool, /*selection_target=*/3 * CENT, /*expected_input_amounts=*/{3 * CENT}, /*expected_attempts=*/3, cs_params); + TestBnBSuccess("Select biggest UTXO", utxo_pool, /*selection_target=*/5 * CENT, /*expected_input_amounts=*/{5 * CENT}, /*expected_attempts=*/4, cs_params); + TestBnBSuccess("Select two UTXOs", utxo_pool, /*selection_target=*/4 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/4, cs_params); + TestBnBSuccess("Select all UTXOs", utxo_pool, /*selection_target=*/9 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT, 5 * CENT}, /*expected_attempts=*/7, cs_params); // BnB finds changeless solution while overshooting by up to cost_of_change - TestBnBSuccess("Select upper bound", utxo_pool, /*selection_target=*/4 * CENT - cs_params.m_cost_of_change, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/6, cs_params); + TestBnBSuccess("Select upper bound", utxo_pool, /*selection_target=*/4 * CENT - cs_params.m_cost_of_change, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/4, cs_params); // BnB fails to find changeless solution when overshooting by cost_of_change + 1 sat TestBnBFail("Overshoot upper bound", utxo_pool, /*selection_target=*/4 * CENT - cs_params.m_cost_of_change - 1, cs_params); - TestBnBSuccess("Select max weight", utxo_pool, /*selection_target=*/4 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/6, cs_params, /*custom_spending_vsize=*/P2WPKH_INPUT_VSIZE, /*max_selection_weight=*/4 * 2 * P2WPKH_INPUT_VSIZE); + TestBnBSuccess("Select max weight", utxo_pool, /*selection_target=*/4 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/4, cs_params, /*custom_spending_vsize=*/P2WPKH_INPUT_VSIZE, /*max_selection_weight=*/4 * 2 * P2WPKH_INPUT_VSIZE); TestBnBFail("Exceed max weight", utxo_pool, /*selection_target=*/4 * CENT, cs_params, /*max_selection_weight=*/4 * 2 * P2WPKH_INPUT_VSIZE - 1, /*expect_max_weight_exceeded=*/true); @@ -175,7 +175,7 @@ BOOST_AUTO_TEST_CASE(bnb_test) std::vector clone_pool; AddCoins(clone_pool, {2 * CENT, 7 * CENT, 7 * CENT}, cs_params); AddDuplicateCoins(clone_pool, /*count=*/50'000, /*amount=*/5 * CENT, cs_params); - TestBnBSuccess("Skip equivalent input sets", clone_pool, /*selection_target=*/16 * CENT, /*expected_input_amounts=*/{2 * CENT, 7 * CENT, 7 * CENT}, /*expected_attempts=*/99'999, cs_params); + TestBnBSuccess("Skip equivalent input sets", clone_pool, /*selection_target=*/16 * CENT, /*expected_input_amounts=*/{2 * CENT, 7 * CENT, 7 * CENT}, /*expected_attempts=*/100'000, cs_params); /* Test BnB attempt limit (`TOTAL_TRIES`) * @@ -208,7 +208,7 @@ BOOST_AUTO_TEST_CASE(bnb_test) } AddCoins(doppelganger_pool, doppelgangers, cs_params); // Among up to 17 unique UTXOs of similar effective value we will find a solution composed of the eight smallest UTXOs - TestBnBSuccess("Combine smallest 8 of 17 unique UTXOs", doppelganger_pool, /*selection_target=*/8 * CENT, /*expected_input_amounts=*/expected_inputs, /*expected_attempts=*/87'514, cs_params); + TestBnBSuccess("Combine smallest 8 of 17 unique UTXOs", doppelganger_pool, /*selection_target=*/8 * CENT, /*expected_input_amounts=*/expected_inputs, /*expected_attempts=*/65'535, cs_params); // Starting with 18 unique UTXOs of similar effective value we will not find the solution due to exceeding the attempt limit AddCoins(doppelganger_pool, {1 * CENT + cs_params.m_cost_of_change + 17}, cs_params); @@ -226,7 +226,7 @@ BOOST_AUTO_TEST_CASE(bnb_feerate_sensitivity_test) const CoinSelectionParams high_feerate_params = init_cs_params(/*eff_feerate=*/25'000); std::vector high_feerate_pool; // 25 sat/vB (greater than long_term_feerate of 10 sat/vB) AddCoins(high_feerate_pool, {2 * CENT, 3 * CENT, 5 * CENT, 10 * CENT}, high_feerate_params); - TestBnBSuccess("Select one input at high feerates", high_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{10 * CENT}, /*expected_attempts=*/6, high_feerate_params); + TestBnBSuccess("Select one input at high feerates", high_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{10 * CENT}, /*expected_attempts=*/8, high_feerate_params); // Add heavy inputs {6, 7} to existing {2, 3, 5, 10} low_feerate_pool.push_back(MakeCoin(6 * CENT, true, default_cs_params, /*custom_spending_vsize=*/500)); @@ -235,7 +235,7 @@ BOOST_AUTO_TEST_CASE(bnb_feerate_sensitivity_test) high_feerate_pool.push_back(MakeCoin(6 * CENT, true, high_feerate_params, /*custom_spending_vsize=*/500)); high_feerate_pool.push_back(MakeCoin(7 * CENT, true, high_feerate_params, /*custom_spending_vsize=*/500)); - TestBnBSuccess("Prefer two light inputs over two heavy inputs at high feerates", high_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{3 * CENT, 10 * CENT}, /*expected_attempts=*/14, high_feerate_params); + TestBnBSuccess("Prefer two light inputs over two heavy inputs at high feerates", high_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{3 * CENT, 10 * CENT}, /*expected_attempts=*/28, high_feerate_params); } static void TestSRDSuccess(std::string test_title, std::vector& utxo_pool, const CAmount& selection_target, const CoinSelectionParams& cs_params = default_cs_params, const int max_selection_weight = MAX_STANDARD_TX_WEIGHT) diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 2b8150c6ee0..f393b473069 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -209,7 +209,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) const auto result9 = SelectCoinsBnB(GroupCoins(available_coins.All()), 1 * CENT, coin_selection_params_bnb.m_cost_of_change); BOOST_CHECK(result9); BOOST_CHECK_EQUAL(result9->GetSelectedValue(), 1 * CENT); - expected_attempts = 2; + expected_attempts = 1; BOOST_CHECK_MESSAGE(result9->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, result9->GetSelectionsEvaluated())); } @@ -233,7 +233,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) LOCK(wallet->cs_wallet); const auto result10 = SelectCoins(*wallet, available_coins, selected_input, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(result10); - expected_attempts = 4; + expected_attempts = 3; BOOST_CHECK_MESSAGE(result10->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, result10->GetSelectionsEvaluated())); } { @@ -264,7 +264,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) available_coins.Erase({(++available_coins.coins[OutputType::BECH32].begin())->outpoint}); const auto result13 = SelectCoins(*wallet, available_coins, selected_input, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result13)); - expected_attempts = 4; + expected_attempts = 2; BOOST_CHECK_MESSAGE(result13->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, result13->GetSelectionsEvaluated())); } @@ -297,7 +297,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) add_coin(5 * CENT, 2, expected_result); add_coin(3 * CENT, 2, expected_result); BOOST_CHECK(EquivalentResult(expected_result, *res)); - expected_attempts = 38; + expected_attempts = 39; BOOST_CHECK_MESSAGE(res->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, res->GetSelectionsEvaluated())); } } From 7ecea1dc5dcea31ea166cb779f458d3993ee733e Mon Sep 17 00:00:00 2001 From: Murch Date: Wed, 26 Mar 2025 17:18:18 -0700 Subject: [PATCH 3/7] coinselection: Track whether BnB completed BnB may not be able to exhaustively search all potentially interesting combinations for large UTXO pools, so we keep track of whether the search was terminated by the iteration limit. --- src/wallet/coinselection.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 6755376ff51..e671c3d091f 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -131,6 +131,7 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool }; size_t curr_try = 0; + SelectionResult result(selection_target, SelectionAlgorithm::BNB); while (true) { bool should_shift{false}, should_cut{false}; // Select `next_utxo` @@ -166,6 +167,7 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool if (curr_try >= TOTAL_TRIES) { // Solution is not guaranteed to be optimal if `curr_try` hit TOTAL_TRIES + result.SetAlgoCompleted(false); break; } @@ -185,6 +187,7 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool if (should_shift) { if (curr_selection.empty()) { // Exhausted search space before running into attempt limit + result.SetAlgoCompleted(true); break; } // Set `next_utxo` to one after last selected, then deselect last selected UTXO @@ -193,7 +196,6 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool } } - SelectionResult result(selection_target, SelectionAlgorithm::BNB); result.SetSelectionsEvaluated(curr_try); if (best_selection.empty()) { From fa226ab902fe1aa51ec039a2cdfe586855b75f9d Mon Sep 17 00:00:00 2001 From: Murch Date: Wed, 26 Mar 2025 18:17:27 -0700 Subject: [PATCH 4/7] coinselection: BnB skip exploring high waste MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At high feerates adding more inputs will increase the waste score. If the current waste is already higher than the best selection’s we cannot improve upon the best selection. All solutions that include the current selection with more additional inputs must be worse than the best selection so far: SHIFT This optimization only works at high feerates, because at low feerates, adding more inputs decreases waste, so this condition would exit prematurely. We would never attempt input sets with higher weight than the prior best selection, even though we would prefer those at low feerates. --- src/wallet/coinselection.cpp | 9 +++++++++ src/wallet/test/coinselection_tests.cpp | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index e671c3d091f..cce4c59ca15 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -132,6 +132,8 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool size_t curr_try = 0; SelectionResult result(selection_target, SelectionAlgorithm::BNB); + // We don’t have access to the feerate here, but fee to long_term_fee is as feerate to LTFRE + bool is_feerate_high = utxo_pool.at(0).fee > utxo_pool.at(0).long_term_fee; while (true) { bool should_shift{false}, should_cut{false}; // Select `next_utxo` @@ -151,6 +153,13 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool } else if (curr_amount > selection_target + cost_of_change) { // Overshot target range: SHIFT should_shift = true; + } else if (is_feerate_high && curr_selection_waste > best_waste) { + // At high feerates adding more inputs will increase the waste score. If the current waste is already worse + // than the best selection’s while we have insufficient funds, it is impossible for this partial selection + // to beat the best selection by adding more inputs: SHIFT + // At low feerates, additional inputs lower the waste score, and using this would cause us to skip exploring + // combinations with more inputs of lower amounts. + should_shift = true; } else if (curr_amount >= selection_target) { // Selection is within target window: potential solution // Adding more UTXOs only increases fees and cannot be better: SHIFT diff --git a/src/wallet/test/coinselection_tests.cpp b/src/wallet/test/coinselection_tests.cpp index 05e36167a09..6a0d5e48d21 100644 --- a/src/wallet/test/coinselection_tests.cpp +++ b/src/wallet/test/coinselection_tests.cpp @@ -226,7 +226,7 @@ BOOST_AUTO_TEST_CASE(bnb_feerate_sensitivity_test) const CoinSelectionParams high_feerate_params = init_cs_params(/*eff_feerate=*/25'000); std::vector high_feerate_pool; // 25 sat/vB (greater than long_term_feerate of 10 sat/vB) AddCoins(high_feerate_pool, {2 * CENT, 3 * CENT, 5 * CENT, 10 * CENT}, high_feerate_params); - TestBnBSuccess("Select one input at high feerates", high_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{10 * CENT}, /*expected_attempts=*/8, high_feerate_params); + TestBnBSuccess("Select one input at high feerates", high_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{10 * CENT}, /*expected_attempts=*/7, high_feerate_params); // Add heavy inputs {6, 7} to existing {2, 3, 5, 10} low_feerate_pool.push_back(MakeCoin(6 * CENT, true, default_cs_params, /*custom_spending_vsize=*/500)); @@ -235,7 +235,7 @@ BOOST_AUTO_TEST_CASE(bnb_feerate_sensitivity_test) high_feerate_pool.push_back(MakeCoin(6 * CENT, true, high_feerate_params, /*custom_spending_vsize=*/500)); high_feerate_pool.push_back(MakeCoin(7 * CENT, true, high_feerate_params, /*custom_spending_vsize=*/500)); - TestBnBSuccess("Prefer two light inputs over two heavy inputs at high feerates", high_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{3 * CENT, 10 * CENT}, /*expected_attempts=*/28, high_feerate_params); + TestBnBSuccess("Prefer two light inputs over two heavy inputs at high feerates", high_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{3 * CENT, 10 * CENT}, /*expected_attempts=*/15, high_feerate_params); } static void TestSRDSuccess(std::string test_title, std::vector& utxo_pool, const CAmount& selection_target, const CoinSelectionParams& cs_params = default_cs_params, const int max_selection_weight = MAX_STANDARD_TX_WEIGHT) From ba1807b981a966b08eee7ba848acab190214977a Mon Sep 17 00:00:00 2001 From: Murch Date: Wed, 26 Mar 2025 18:24:22 -0700 Subject: [PATCH 5/7] coinselection: Track effective_value lookahead Introduces a dedicated data structure to track the total effective_value available in the remaining UTXOs at each index of the UTXO pool. In contrast to the original approach in BnB, this allows us to immediately jump to a lower index instead of visiting every UTXO to add back their eff_value to the lookahead. --- src/wallet/coinselection.cpp | 25 ++++++++++++++++++------- src/wallet/test/coinselection_tests.cpp | 24 ++++++++++++++---------- src/wallet/test/coinselector_tests.cpp | 2 +- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index cce4c59ca15..febac429bc7 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -68,6 +68,10 @@ struct { * * waste = selectionTotal - target + inputs × (currentFeeRate - longTermFeeRate) * + * The algorithm uses one additional optimization: a lookahead keeps track of the total value of + * the unexplored UTXOs. A subtree is not explored if the lookahead indicates that the target range + * cannot be reached. + * * The Branch and Bound algorithm is described in detail in Murch's Master Thesis: * https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf * @@ -87,12 +91,17 @@ static const size_t TOTAL_TRIES = 100000; util::Result SelectCoinsBnB(std::vector& utxo_pool, const CAmount& selection_target, const CAmount& cost_of_change, int max_selection_weight) { - // Check that there are sufficient funds + std::sort(utxo_pool.begin(), utxo_pool.end(), descending); + // The sum of UTXO amounts after this UTXO index, e.g. lookahead[5] = Σ(UTXO[6+].amount) + std::vector lookahead(utxo_pool.size()); + + // Calculate lookahead values, and check that there are sufficient funds CAmount total_available = 0; - for (const OutputGroup& utxo : utxo_pool) { - // Assert UTXOs with non-positive effective value have been filtered - Assume(utxo.GetSelectionAmount() > 0); - total_available += utxo.GetSelectionAmount(); + for (int index = static_cast(utxo_pool.size()) - 1; index >= 0; --index) { + lookahead[index] = total_available; + // UTXOs with non-positive effective value must have been filtered + Assume(utxo_pool[index].GetSelectionAmount() > 0); + total_available += utxo_pool[index].GetSelectionAmount(); } if (total_available < selection_target) { @@ -100,7 +109,6 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool return util::Error(); } - std::sort(utxo_pool.begin(), utxo_pool.end(), descending); // The current selection and the best input set found so far, stored as the utxo_pool indices of the UTXOs forming them std::vector curr_selection; @@ -146,7 +154,10 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool ++curr_try; // EVALUATE current selection: check for solutions and see whether we can CUT or SHIFT before EXPLORING further - if (curr_weight > max_selection_weight) { + if (curr_amount + lookahead[curr_selection.back()] < selection_target) { + // Insufficient funds with lookahead: CUT + should_cut = true; + } else if (curr_weight > max_selection_weight) { // max_weight exceeded: SHIFT max_tx_weight_exceeded = true; should_shift = true; diff --git a/src/wallet/test/coinselection_tests.cpp b/src/wallet/test/coinselection_tests.cpp index 6a0d5e48d21..e1b1b0748f5 100644 --- a/src/wallet/test/coinselection_tests.cpp +++ b/src/wallet/test/coinselection_tests.cpp @@ -152,9 +152,9 @@ BOOST_AUTO_TEST_CASE(bnb_test) // Simple success cases TestBnBSuccess("Select smallest UTXO", utxo_pool, /*selection_target=*/1 * CENT, /*expected_input_amounts=*/{1 * CENT}, /*expected_attempts=*/3, cs_params); TestBnBSuccess("Select middle UTXO", utxo_pool, /*selection_target=*/3 * CENT, /*expected_input_amounts=*/{3 * CENT}, /*expected_attempts=*/3, cs_params); - TestBnBSuccess("Select biggest UTXO", utxo_pool, /*selection_target=*/5 * CENT, /*expected_input_amounts=*/{5 * CENT}, /*expected_attempts=*/4, cs_params); + TestBnBSuccess("Select biggest UTXO", utxo_pool, /*selection_target=*/5 * CENT, /*expected_input_amounts=*/{5 * CENT}, /*expected_attempts=*/2, cs_params); TestBnBSuccess("Select two UTXOs", utxo_pool, /*selection_target=*/4 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/4, cs_params); - TestBnBSuccess("Select all UTXOs", utxo_pool, /*selection_target=*/9 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT, 5 * CENT}, /*expected_attempts=*/7, cs_params); + TestBnBSuccess("Select all UTXOs", utxo_pool, /*selection_target=*/9 * CENT, /*expected_input_amounts=*/{1 * CENT, 3 * CENT, 5 * CENT}, /*expected_attempts=*/5, cs_params); // BnB finds changeless solution while overshooting by up to cost_of_change TestBnBSuccess("Select upper bound", utxo_pool, /*selection_target=*/4 * CENT - cs_params.m_cost_of_change, /*expected_input_amounts=*/{1 * CENT, 3 * CENT}, /*expected_attempts=*/4, cs_params); @@ -207,12 +207,16 @@ BOOST_AUTO_TEST_CASE(bnb_test) } } AddCoins(doppelganger_pool, doppelgangers, cs_params); - // Among up to 17 unique UTXOs of similar effective value we will find a solution composed of the eight smallest UTXOs - TestBnBSuccess("Combine smallest 8 of 17 unique UTXOs", doppelganger_pool, /*selection_target=*/8 * CENT, /*expected_input_amounts=*/expected_inputs, /*expected_attempts=*/65'535, cs_params); + // Among 17 unique UTXOs of similar effective value we will find a solution composed of the eight smallest UTXOs + TestBnBSuccess("Combine smallest 8 of 17 unique UTXOs", doppelganger_pool, /*selection_target=*/8 * CENT, /*expected_input_amounts=*/expected_inputs, /*expected_attempts=*/51'765, cs_params); - // Starting with 18 unique UTXOs of similar effective value we will not find the solution due to exceeding the attempt limit + // Among up to 18 unique UTXOs of similar effective value we will find a solution composed of the eight smallest UTXOs AddCoins(doppelganger_pool, {1 * CENT + cs_params.m_cost_of_change + 17}, cs_params); - TestBnBFail("Exhaust looking for smallest 8 of 18 unique UTXOs", doppelganger_pool, /*selection_target=*/8 * CENT, cs_params); + TestBnBSuccess("Combine smallest 8 of 18 unique UTXOs", doppelganger_pool, /*selection_target=*/8 * CENT, /*expected_input_amounts=*/expected_inputs, /*expected_attempts=*/87'957, cs_params); + + // Starting with 19 unique UTXOs of similar effective value we will not find the solution due to exceeding the attempt limit + AddCoins(doppelganger_pool, {1 * CENT + cs_params.m_cost_of_change + 18}, cs_params); + TestBnBFail("Exhaust looking for smallest 8 of 19 unique UTXOs", doppelganger_pool, /*selection_target=*/8 * CENT, cs_params); } } @@ -221,21 +225,21 @@ BOOST_AUTO_TEST_CASE(bnb_feerate_sensitivity_test) // Create sets of UTXOs with the same effective amounts at different feerates (but different absolute amounts) std::vector low_feerate_pool; // 5 sat/vB (default, and lower than long_term_feerate of 10 sat/vB) AddCoins(low_feerate_pool, {2 * CENT, 3 * CENT, 5 * CENT, 10 * CENT}); - TestBnBSuccess("Select many inputs at low feerates", low_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{2 * CENT, 3 * CENT, 5 * CENT}, /*expected_attempts=*/8); + TestBnBSuccess("Select many inputs at low feerates", low_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{2 * CENT, 3 * CENT, 5 * CENT}, /*expected_attempts=*/6); const CoinSelectionParams high_feerate_params = init_cs_params(/*eff_feerate=*/25'000); std::vector high_feerate_pool; // 25 sat/vB (greater than long_term_feerate of 10 sat/vB) AddCoins(high_feerate_pool, {2 * CENT, 3 * CENT, 5 * CENT, 10 * CENT}, high_feerate_params); - TestBnBSuccess("Select one input at high feerates", high_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{10 * CENT}, /*expected_attempts=*/7, high_feerate_params); + TestBnBSuccess("Select one input at high feerates", high_feerate_pool, /*selection_target=*/10 * CENT, /*expected_input_amounts=*/{10 * CENT}, /*expected_attempts=*/5, high_feerate_params); // Add heavy inputs {6, 7} to existing {2, 3, 5, 10} low_feerate_pool.push_back(MakeCoin(6 * CENT, true, default_cs_params, /*custom_spending_vsize=*/500)); low_feerate_pool.push_back(MakeCoin(7 * CENT, true, default_cs_params, /*custom_spending_vsize=*/500)); - TestBnBSuccess("Prefer two heavy inputs over two light inputs at low feerates", low_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{6 * CENT, 7 * CENT}, /*expected_attempts=*/28, default_cs_params, /*custom_spending_vsize=*/500); + TestBnBSuccess("Prefer two heavy inputs over two light inputs at low feerates", low_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{6 * CENT, 7 * CENT}, /*expected_attempts=*/18, default_cs_params, /*custom_spending_vsize=*/500); high_feerate_pool.push_back(MakeCoin(6 * CENT, true, high_feerate_params, /*custom_spending_vsize=*/500)); high_feerate_pool.push_back(MakeCoin(7 * CENT, true, high_feerate_params, /*custom_spending_vsize=*/500)); - TestBnBSuccess("Prefer two light inputs over two heavy inputs at high feerates", high_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{3 * CENT, 10 * CENT}, /*expected_attempts=*/15, high_feerate_params); + TestBnBSuccess("Prefer two light inputs over two heavy inputs at high feerates", high_feerate_pool, /*selection_target=*/13 * CENT, /*expected_input_amounts=*/{3 * CENT, 10 * CENT}, /*expected_attempts=*/9, high_feerate_params); } static void TestSRDSuccess(std::string test_title, std::vector& utxo_pool, const CAmount& selection_target, const CoinSelectionParams& cs_params = default_cs_params, const int max_selection_weight = MAX_STANDARD_TX_WEIGHT) diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index f393b473069..126c18835d1 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -297,7 +297,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) add_coin(5 * CENT, 2, expected_result); add_coin(3 * CENT, 2, expected_result); BOOST_CHECK(EquivalentResult(expected_result, *res)); - expected_attempts = 39; + expected_attempts = 25; BOOST_CHECK_MESSAGE(res->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, res->GetSelectionsEvaluated())); } } From 52042918606efd7bf26a3dfa5bb207187d7d481c Mon Sep 17 00:00:00 2001 From: Murch Date: Wed, 26 Mar 2025 18:46:21 -0700 Subject: [PATCH 6/7] opt: Skip evaluation of equivalent input sets When two successive UTXOs match in effective value and weight, we can skip the second if the prior is not selected: adding it would create an equivalent input set to a previously evaluated. E.g. if we have three UTXOs with effective values {5, 3, 3} of the same weight each, we want to evaluate {5, _, _}, {5, 3, _}, {5, 3, 3}, {_, 3, _}, {_, 3, 3}, but skip {5, _, 3}, and {_, _, 3}, because the first 3 is not selected, and we therefore do not need to evaluate the second 3 at the same position in the input set. If we reach the end of the branch, we must SHIFT the previously selected UTXO group instead. --- src/wallet/coinselection.cpp | 29 +++++++++++++++++++++---- src/wallet/test/coinselection_tests.cpp | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index febac429bc7..8a4919a5588 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -68,9 +68,11 @@ struct { * * waste = selectionTotal - target + inputs × (currentFeeRate - longTermFeeRate) * - * The algorithm uses one additional optimization: a lookahead keeps track of the total value of + * The algorithm uses two additional optimizations. A lookahead keeps track of the total value of * the unexplored UTXOs. A subtree is not explored if the lookahead indicates that the target range - * cannot be reached. + * cannot be reached. Further, it is unnecessary to test equivalent combinations. This allows us + * to skip testing the inclusion of UTXOs that match the effective value and waste of an omitted + * predecessor. * * The Branch and Bound algorithm is described in detail in Murch's Master Thesis: * https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf @@ -140,9 +142,10 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool size_t curr_try = 0; SelectionResult result(selection_target, SelectionAlgorithm::BNB); + bool is_done = false; // We don’t have access to the feerate here, but fee to long_term_fee is as feerate to LTFRE bool is_feerate_high = utxo_pool.at(0).fee > utxo_pool.at(0).long_term_fee; - while (true) { + while (!is_done) { bool should_shift{false}, should_cut{false}; // Select `next_utxo` OutputGroup& utxo = utxo_pool[next_utxo]; @@ -204,15 +207,33 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool should_shift = true; } - if (should_shift) { + while (should_shift) { if (curr_selection.empty()) { // Exhausted search space before running into attempt limit + is_done = true; result.SetAlgoCompleted(true); break; } // Set `next_utxo` to one after last selected, then deselect last selected UTXO next_utxo = curr_selection.back() + 1; deselect_last(); + should_shift = false; + + // After SHIFTing to an omission branch, the `next_utxo` might have the same value and same weight as the + // UTXO we just omitted (i.e. it is a "clone"). If so, selecting `next_utxo` would produce an equivalent + // selection as one we previously evaluated. In that case, increment `next_utxo` until we find a UTXO with a + // differing amount or weight. + Assume(next_utxo < utxo_pool.size()); + while (utxo_pool[next_utxo - 1].GetSelectionAmount() == utxo_pool[next_utxo].GetSelectionAmount() + && utxo_pool[next_utxo - 1].m_weight == utxo_pool[next_utxo].m_weight) { + if (next_utxo >= utxo_pool.size() - 1) { + // Reached end of UTXO pool skipping clones: SHIFT instead + should_shift = true; + break; + } + // Skip clone: previous UTXO is equivalent and unselected + ++next_utxo; + } } } diff --git a/src/wallet/test/coinselection_tests.cpp b/src/wallet/test/coinselection_tests.cpp index e1b1b0748f5..a8d2ef2c58a 100644 --- a/src/wallet/test/coinselection_tests.cpp +++ b/src/wallet/test/coinselection_tests.cpp @@ -175,7 +175,7 @@ BOOST_AUTO_TEST_CASE(bnb_test) std::vector clone_pool; AddCoins(clone_pool, {2 * CENT, 7 * CENT, 7 * CENT}, cs_params); AddDuplicateCoins(clone_pool, /*count=*/50'000, /*amount=*/5 * CENT, cs_params); - TestBnBSuccess("Skip equivalent input sets", clone_pool, /*selection_target=*/16 * CENT, /*expected_input_amounts=*/{2 * CENT, 7 * CENT, 7 * CENT}, /*expected_attempts=*/100'000, cs_params); + TestBnBSuccess("Skip equivalent input sets", clone_pool, /*selection_target=*/16 * CENT, /*expected_input_amounts=*/{2 * CENT, 7 * CENT, 7 * CENT}, /*expected_attempts=*/16, cs_params); /* Test BnB attempt limit (`TOTAL_TRIES`) * From 7249b376a0adac3987977b47e74792dcb148af74 Mon Sep 17 00:00:00 2001 From: Murch Date: Wed, 26 Mar 2025 18:59:30 -0700 Subject: [PATCH 7/7] opt: Skip UTXOs with worse waste, same eff_value When two successive UTXOs differ in waste but match in effective value, we can skip the second if the first is not selected, because all input sets we can generate by swapping out a less wasteful UTXOs with a more wastefull UTXO of matching effective value would be strictly worse. Also expand documentation of Branch and Bound. --- src/wallet/coinselection.cpp | 53 ++++++++++++++++++-------- src/wallet/test/coinselector_tests.cpp | 2 +- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 8a4919a5588..c3fc15eaf66 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -54,25 +54,47 @@ struct { * set that can pay for the spending target and does not exceed the spending target by more than the * cost of creating and spending a change output. The algorithm uses a depth-first search on a binary * tree. In the binary tree, each node corresponds to the inclusion or the omission of a UTXO. UTXOs - * are sorted by their effective values and the tree is explored deterministically per the inclusion + * are sorted by their effective values, tie-broken by their waste score, and the tree is explored deterministically per the inclusion * branch first. For each new input set candidate, the algorithm checks whether the selection is within the target range. * While the selection has not reached the target range, more UTXOs are included. When a selection's - * value exceeds the target range, the complete subtree deriving from this selection can be omitted. + * value exceeds the target range, the complete subtree deriving from this selection prefix can be omitted. * At that point, the last included UTXO is deselected and the corresponding omission branch explored * instead starting by adding the subsequent UTXO. The search ends after the complete tree has been searched or after a limited number of tries. * * The algorithm continues to search for better solutions after one solution has been found. The best - * solution is chosen by minimizing the waste metric. The waste metric is defined as the cost to + * solution is chosen by minimal waste score. The waste metric is defined as the cost to * spend the current inputs at the given fee rate minus the long term expected cost to spend the - * inputs, plus the amount by which the selection exceeds the spending target: + * inputs, plus the amount by which the selection exceeds the spending target (the "excess"): * - * waste = selectionTotal - target + inputs × (currentFeeRate - longTermFeeRate) + * excess = selected_amount - target + * waste = inputs × (currentFeeRate - longTermFeeRate) + excess * - * The algorithm uses two additional optimizations. A lookahead keeps track of the total value of - * the unexplored UTXOs. A subtree is not explored if the lookahead indicates that the target range - * cannot be reached. Further, it is unnecessary to test equivalent combinations. This allows us - * to skip testing the inclusion of UTXOs that match the effective value and waste of an omitted - * predecessor. + * Note that this means that at fee rates higher than longTermFeeRate additional inputs increase the + * waste score, while at fee rates lower than longTermFeeRate additional inputs decrease the waste + * score. + * + * The algorithm uses the following optimizations: + * 1. Lookahead: The lookahead stores the total remaining effective value of the undecided UTXOs for + * every depth of the search tree. Whenever the currently selected amount plus the potential + * amount from the lookahead falls short of the target, we can immediately stop searching the + * subtree as no more input set candidates can be found in it. + * 2. Skip clones: When two UTXOs match in weight and effective value ("are clones"), naive + * exploration would cause redundant work: e.g., given the UTXOs A, A', and B, where A and A' are + * clones, naive exploration would combine (read underscore as omission): + * [{}, {A}, {A, A'}, {A, A', B}, {A, _, B}, {_, A'}, {_, A', B}, {_, _, B}]. + * In this case the input set candidates {A} and {A'} as well as {A, B} and {A', B} are + * equivalent. It is sufficient to explore combinations that select either both UTXOs or the + * first UTXO. Whenever the first UTXO is omitted, we can also skip the clone as we have already + * explored a set of equivalent combination as the one we could generate with the second clone. + * Concretely, we skip a UTXO when its predecessor is omitted and the UTXO matches the + * effective value and the waste of the predecessor. + * 3. Skip similar UTXOs that are more wasteful: This search algorithm operates on the list of UTXOs + * sorted by effective value, tie-broken to prefer lower waste. This means that among two + * subsequent UTXOs with the same effective value, the second UTXO’s waste score will either be + * equal _or higher_ than the first UTXO’s. This allows us to apply the clone skipping idea more + * broadly: any combination with the second UTXO is equivalent _or worse_ than what we already + * combined with the first UTXO. We skip a UTXO if its predecessor is omitted and the predecessor + * matches in effective value. * * The Branch and Bound algorithm is described in detail in Murch's Master Thesis: * https://murch.one/wp-content/uploads/2016/11/erhardt2016coinselection.pdf @@ -219,13 +241,14 @@ util::Result SelectCoinsBnB(std::vector& utxo_pool deselect_last(); should_shift = false; - // After SHIFTing to an omission branch, the `next_utxo` might have the same value and same weight as the - // UTXO we just omitted (i.e. it is a "clone"). If so, selecting `next_utxo` would produce an equivalent + // After SHIFTing to an omission branch, the `next_utxo` might have the same effective value as the + // UTXO we just omitted. Since lower waste is our tiebreaker on UTXOs with equal effective value for sorting, if it + // ties on the effective value, it _must_ have the same waste (i.e. be a "clone" of the prior UTXO) or a + // higher waste. If so, selecting `next_utxo` would produce an equivalent or worse // selection as one we previously evaluated. In that case, increment `next_utxo` until we find a UTXO with a - // differing amount or weight. + // differing amount. Assume(next_utxo < utxo_pool.size()); - while (utxo_pool[next_utxo - 1].GetSelectionAmount() == utxo_pool[next_utxo].GetSelectionAmount() - && utxo_pool[next_utxo - 1].m_weight == utxo_pool[next_utxo].m_weight) { + while (utxo_pool[next_utxo - 1].GetSelectionAmount() == utxo_pool[next_utxo].GetSelectionAmount()) { if (next_utxo >= utxo_pool.size() - 1) { // Reached end of UTXO pool skipping clones: SHIFT instead should_shift = true; diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 126c18835d1..17278a68517 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -297,7 +297,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) add_coin(5 * CENT, 2, expected_result); add_coin(3 * CENT, 2, expected_result); BOOST_CHECK(EquivalentResult(expected_result, *res)); - expected_attempts = 25; + expected_attempts = 22; BOOST_CHECK_MESSAGE(res->GetSelectionsEvaluated() == expected_attempts, strprintf("Expected %i attempts, but got %i", expected_attempts, res->GetSelectionsEvaluated())); } }