highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-03-17 13:22:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 8400928ba1d0e89547c480482a41e05edda05126 into db2c57ae9eebdb75c58cd165ac929919969c19a9

Browse Source
This commit is contained in:
Pieter Wuille 2025-03-17 10:35:19 +01:00 committed by GitHub
commit 91b716a3a7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 4306 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/CMakeLists.txt
View File

@ -301,6 +301,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL
signet.cpp
torcontrol.cpp
txdb.cpp
txgraph.cpp
txmempool.cpp
txorphanage.cpp
txrequest.cpp

44
src/bench/cluster_linearize.cpp
View File

@ -23,10 +23,10 @@ namespace {
* remaining transaction, whose removal requires updating all remaining transactions' ancestor
* set feerates. */
template<typename SetType>
DepGraph<SetType> MakeLinearGraph(ClusterIndex ntx)
DepGraph<SetType> MakeLinearGraph(DepGraphIndex ntx)
{
DepGraph<SetType> depgraph;
for (ClusterIndex i = 0; i < ntx; ++i) {
for (DepGraphIndex i = 0; i < ntx; ++i) {
depgraph.AddTransaction({-int32_t(i), 1});
if (i > 0) depgraph.AddDependencies(SetType::Singleton(i - 1), i);
}
@ -38,10 +38,10 @@ DepGraph<SetType> MakeLinearGraph(ClusterIndex ntx)
* rechunking is needed after every candidate (the last transaction gets picked every time).
*/
template<typename SetType>
DepGraph<SetType> MakeWideGraph(ClusterIndex ntx)
DepGraph<SetType> MakeWideGraph(DepGraphIndex ntx)
{
DepGraph<SetType> depgraph;
for (ClusterIndex i = 0; i < ntx; ++i) {
for (DepGraphIndex i = 0; i < ntx; ++i) {
depgraph.AddTransaction({int32_t(i) + 1, 1});
if (i > 0) depgraph.AddDependencies(SetType::Singleton(0), i);
}
@ -51,10 +51,10 @@ DepGraph<SetType> MakeWideGraph(ClusterIndex ntx)
// Construct a difficult graph. These need at least sqrt(2^(n-1)) iterations in the implemented
// algorithm (purely empirically determined).
template<typename SetType>
DepGraph<SetType> MakeHardGraph(ClusterIndex ntx)
DepGraph<SetType> MakeHardGraph(DepGraphIndex ntx)
{
DepGraph<SetType> depgraph;
for (ClusterIndex i = 0; i < ntx; ++i) {
for (DepGraphIndex i = 0; i < ntx; ++i) {
if (ntx & 1) {
// Odd cluster size.
//
@ -121,7 +121,7 @@ DepGraph<SetType> MakeHardGraph(ClusterIndex ntx)
* iterations difference.
*/
template<typename SetType>
void BenchLinearizeWorstCase(ClusterIndex ntx, benchmark::Bench& bench, uint64_t iter_limit)
void BenchLinearizeWorstCase(DepGraphIndex ntx, benchmark::Bench& bench, uint64_t iter_limit)
{
const auto depgraph = MakeHardGraph<SetType>(ntx);
uint64_t rng_seed = 0;
@ -147,12 +147,12 @@ void BenchLinearizeWorstCase(ClusterIndex ntx, benchmark::Bench& bench, uint64_t
* cheap.
*/
template<typename SetType>
void BenchLinearizeNoItersWorstCaseAnc(ClusterIndex ntx, benchmark::Bench& bench)
void BenchLinearizeNoItersWorstCaseAnc(DepGraphIndex ntx, benchmark::Bench& bench)
{
const auto depgraph = MakeLinearGraph<SetType>(ntx);
uint64_t rng_seed = 0;
std::vector<ClusterIndex> old_lin(ntx);
for (ClusterIndex i = 0; i < ntx; ++i) old_lin[i] = i;
std::vector<DepGraphIndex> old_lin(ntx);
for (DepGraphIndex i = 0; i < ntx; ++i) old_lin[i] = i;
bench.run([&] {
Linearize(depgraph, /*max_iterations=*/0, rng_seed++, old_lin);
});
@ -167,41 +167,41 @@ void BenchLinearizeNoItersWorstCaseAnc(ClusterIndex ntx, benchmark::Bench& bench
* AncestorCandidateFinder is cheap.
*/
template<typename SetType>
void BenchLinearizeNoItersWorstCaseLIMO(ClusterIndex ntx, benchmark::Bench& bench)
void BenchLinearizeNoItersWorstCaseLIMO(DepGraphIndex ntx, benchmark::Bench& bench)
{
const auto depgraph = MakeWideGraph<SetType>(ntx);
uint64_t rng_seed = 0;
std::vector<ClusterIndex> old_lin(ntx);
for (ClusterIndex i = 0; i < ntx; ++i) old_lin[i] = i;
std::vector<DepGraphIndex> old_lin(ntx);
for (DepGraphIndex i = 0; i < ntx; ++i) old_lin[i] = i;
bench.run([&] {
Linearize(depgraph, /*max_iterations=*/0, rng_seed++, old_lin);
});
}
template<typename SetType>
void BenchPostLinearizeWorstCase(ClusterIndex ntx, benchmark::Bench& bench)
void BenchPostLinearizeWorstCase(DepGraphIndex ntx, benchmark::Bench& bench)
{
DepGraph<SetType> depgraph = MakeWideGraph<SetType>(ntx);
std::vector<ClusterIndex> lin(ntx);
std::vector<DepGraphIndex> lin(ntx);
bench.run([&] {
for (ClusterIndex i = 0; i < ntx; ++i) lin[i] = i;
for (DepGraphIndex i = 0; i < ntx; ++i) lin[i] = i;
PostLinearize(depgraph, lin);
});
}
template<typename SetType>
void BenchMergeLinearizationsWorstCase(ClusterIndex ntx, benchmark::Bench& bench)
void BenchMergeLinearizationsWorstCase(DepGraphIndex ntx, benchmark::Bench& bench)
{
DepGraph<SetType> depgraph;
for (ClusterIndex i = 0; i < ntx; ++i) {
for (DepGraphIndex i = 0; i < ntx; ++i) {
depgraph.AddTransaction({i, 1});
if (i) depgraph.AddDependencies(SetType::Singleton(0), i);
}
std::vector<ClusterIndex> lin1;
std::vector<ClusterIndex> lin2;
std::vector<DepGraphIndex> lin1;
std::vector<DepGraphIndex> lin2;
lin1.push_back(0);
lin2.push_back(0);
for (ClusterIndex i = 1; i < ntx; ++i) {
for (DepGraphIndex i = 1; i < ntx; ++i) {
lin1.push_back(i);
lin2.push_back(ntx - i);
}
@ -214,7 +214,7 @@ template<size_t N>
void BenchLinearizeOptimally(benchmark::Bench& bench, const std::array<uint8_t, N>& serialized)
{
// Determine how many transactions the serialized cluster has.
ClusterIndex num_tx{0};
DepGraphIndex num_tx{0};
{
SpanReader reader{serialized};
DepGraph<BitSet<128>> depgraph;

167
src/cluster_linearize.h
View File

@ -19,8 +19,8 @@
namespace cluster_linearize {
/** Data type to represent transaction indices in clusters. */
using ClusterIndex = uint32_t;
/** Data type to represent transaction indices in DepGraphs and the clusters they represent. */
using DepGraphIndex = uint32_t;
/** Data structure that holds a transaction graph's preprocessed data (fee, size, ancestors,
* descendants). */
@ -86,11 +86,11 @@ public:
*
* Complexity: O(N^2) where N=depgraph.TxCount().
*/
DepGraph(const DepGraph<SetType>& depgraph, Span<const ClusterIndex> mapping, ClusterIndex pos_range) noexcept : entries(pos_range)
DepGraph(const DepGraph<SetType>& depgraph, Span<const DepGraphIndex> mapping, DepGraphIndex pos_range) noexcept : entries(pos_range)
{
Assume(mapping.size() == depgraph.PositionRange());
Assume((pos_range == 0) == (depgraph.TxCount() == 0));
for (ClusterIndex i : depgraph.Positions()) {
for (DepGraphIndex i : depgraph.Positions()) {
auto new_idx = mapping[i];
Assume(new_idx < pos_range);
// Add transaction.
@ -100,7 +100,7 @@ public:
// Fill in fee and size.
entries[new_idx].feerate = depgraph.entries[i].feerate;
}
for (ClusterIndex i : depgraph.Positions()) {
for (DepGraphIndex i : depgraph.Positions()) {
// Fill in dependencies by mapping direct parents.
SetType parents;
for (auto j : depgraph.GetReducedParents(i)) parents.Set(mapping[j]);
@ -113,29 +113,29 @@ public:
/** Get the set of transactions positions in use. Complexity: O(1). */
const SetType& Positions() const noexcept { return m_used; }
/** Get the range of positions in this DepGraph. All entries in Positions() are in [0, PositionRange() - 1]. */
ClusterIndex PositionRange() const noexcept { return entries.size(); }
DepGraphIndex PositionRange() const noexcept { return entries.size(); }
/** Get the number of transactions in the graph. Complexity: O(1). */
auto TxCount() const noexcept { return m_used.Count(); }
/** Get the feerate of a given transaction i. Complexity: O(1). */
const FeeFrac& FeeRate(ClusterIndex i) const noexcept { return entries[i].feerate; }
const FeeFrac& FeeRate(DepGraphIndex i) const noexcept { return entries[i].feerate; }
/** Get the mutable feerate of a given transaction i. Complexity: O(1). */
FeeFrac& FeeRate(ClusterIndex i) noexcept { return entries[i].feerate; }
FeeFrac& FeeRate(DepGraphIndex i) noexcept { return entries[i].feerate; }
/** Get the ancestors of a given transaction i. Complexity: O(1). */
const SetType& Ancestors(ClusterIndex i) const noexcept { return entries[i].ancestors; }
const SetType& Ancestors(DepGraphIndex i) const noexcept { return entries[i].ancestors; }
/** Get the descendants of a given transaction i. Complexity: O(1). */
const SetType& Descendants(ClusterIndex i) const noexcept { return entries[i].descendants; }
const SetType& Descendants(DepGraphIndex i) const noexcept { return entries[i].descendants; }
/** Add a new unconnected transaction to this transaction graph (in the first available
* position), and return its ClusterIndex.
* position), and return its DepGraphIndex.
*
* Complexity: O(1) (amortized, due to resizing of backing vector).
*/
ClusterIndex AddTransaction(const FeeFrac& feefrac) noexcept
DepGraphIndex AddTransaction(const FeeFrac& feefrac) noexcept
{
static constexpr auto ALL_POSITIONS = SetType::Fill(SetType::Size());
auto available = ALL_POSITIONS - m_used;
Assume(available.Any());
ClusterIndex new_idx = available.First();
DepGraphIndex new_idx = available.First();
if (new_idx == entries.size()) {
entries.emplace_back(feefrac, SetType::Singleton(new_idx), SetType::Singleton(new_idx));
} else {
@ -174,7 +174,7 @@ public:
*
* Complexity: O(N) where N=TxCount().
*/
void AddDependencies(const SetType& parents, ClusterIndex child) noexcept
void AddDependencies(const SetType& parents, DepGraphIndex child) noexcept
{
Assume(m_used[child]);
Assume(parents.IsSubsetOf(m_used));
@ -205,7 +205,7 @@ public:
*
* Complexity: O(N) where N=Ancestors(i).Count() (which is bounded by TxCount()).
*/
SetType GetReducedParents(ClusterIndex i) const noexcept
SetType GetReducedParents(DepGraphIndex i) const noexcept
{
SetType parents = Ancestors(i);
parents.Reset(i);
@ -226,7 +226,7 @@ public:
*
* Complexity: O(N) where N=Descendants(i).Count() (which is bounded by TxCount()).
*/
SetType GetReducedChildren(ClusterIndex i) const noexcept
SetType GetReducedChildren(DepGraphIndex i) const noexcept
{
SetType children = Descendants(i);
children.Reset(i);
@ -298,17 +298,28 @@ public:
*
* Complexity: O(select.Count() * log(select.Count())).
*/
void AppendTopo(std::vector<ClusterIndex>& list, const SetType& select) const noexcept
void AppendTopo(std::vector<DepGraphIndex>& list, const SetType& select) const noexcept
{
ClusterIndex old_len = list.size();
DepGraphIndex old_len = list.size();
for (auto i : select) list.push_back(i);
std::sort(list.begin() + old_len, list.end(), [&](ClusterIndex a, ClusterIndex b) noexcept {
std::sort(list.begin() + old_len, list.end(), [&](DepGraphIndex a, DepGraphIndex b) noexcept {
const auto a_anc_count = entries[a].ancestors.Count();
const auto b_anc_count = entries[b].ancestors.Count();
if (a_anc_count != b_anc_count) return a_anc_count < b_anc_count;
return a < b;
});
}
/** Check if this graph is acyclic. */
bool IsAcyclic() const noexcept
{
for (auto i : Positions()) {
if ((Ancestors(i) & Descendants(i)) != SetType::Singleton(i)) {
return false;
}
}
return true;
}
};
/** A set of transactions together with their aggregate feerate. */
@ -327,7 +338,7 @@ struct SetInfo
SetInfo(const SetType& txn, const FeeFrac& fr) noexcept : transactions(txn), feerate(fr) {}
/** Construct a SetInfo for a given transaction in a depgraph. */
explicit SetInfo(const DepGraph<SetType>& depgraph, ClusterIndex pos) noexcept :
explicit SetInfo(const DepGraph<SetType>& depgraph, DepGraphIndex pos) noexcept :
transactions(SetType::Singleton(pos)), feerate(depgraph.FeeRate(pos)) {}
/** Construct a SetInfo for a set of transactions in a depgraph. */
@ -335,7 +346,7 @@ struct SetInfo
transactions(txn), feerate(depgraph.FeeRate(txn)) {}
/** Add a transaction to this SetInfo (which must not yet be in it). */
void Set(const DepGraph<SetType>& depgraph, ClusterIndex pos) noexcept
void Set(const DepGraph<SetType>& depgraph, DepGraphIndex pos) noexcept
{
Assume(!transactions[pos]);
transactions.Set(pos);
@ -371,10 +382,10 @@ struct SetInfo
/** Compute the feerates of the chunks of linearization. */
template<typename SetType>
std::vector<FeeFrac> ChunkLinearization(const DepGraph<SetType>& depgraph, Span<const ClusterIndex> linearization) noexcept
std::vector<FeeFrac> ChunkLinearization(const DepGraph<SetType>& depgraph, Span<const DepGraphIndex> linearization) noexcept
{
std::vector<FeeFrac> ret;
for (ClusterIndex i : linearization) {
for (DepGraphIndex i : linearization) {
/** The new chunk to be added, initially a singleton. */
auto new_chunk = depgraph.FeeRate(i);
// As long as the new chunk has a higher feerate than the last chunk so far, absorb it.
@ -396,13 +407,13 @@ class LinearizationChunking
const DepGraph<SetType>& m_depgraph;
/** The linearization we started from, possibly with removed prefix stripped. */
Span<const ClusterIndex> m_linearization;
Span<const DepGraphIndex> m_linearization;
/** Chunk sets and their feerates, of what remains of the linearization. */
std::vector<SetInfo<SetType>> m_chunks;
/** How large a prefix of m_chunks corresponds to removed transactions. */
ClusterIndex m_chunks_skip{0};
DepGraphIndex m_chunks_skip{0};
/** Which transactions remain in the linearization. */
SetType m_todo;
@ -437,7 +448,7 @@ class LinearizationChunking
public:
/** Initialize a LinearizationSubset object for a given length of linearization. */
explicit LinearizationChunking(const DepGraph<SetType>& depgraph LIFETIMEBOUND, Span<const ClusterIndex> lin LIFETIMEBOUND) noexcept :
explicit LinearizationChunking(const DepGraph<SetType>& depgraph LIFETIMEBOUND, Span<const DepGraphIndex> lin LIFETIMEBOUND) noexcept :
m_depgraph(depgraph), m_linearization(lin)
{
// Mark everything in lin as todo still.
@ -448,10 +459,10 @@ public:
}
/** Determine how many chunks remain in the linearization. */
ClusterIndex NumChunksLeft() const noexcept { return m_chunks.size() - m_chunks_skip; }
DepGraphIndex NumChunksLeft() const noexcept { return m_chunks.size() - m_chunks_skip; }
/** Access a chunk. Chunk 0 is the highest-feerate prefix of what remains. */
const SetInfo<SetType>& GetChunk(ClusterIndex n) const noexcept
const SetInfo<SetType>& GetChunk(DepGraphIndex n) const noexcept
{
Assume(n + m_chunks_skip < m_chunks.size());
return m_chunks[n + m_chunks_skip];
@ -494,7 +505,7 @@ public:
Assume(subset.transactions.IsSubsetOf(m_todo));
SetInfo<SetType> accumulator;
// Iterate over all chunks of the remaining linearization.
for (ClusterIndex i = 0; i < NumChunksLeft(); ++i) {
for (DepGraphIndex i = 0; i < NumChunksLeft(); ++i) {
// Find what (if any) intersection the chunk has with subset.
const SetType to_add = GetChunk(i).transactions & subset.transactions;
if (to_add.Any()) {
@ -546,13 +557,13 @@ public:
m_ancestor_set_feerates(depgraph.PositionRange())
{
// Precompute ancestor-set feerates.
for (ClusterIndex i : m_depgraph.Positions()) {
for (DepGraphIndex i : m_depgraph.Positions()) {
/** The remaining ancestors for transaction i. */
SetType anc_to_add = m_depgraph.Ancestors(i);
FeeFrac anc_feerate;
// Reuse accumulated feerate from first ancestor, if usable.
Assume(anc_to_add.Any());
ClusterIndex first = anc_to_add.First();
DepGraphIndex first = anc_to_add.First();
if (first < i) {
anc_feerate = m_ancestor_set_feerates[first];
Assume(!anc_feerate.IsEmpty());
@ -592,7 +603,7 @@ public:
}
/** Count the number of remaining unlinearized transactions. */
ClusterIndex NumRemaining() const noexcept
DepGraphIndex NumRemaining() const noexcept
{
return m_todo.Count();
}
@ -605,7 +616,7 @@ public:
SetInfo<SetType> FindCandidateSet() const noexcept
{
Assume(!AllDone());
std::optional<ClusterIndex> best;
std::optional<DepGraphIndex> best;
for (auto i : m_todo) {
if (best.has_value()) {
Assume(!m_ancestor_set_feerates[i].IsEmpty());
@ -633,9 +644,9 @@ class SearchCandidateFinder
/** Internal RNG. */
InsecureRandomContext m_rng;
/** m_sorted_to_original[i] is the original position that sorted transaction position i had. */
std::vector<ClusterIndex> m_sorted_to_original;
std::vector<DepGraphIndex> m_sorted_to_original;
/** m_original_to_sorted[i] is the sorted position original transaction position i has. */
std::vector<ClusterIndex> m_original_to_sorted;
std::vector<DepGraphIndex> m_original_to_sorted;
/** Internal dependency graph for the cluster (with transactions in decreasing individual
* feerate order). */
DepGraph<SetType> m_sorted_depgraph;
@ -673,7 +684,7 @@ public:
{
// Determine reordering mapping, by sorting by decreasing feerate. Unused positions are
// not included, as they will never be looked up anyway.
ClusterIndex sorted_pos{0};
DepGraphIndex sorted_pos{0};
for (auto i : depgraph.Positions()) {
m_sorted_to_original[sorted_pos++] = i;
}
@ -683,7 +694,7 @@ public:
return feerate_cmp > 0;
});
// Compute reverse mapping.
for (ClusterIndex i = 0; i < m_sorted_to_original.size(); ++i) {
for (DepGraphIndex i = 0; i < m_sorted_to_original.size(); ++i) {
m_original_to_sorted[m_sorted_to_original[i]] = i;
}
// Compute reordered dependency graph.
@ -782,7 +793,7 @@ public:
/** The set of transactions in m_todo which have feerate > best's. */
SetType imp = m_todo;
while (imp.Any()) {
ClusterIndex check = imp.Last();
DepGraphIndex check = imp.Last();
if (m_sorted_depgraph.FeeRate(check) >> best.feerate) break;
imp.Reset(check);
}
@ -839,7 +850,7 @@ public:
best = inc;
// See if we can remove any entries from imp now.
while (imp.Any()) {
ClusterIndex check = imp.Last();
DepGraphIndex check = imp.Last();
if (m_sorted_depgraph.FeeRate(check) >> best.feerate) break;
imp.Reset(check);
}
@ -880,7 +891,7 @@ public:
// If pot is empty, then so is inc.
Assume(elem.inc.feerate.IsEmpty() == elem.pot_feerate.IsEmpty());
const ClusterIndex first = elem.und.First();
const DepGraphIndex first = elem.und.First();
if (!elem.inc.feerate.IsEmpty()) {
// If no undecided transactions remain with feerate higher than best, this entry
// cannot be improved beyond best.
@ -906,17 +917,17 @@ public:
// most. Let I(t) be the size of the undecided set after including t, and E(t) the size
// of the undecided set after excluding t. Then choose the split transaction t such
// that 2^I(t) + 2^E(t) is minimal, tie-breaking by highest individual feerate for t.
ClusterIndex split = 0;
DepGraphIndex split = 0;
const auto select = elem.und & m_sorted_depgraph.Ancestors(first);
Assume(select.Any());
std::optional<std::pair<ClusterIndex, ClusterIndex>> split_counts;
std::optional<std::pair<DepGraphIndex, DepGraphIndex>> split_counts;
for (auto t : select) {
// Call max = max(I(t), E(t)) and min = min(I(t), E(t)). Let counts = {max,min}.
// Sorting by the tuple counts is equivalent to sorting by 2^I(t) + 2^E(t). This
// expression is equal to 2^max + 2^min = 2^max * (1 + 1/2^(max - min)). The second
// factor (1 + 1/2^(max - min)) there is in (1,2]. Thus increasing max will always
// increase it, even when min decreases. Because of this, we can first sort by max.
std::pair<ClusterIndex, ClusterIndex> counts{
std::pair<DepGraphIndex, DepGraphIndex> counts{
(elem.und - m_sorted_depgraph.Ancestors(t)).Count(),
(elem.und - m_sorted_depgraph.Descendants(t)).Count()};
if (counts.first < counts.second) std::swap(counts.first, counts.second);
@ -1016,13 +1027,13 @@ public:
* Complexity: possibly O(N * min(max_iterations + N, sqrt(2^N))) where N=depgraph.TxCount().
*/
template<typename SetType>
std::pair<std::vector<ClusterIndex>, bool> Linearize(const DepGraph<SetType>& depgraph, uint64_t max_iterations, uint64_t rng_seed, Span<const ClusterIndex> old_linearization = {}) noexcept
std::pair<std::vector<DepGraphIndex>, bool> Linearize(const DepGraph<SetType>& depgraph, uint64_t max_iterations, uint64_t rng_seed, Span<const DepGraphIndex> old_linearization = {}) noexcept
{
Assume(old_linearization.empty() || old_linearization.size() == depgraph.TxCount());
if (depgraph.TxCount() == 0) return {{}, true};
uint64_t iterations_left = max_iterations;
std::vector<ClusterIndex> linearization;
std::vector<DepGraphIndex> linearization;
AncestorCandidateFinder anc_finder(depgraph);
std::optional<SearchCandidateFinder<SetType>> src_finder;
@ -1110,7 +1121,7 @@ std::pair<std::vector<ClusterIndex>, bool> Linearize(const DepGraph<SetType>& de
* postlinearize" process.
*/
template<typename SetType>
void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> linearization)
void PostLinearize(const DepGraph<SetType>& depgraph, Span<DepGraphIndex> linearization)
{
// This algorithm performs a number of passes (currently 2); the even ones operate from back to
// front, the odd ones from front to back. Each results in an equal-or-better linearization
@ -1148,9 +1159,9 @@ void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> lineari
// entries[0].
/** Index of the sentinel in the entries array below. */
static constexpr ClusterIndex SENTINEL{0};
static constexpr DepGraphIndex SENTINEL{0};
/** Indicator that a group has no previous transaction. */
static constexpr ClusterIndex NO_PREV_TX{0};
static constexpr DepGraphIndex NO_PREV_TX{0};
/** Data structure per transaction entry. */
@ -1158,16 +1169,16 @@ void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> lineari
{
/** The index of the previous transaction in this group; NO_PREV_TX if this is the first
* entry of a group. */
ClusterIndex prev_tx;
DepGraphIndex prev_tx;
// The fields below are only used for transactions that are the last one in a group
// (referred to as tail transactions below).
/** Index of the first transaction in this group, possibly itself. */
ClusterIndex first_tx;
DepGraphIndex first_tx;
/** Index of the last transaction in the previous group. The first group (the sentinel)
* points back to the last group here, making it a singly-linked circular list. */
ClusterIndex prev_group;
DepGraphIndex prev_group;
/** All transactions in the group. Empty for the sentinel. */
SetType group;
/** All dependencies of the group (descendants in even passes; ancestors in odd ones). */
@ -1210,12 +1221,12 @@ void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> lineari
Assume(entries[SENTINEL].feerate.IsEmpty());
// Iterate over all elements in the existing linearization.
for (ClusterIndex i = 0; i < linearization.size(); ++i) {
for (DepGraphIndex i = 0; i < linearization.size(); ++i) {
// Even passes are from back to front; odd passes from front to back.
ClusterIndex idx = linearization[rev ? linearization.size() - 1 - i : i];
DepGraphIndex idx = linearization[rev ? linearization.size() - 1 - i : i];
// Construct a new group containing just idx. In even passes, the meaning of
// parent/child and high/low feerate are swapped.
ClusterIndex cur_group = idx + 1;
DepGraphIndex cur_group = idx + 1;
entries[cur_group].group = SetType::Singleton(idx);
entries[cur_group].deps = rev ? depgraph.Descendants(idx): depgraph.Ancestors(idx);
entries[cur_group].feerate = depgraph.FeeRate(idx);
@ -1227,8 +1238,8 @@ void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> lineari
entries[SENTINEL].prev_group = cur_group;
// Start merge/swap cycle.
ClusterIndex next_group = SENTINEL; // We inserted at the end, so next group is sentinel.
ClusterIndex prev_group = entries[cur_group].prev_group;
DepGraphIndex next_group = SENTINEL; // We inserted at the end, so next group is sentinel.
DepGraphIndex prev_group = entries[cur_group].prev_group;
// Continue as long as the current group has higher feerate than the previous one.
while (entries[cur_group].feerate >> entries[prev_group].feerate) {
// prev_group/cur_group/next_group refer to (the last transactions of) 3
@ -1256,7 +1267,7 @@ void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> lineari
entries[cur_group].prev_group = prev_group;
} else {
// There is no dependency between cur_group and prev_group; swap them.
ClusterIndex preprev_group = entries[prev_group].prev_group;
DepGraphIndex preprev_group = entries[prev_group].prev_group;
// If PP, P, C, N were the old preprev, prev, cur, next groups, then the new
// layout becomes [PP, C, P, N]. Update prev_groups to reflect that order.
entries[next_group].prev_group = prev_group;
@ -1271,10 +1282,10 @@ void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> lineari
}
// Convert the entries back to linearization (overwriting the existing one).
ClusterIndex cur_group = entries[0].prev_group;
ClusterIndex done = 0;
DepGraphIndex cur_group = entries[0].prev_group;
DepGraphIndex done = 0;
while (cur_group != SENTINEL) {
ClusterIndex cur_tx = cur_group;
DepGraphIndex cur_tx = cur_group;
// Traverse the transactions of cur_group (from back to front), and write them in the
// same order during odd passes, and reversed (front to back) in even passes.
if (rev) {
@ -1299,7 +1310,7 @@ void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> lineari
* Complexity: O(N^2) where N=depgraph.TxCount(); O(N) if both inputs are identical.
*/
template<typename SetType>
std::vector<ClusterIndex> MergeLinearizations(const DepGraph<SetType>& depgraph, Span<const ClusterIndex> lin1, Span<const ClusterIndex> lin2)
std::vector<DepGraphIndex> MergeLinearizations(const DepGraph<SetType>& depgraph, Span<const DepGraphIndex> lin1, Span<const DepGraphIndex> lin2)
{
Assume(lin1.size() == depgraph.TxCount());
Assume(lin2.size() == depgraph.TxCount());
@ -1307,7 +1318,7 @@ std::vector<ClusterIndex> MergeLinearizations(const DepGraph<SetType>& depgraph,
/** Chunkings of what remains of both input linearizations. */
LinearizationChunking chunking1(depgraph, lin1), chunking2(depgraph, lin2);
/** Output linearization. */
std::vector<ClusterIndex> ret;
std::vector<DepGraphIndex> ret;
if (depgraph.TxCount() == 0) return ret;
ret.reserve(depgraph.TxCount());
@ -1336,6 +1347,38 @@ std::vector<ClusterIndex> MergeLinearizations(const DepGraph<SetType>& depgraph,
return ret;
}
/** Make linearization topological, retaining its ordering where possible. */
template<typename SetType>
void FixLinearization(const DepGraph<SetType>& depgraph, Span<DepGraphIndex> linearization) noexcept
{
// This algorithm can be summarized as moving every element in the linearization backwards
// until it is placed after all its ancestors.
SetType done;
const auto len = linearization.size();
// Iterate over the elements of linearization from back to front (i is distance from back).
for (DepGraphIndex i = 0; i < len; ++i) {
/** The element at that position. */
DepGraphIndex elem = linearization[len - 1 - i];
/** j represents how far from the back of the linearization elem should be placed. */
DepGraphIndex j = i;
// Figure out which elements need to be moved before elem.
SetType place_before = done & depgraph.Ancestors(elem);
// Find which position to place elem in (updating j), continuously moving the elements
// in between forward.
while (place_before.Any()) {
// j cannot be 0 here; if it was, then there was necessarily nothing earlier which
// elem needs to be place before anymore, and place_before would be empty.
Assume(j > 0);
auto to_swap = linearization[len - 1 - (j - 1)];
place_before.Reset(to_swap);
linearization[len - 1 - (j--)] = to_swap;
}
// Put elem in its final position and mark it as done.
linearization[len - 1 - j] = elem;
done.Set(elem);
}
}
} // namespace cluster_linearize
#endif // BITCOIN_CLUSTER_LINEARIZE_H

4
src/test/cluster_linearize_tests.cpp
View File

@ -28,11 +28,11 @@ void TestDepGraphSerialization(const std::vector<std::pair<FeeFrac, SetType>>& c
// Construct DepGraph from cluster argument.
DepGraph<SetType> depgraph;
SetType holes;
for (ClusterIndex i = 0; i < cluster.size(); ++i) {
for (DepGraphIndex i = 0; i < cluster.size(); ++i) {
depgraph.AddTransaction(cluster[i].first);
if (cluster[i] == HOLE) holes.Set(i);
}
for (ClusterIndex i = 0; i < cluster.size(); ++i) {
for (DepGraphIndex i = 0; i < cluster.size(); ++i) {
depgraph.AddDependencies(cluster[i].second, i);
}
depgraph.RemoveTransactions(holes);

1
src/test/fuzz/CMakeLists.txt
View File

@ -124,6 +124,7 @@ add_executable(fuzz
tx_in.cpp
tx_out.cpp
tx_pool.cpp
txgraph.cpp
txorphan.cpp
txrequest.cpp
# Visual Studio 2022 version 17.12 introduced a bug

157
src/test/fuzz/cluster_linearize.cpp
View File

@ -149,9 +149,9 @@ public:
* than AncestorCandidateFinder and SearchCandidateFinder.
*/
template<typename SetType>
std::pair<std::vector<ClusterIndex>, bool> SimpleLinearize(const DepGraph<SetType>& depgraph, uint64_t max_iterations)
std::pair<std::vector<DepGraphIndex>, bool> SimpleLinearize(const DepGraph<SetType>& depgraph, uint64_t max_iterations)
{
std::vector<ClusterIndex> linearization;
std::vector<DepGraphIndex> linearization;
SimpleCandidateFinder finder(depgraph);
SetType todo = depgraph.Positions();
bool optimal = true;
@ -203,9 +203,9 @@ SetType ReadTopologicalSet(const DepGraph<SetType>& depgraph, const SetType& tod
/** Given a dependency graph, construct any valid linearization for it, reading from a SpanReader. */
template<typename BS>
std::vector<ClusterIndex> ReadLinearization(const DepGraph<BS>& depgraph, SpanReader& reader)
std::vector<DepGraphIndex> ReadLinearization(const DepGraph<BS>& depgraph, SpanReader& reader)
{
std::vector<ClusterIndex> linearization;
std::vector<DepGraphIndex> linearization;
TestBitSet todo = depgraph.Positions();
// In every iteration one topologically-valid transaction is appended to linearization.
while (todo.Any()) {
@ -253,18 +253,18 @@ FUZZ_TARGET(clusterlin_depgraph_sim)
* sim[i]->first is its individual feerate, and sim[i]->second is its set of ancestors. */
std::array<std::optional<std::pair<FeeFrac, TestBitSet>>, TestBitSet::Size()> sim;
/** The number of non-nullopt position in sim. */
ClusterIndex num_tx_sim{0};
DepGraphIndex num_tx_sim{0};
/** Read a valid index of a transaction from the provider. */
auto idx_fn = [&]() {
auto offset = provider.ConsumeIntegralInRange<ClusterIndex>(0, num_tx_sim - 1);
for (ClusterIndex i = 0; i < sim.size(); ++i) {
auto offset = provider.ConsumeIntegralInRange<DepGraphIndex>(0, num_tx_sim - 1);
for (DepGraphIndex i = 0; i < sim.size(); ++i) {
if (!sim[i].has_value()) continue;
if (offset == 0) return i;
--offset;
}
assert(false);
return ClusterIndex(-1);
return DepGraphIndex(-1);
};
/** Read a valid subset of the transactions from the provider. */
@ -273,7 +273,7 @@ FUZZ_TARGET(clusterlin_depgraph_sim)
const auto mask = provider.ConsumeIntegralInRange<uint64_t>(0, range);
auto mask_shifted = mask;
TestBitSet subset;
for (ClusterIndex i = 0; i < sim.size(); ++i) {
for (DepGraphIndex i = 0; i < sim.size(); ++i) {
if (!sim[i].has_value()) continue;
if (mask_shifted & 1) {
subset.Set(i);
@ -289,7 +289,7 @@ FUZZ_TARGET(clusterlin_depgraph_sim)
auto range = (uint64_t{1} << sim.size()) - 1;
const auto mask = provider.ConsumeIntegralInRange<uint64_t>(0, range);
TestBitSet set;
for (ClusterIndex i = 0; i < sim.size(); ++i) {
for (DepGraphIndex i = 0; i < sim.size(); ++i) {
if ((mask >> i) & 1) {
set.Set(i);
}
@ -301,7 +301,7 @@ FUZZ_TARGET(clusterlin_depgraph_sim)
auto anc_update_fn = [&]() {
while (true) {
bool updates{false};
for (ClusterIndex chl = 0; chl < sim.size(); ++chl) {
for (DepGraphIndex chl = 0; chl < sim.size(); ++chl) {
if (!sim[chl].has_value()) continue;
for (auto par : sim[chl]->second) {
if (!sim[chl]->second.IsSupersetOf(sim[par]->second)) {
@ -315,7 +315,7 @@ FUZZ_TARGET(clusterlin_depgraph_sim)
};
/** Compare the state of transaction i in the simulation with the real one. */
auto check_fn = [&](ClusterIndex i) {
auto check_fn = [&](DepGraphIndex i) {
// Compare used positions.
assert(real.Positions()[i] == sim[i].has_value());
if (sim[i].has_value()) {
@ -338,7 +338,7 @@ FUZZ_TARGET(clusterlin_depgraph_sim)
auto idx = real.AddTransaction(feerate);
// Verify that the returned index is correct.
assert(!sim[idx].has_value());
for (ClusterIndex i = 0; i < TestBitSet::Size(); ++i) {
for (DepGraphIndex i = 0; i < TestBitSet::Size(); ++i) {
if (!sim[i].has_value()) {
assert(idx == i);
break;
@ -351,7 +351,7 @@ FUZZ_TARGET(clusterlin_depgraph_sim)
}
if ((command % 3) <= 1 && num_tx_sim > 0) {
// AddDependencies.
ClusterIndex child = idx_fn();
DepGraphIndex child = idx_fn();
auto parents = subset_fn();
// Apply to DepGraph.
real.AddDependencies(parents, child);
@ -370,7 +370,7 @@ FUZZ_TARGET(clusterlin_depgraph_sim)
// Apply to DepGraph.
real.RemoveTransactions(del);
// Apply to sim.
for (ClusterIndex i = 0; i < sim.size(); ++i) {
for (DepGraphIndex i = 0; i < sim.size(); ++i) {
if (sim[i].has_value()) {
if (del[i]) {
--num_tx_sim;
@ -388,7 +388,7 @@ FUZZ_TARGET(clusterlin_depgraph_sim)
// Compare the real obtained depgraph against the simulation.
anc_update_fn();
for (ClusterIndex i = 0; i < sim.size(); ++i) check_fn(i);
for (DepGraphIndex i = 0; i < sim.size(); ++i) check_fn(i);
assert(real.TxCount() == num_tx_sim);
// Sanity check the result (which includes round-tripping serialization, if applicable).
SanityCheck(real);
@ -401,13 +401,42 @@ FUZZ_TARGET(clusterlin_depgraph_serialization)
// Construct a graph by deserializing.
SpanReader reader(buffer);
DepGraph<TestBitSet> depgraph;
DepGraphIndex par_code{0}, chl_code{0};
try {
reader >> Using<DepGraphFormatter>(depgraph);
reader >> Using<DepGraphFormatter>(depgraph) >> VARINT(par_code) >> VARINT(chl_code);
} catch (const std::ios_base::failure&) {}
SanityCheck(depgraph);
// Verify the graph is a DAG.
assert(IsAcyclic(depgraph));
assert(depgraph.IsAcyclic());
// Introduce a cycle, and then test that IsAcyclic returns false.
if (depgraph.TxCount() < 2) return;
DepGraphIndex par(0), chl(0);
// Pick any transaction of depgraph as parent.
par_code %= depgraph.TxCount();
for (auto i : depgraph.Positions()) {
if (par_code == 0) {
par = i;
break;
}
--par_code;
}
// Pick any ancestor of par (excluding itself) as child, if any.
auto ancestors = depgraph.Ancestors(par) - TestBitSet::Singleton(par);
if (ancestors.None()) return;
chl_code %= ancestors.Count();
for (auto i : ancestors) {
if (chl_code == 0) {
chl = i;
break;
}
--chl_code;
}
// Add the cycle-introducing dependency.
depgraph.AddDependencies(TestBitSet::Singleton(par), chl);
// Check that we now detect a cycle.
assert(!depgraph.IsAcyclic());
}
FUZZ_TARGET(clusterlin_components)
@ -469,7 +498,7 @@ FUZZ_TARGET(clusterlin_components)
reader >> VARINT(subset_bits);
} catch (const std::ios_base::failure&) {}
TestBitSet subset;
for (ClusterIndex i : depgraph.Positions()) {
for (DepGraphIndex i : depgraph.Positions()) {
if (todo[i]) {
if (subset_bits & 1) subset.Set(i);
subset_bits >>= 1;
@ -526,7 +555,7 @@ FUZZ_TARGET(clusterlin_chunking)
for (const auto& chunk_feerate : chunking) {
assert(todo.Any());
SetInfo<TestBitSet> accumulator, best;
for (ClusterIndex idx : linearization) {
for (DepGraphIndex idx : linearization) {
if (todo[idx]) {
accumulator.Set(depgraph, idx);
if (best.feerate.IsEmpty() || accumulator.feerate >> best.feerate) {
@ -737,7 +766,7 @@ FUZZ_TARGET(clusterlin_linearization_chunking)
assert(chunking.NumChunksLeft() > 0);
// Construct linearization with just todo.
std::vector<ClusterIndex> linearization_left;
std::vector<DepGraphIndex> linearization_left;
for (auto i : linearization) {
if (todo[i]) linearization_left.push_back(i);
}
@ -747,13 +776,13 @@ FUZZ_TARGET(clusterlin_linearization_chunking)
// Verify that it matches the feerates of the chunks of chunking.
assert(chunking.NumChunksLeft() == chunking_left.size());
for (ClusterIndex i = 0; i < chunking.NumChunksLeft(); ++i) {
for (DepGraphIndex i = 0; i < chunking.NumChunksLeft(); ++i) {
assert(chunking.GetChunk(i).feerate == chunking_left[i]);
}
// Check consistency of chunking.
TestBitSet combined;
for (ClusterIndex i = 0; i < chunking.NumChunksLeft(); ++i) {
for (DepGraphIndex i = 0; i < chunking.NumChunksLeft(); ++i) {
const auto& chunk_info = chunking.GetChunk(i);
// Chunks must be non-empty.
assert(chunk_info.transactions.Any());
@ -804,7 +833,7 @@ FUZZ_TARGET(clusterlin_linearization_chunking)
// - No non-empty intersection between the intersection and a prefix of the chunks of the
// remainder of the linearization may be better than the intersection.
TestBitSet prefix;
for (ClusterIndex i = 0; i < chunking.NumChunksLeft(); ++i) {
for (DepGraphIndex i = 0; i < chunking.NumChunksLeft(); ++i) {
prefix |= chunking.GetChunk(i).transactions;
auto reintersect = SetInfo(depgraph, prefix & intersect.transactions);
if (!reintersect.feerate.IsEmpty()) {
@ -846,7 +875,7 @@ FUZZ_TARGET(clusterlin_linearize)
if (make_connected) MakeConnected(depgraph);
// Optionally construct an old linearization for it.
std::vector<ClusterIndex> old_linearization;
std::vector<DepGraphIndex> old_linearization;
{
uint8_t have_old_linearization{0};
try {
@ -905,8 +934,8 @@ FUZZ_TARGET(clusterlin_linearize)
// Only for very small clusters, test every topologically-valid permutation.
if (depgraph.TxCount() <= 7) {
std::vector<ClusterIndex> perm_linearization;
for (ClusterIndex i : depgraph.Positions()) perm_linearization.push_back(i);
std::vector<DepGraphIndex> perm_linearization;
for (DepGraphIndex i : depgraph.Positions()) perm_linearization.push_back(i);
// Iterate over all valid permutations.
do {
// Determine whether perm_linearization is topological.
@ -942,7 +971,7 @@ FUZZ_TARGET(clusterlin_postlinearize)
} catch (const std::ios_base::failure&) {}
// Retrieve a linearization from the fuzz input.
std::vector<ClusterIndex> linearization;
std::vector<DepGraphIndex> linearization;
linearization = ReadLinearization(depgraph, reader);
SanityCheck(depgraph, linearization);
@ -990,7 +1019,7 @@ FUZZ_TARGET(clusterlin_postlinearize_tree)
// Now construct a new graph, copying the nodes, but leaving only the first parent (even
// direction) or the first child (odd direction).
DepGraph<TestBitSet> depgraph_tree;
for (ClusterIndex i = 0; i < depgraph_gen.PositionRange(); ++i) {
for (DepGraphIndex i = 0; i < depgraph_gen.PositionRange(); ++i) {
if (depgraph_gen.Positions()[i]) {
depgraph_tree.AddTransaction(depgraph_gen.FeeRate(i));
} else {
@ -1002,14 +1031,14 @@ FUZZ_TARGET(clusterlin_postlinearize_tree)
depgraph_tree.RemoveTransactions(TestBitSet::Fill(depgraph_gen.PositionRange()) - depgraph_gen.Positions());
if (direction & 1) {
for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) {
for (DepGraphIndex i = 0; i < depgraph_gen.TxCount(); ++i) {
auto children = depgraph_gen.GetReducedChildren(i);
if (children.Any()) {
depgraph_tree.AddDependencies(TestBitSet::Singleton(i), children.First());
}
}
} else {
for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) {
for (DepGraphIndex i = 0; i < depgraph_gen.TxCount(); ++i) {
auto parents = depgraph_gen.GetReducedParents(i);
if (parents.Any()) {
depgraph_tree.AddDependencies(TestBitSet::Singleton(parents.First()), i);
@ -1018,7 +1047,7 @@ FUZZ_TARGET(clusterlin_postlinearize_tree)
}
// Retrieve a linearization from the fuzz input.
std::vector<ClusterIndex> linearization;
std::vector<DepGraphIndex> linearization;
linearization = ReadLinearization(depgraph_tree, reader);
SanityCheck(depgraph_tree, linearization);
@ -1075,7 +1104,7 @@ FUZZ_TARGET(clusterlin_postlinearize_moved_leaf)
// Construct a linearization identical to lin, but with the tail end of lin_leaf moved to the
// back.
std::vector<ClusterIndex> lin_moved;
std::vector<DepGraphIndex> lin_moved;
for (auto i : lin) {
if (i != lin_leaf.back()) lin_moved.push_back(i);
}
@ -1118,3 +1147,65 @@ FUZZ_TARGET(clusterlin_merge)
auto cmp2 = CompareChunks(chunking_merged, chunking2);
assert(cmp2 >= 0);
}
FUZZ_TARGET(clusterlin_fix_linearization)
{
// Verify expected properties of FixLinearization() on arbitrary linearizations.
// Retrieve a depgraph from the fuzz input.
SpanReader reader(buffer);
DepGraph<TestBitSet> depgraph;
try {
reader >> Using<DepGraphFormatter>(depgraph);
} catch (const std::ios_base::failure&) {}
// Construct an arbitrary linearization (not necessarily topological for depgraph).
std::vector<DepGraphIndex> linearization;
/** Which transactions of depgraph are yet to be included in linearization. */
TestBitSet todo = depgraph.Positions();
while (todo.Any()) {
// Read a number from the fuzz input in range [0, todo.Count()).
uint64_t val{0};
try {
reader >> VARINT(val);
} catch (const std::ios_base::failure&) {}
val %= todo.Count();
// Find the val'th element in todo, remove it from todo, and append it to linearization.
for (auto idx : todo) {
if (val == 0) {
linearization.push_back(idx);
todo.Reset(idx);
break;
}
--val;
}
}
assert(linearization.size() == depgraph.TxCount());
// Determine what prefix of linearization is topological, i.e., the position of the first entry
// in linearization which corresponds to a transaction that is not preceded by all its
// ancestors.
size_t topo_prefix = 0;
todo = depgraph.Positions();
while (topo_prefix < linearization.size()) {
DepGraphIndex idx = linearization[topo_prefix];
todo.Reset(idx);
if (todo.Overlaps(depgraph.Ancestors(idx))) break;
++topo_prefix;
}
// Then make a fixed copy of linearization.
auto linearization_fixed = linearization;
FixLinearization(depgraph, linearization_fixed);
// Sanity check it (which includes testing whether it is topological).
SanityCheck(depgraph, linearization_fixed);
// FixLinearization does not modify the topological prefix of linearization.
assert(std::equal(linearization.begin(), linearization.begin() + topo_prefix,
linearization_fixed.begin()));
// This also means that if linearization was entirely topological, FixLinearization cannot have
// modified it. This is implied by the assertion above already, but repeat it explicitly.
if (topo_prefix == linearization.size()) {
assert(linearization == linearization_fixed);
}
}

993
src/test/fuzz/txgraph.cpp Normal file
View File

@ -0,0 +1,993 @@
// Copyright (c) The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <txgraph.h>
#include <cluster_linearize.h>
#include <test/fuzz/fuzz.h>
#include <test/fuzz/FuzzedDataProvider.h>
#include <test/util/random.h>
#include <util/bitset.h>
#include <util/feefrac.h>
#include <algorithm>
#include <iterator>
#include <map>
#include <memory>
#include <set>
#include <stdint.h>
#include <utility>
using namespace cluster_linearize;
namespace {
/** Data type representing a naive simulated TxGraph, keeping all transactions (even from
* disconnected components) in a single DepGraph. Unlike the real TxGraph, this only models
* a single graph, and multiple instances are used to simulate main/staging. */
struct SimTxGraph
{
/** Maximum number of transactions to support simultaneously. Set this higher than txgraph's
* cluster count, so we can exercise situations with more transactions than fit in one
* cluster. */
static constexpr unsigned MAX_TRANSACTIONS = MAX_CLUSTER_COUNT_LIMIT * 2;
/** Set type to use in the simulation. */
using SetType = BitSet<MAX_TRANSACTIONS>;
/** Data type for representing positions within SimTxGraph::graph. */
using Pos = DepGraphIndex;
/** Constant to mean "missing in this graph". */
static constexpr auto MISSING = Pos(-1);
/** The dependency graph (for all transactions in the simulation, regardless of
* connectivity/clustering). */
DepGraph<SetType> graph;
/** For each position in graph, which TxGraph::Ref it corresponds with (if any). Use shared_ptr
* so that a SimTxGraph can be copied to create a staging one, while sharing Refs with
* the main graph. */
std::array<std::shared_ptr<TxGraph::Ref>, MAX_TRANSACTIONS> simmap;
/** For each TxGraph::Ref in graph, the position it corresponds with. */
std::map<const TxGraph::Ref*, Pos> simrevmap;
/** The set of TxGraph::Ref entries that have been removed, but not yet destroyed. */
std::vector<std::shared_ptr<TxGraph::Ref>> removed;
/** Whether the graph is oversized (true = yes, false = no, std::nullopt = unknown). */
std::optional<bool> oversized;
/** The configured maximum number of transactions per cluster. */
DepGraphIndex max_cluster_count;
/** The configured maximum total size of transactions per cluster. */
uint64_t max_cluster_size;
/** Construct a new SimTxGraph with the specified maximum cluster count. */
explicit SimTxGraph(DepGraphIndex max_cluster, uint64_t max_size) :
max_cluster_count(max_cluster), max_cluster_size(max_size) {}
// Permit copying and moving.
SimTxGraph(const SimTxGraph&) noexcept = default;
SimTxGraph& operator=(const SimTxGraph&) noexcept = default;
SimTxGraph(SimTxGraph&&) noexcept = default;
SimTxGraph& operator=(SimTxGraph&&) noexcept = default;
/** Check whether this graph is oversized (contains a connected component whose number of
* transactions exceeds max_cluster_count. */
bool IsOversized()
{
if (!oversized.has_value()) {
// Only recompute when oversized isn't already known.
oversized = false;
auto todo = graph.Positions();
// Iterate over all connected components of the graph.
while (todo.Any()) {
auto component = graph.FindConnectedComponent(todo);
if (component.Count() > max_cluster_count) oversized = true;
uint64_t component_size{0};
for (auto i : component) component_size += graph.FeeRate(i).size;
if (component_size > max_cluster_size) oversized = true;
todo -= component;
}
}
return *oversized;
}
/** Determine the number of (non-removed) transactions in the graph. */
DepGraphIndex GetTransactionCount() const { return graph.TxCount(); }
/** Get the sum of all fees/sizes in the graph. */
FeePerWeight SumAll() const
{
FeePerWeight ret;
for (auto i : graph.Positions()) {
ret += graph.FeeRate(i);
}
return ret;
}
/** Get the position where ref occurs in this simulated graph, or -1 if it does not. */
Pos Find(const TxGraph::Ref* ref) const
{
auto it = simrevmap.find(ref);
if (it != simrevmap.end()) return it->second;
return MISSING;
}
/** Given a position in this simulated graph, get the corresponding TxGraph::Ref. */
TxGraph::Ref* GetRef(Pos pos)
{
assert(graph.Positions()[pos]);
assert(simmap[pos]);
return simmap[pos].get();
}
/** Add a new transaction to the simulation. */
TxGraph::Ref* AddTransaction(const FeePerWeight& feerate)
{
assert(graph.TxCount() < MAX_TRANSACTIONS);
auto simpos = graph.AddTransaction(feerate);
assert(graph.Positions()[simpos]);
simmap[simpos] = std::make_shared<TxGraph::Ref>();
auto ptr = simmap[simpos].get();
simrevmap[ptr] = simpos;
// This may invalidate our cached oversized value.
if (oversized.has_value() && !*oversized) oversized = std::nullopt;
return ptr;
}
/** Add a dependency between two positions in this graph. */
void AddDependency(TxGraph::Ref* parent, TxGraph::Ref* child)
{
auto par_pos = Find(parent);
if (par_pos == MISSING) return;
auto chl_pos = Find(child);
if (chl_pos == MISSING) return;
graph.AddDependencies(SetType::Singleton(par_pos), chl_pos);
// This may invalidate our cached oversized value.
if (oversized.has_value() && !*oversized) oversized = std::nullopt;
}
/** Modify the transaction fee of a ref, if it exists. */
void SetTransactionFee(TxGraph::Ref* ref, int64_t fee)
{
auto pos = Find(ref);
if (pos == MISSING) return;
graph.FeeRate(pos).fee = fee;
}
/** Remove the transaction in the specified position from the graph. */
void RemoveTransaction(TxGraph::Ref* ref)
{
auto pos = Find(ref);
if (pos == MISSING) return;
graph.RemoveTransactions(SetType::Singleton(pos));
simrevmap.erase(simmap[pos].get());
// Retain the TxGraph::Ref corresponding to this position, so the Ref destruction isn't
// invoked until the simulation explicitly decided to do so.
removed.push_back(std::move(simmap[pos]));
simmap[pos].reset();
// This may invalidate our cached oversized value.
if (oversized.has_value() && *oversized) oversized = std::nullopt;
}
/** Destroy the transaction from the graph, including from the removed set. This will
* trigger TxGraph::Ref::~Ref. reset_oversize controls whether the cached oversized
* value is cleared (destroying does not clear oversizedness in TxGraph of the main
* graph while staging exists). */
void DestroyTransaction(TxGraph::Ref* ref, bool reset_oversize)
{
auto pos = Find(ref);
if (pos == MISSING) {
// Wipe the ref, if it exists, from the removed vector. Use std::partition rather
// than std::erase because we don't care about the order of the entries that
// remain.
auto remove = std::partition(removed.begin(), removed.end(), [&](auto& arg) { return arg.get() != ref; });
removed.erase(remove, removed.end());
} else {
graph.RemoveTransactions(SetType::Singleton(pos));
simrevmap.erase(simmap[pos].get());
simmap[pos].reset();
// This may invalidate our cached oversized value.
if (reset_oversize && oversized.has_value() && *oversized) {
oversized = std::nullopt;
}
}
}
/** Construct the set with all positions in this graph corresponding to the specified
* TxGraph::Refs. All of them must occur in this graph and not be removed. */
SetType MakeSet(std::span<TxGraph::Ref* const> arg)
{
SetType ret;
for (TxGraph::Ref* ptr : arg) {
auto pos = Find(ptr);
assert(pos != Pos(-1));
ret.Set(pos);
}
return ret;
}
/** Get the set of ancestors (desc=false) or descendants (desc=true) in this graph. */
SetType GetAncDesc(TxGraph::Ref* arg, bool desc)
{
auto pos = Find(arg);
if (pos == MISSING) return {};
return desc ? graph.Descendants(pos) : graph.Ancestors(pos);
}
/** Given a set of Refs (given as a vector of pointers), expand the set to include all its
* ancestors (desc=false) or all its descendants (desc=true) in this graph. */
void IncludeAncDesc(std::vector<TxGraph::Ref*>& arg, bool desc)
{
std::vector<TxGraph::Ref*> ret;
for (auto ptr : arg) {
auto simpos = Find(ptr);
if (simpos != MISSING) {
for (auto i : desc ? graph.Descendants(simpos) : graph.Ancestors(simpos)) {
ret.push_back(simmap[i].get());
}
} else {
ret.push_back(ptr);
}
}
// Deduplicate.
std::sort(ret.begin(), ret.end());
ret.erase(std::unique(ret.begin(), ret.end()), ret.end());
// Replace input.
arg = std::move(ret);
}
};
} // namespace
FUZZ_TARGET(txgraph)
{
// This is a big simulation test for TxGraph, which performs a fuzz-derived sequence of valid
// operations on a TxGraph instance, as well as on a simpler (mostly) reimplementation (see
// SimTxGraph above), comparing the outcome of functions that return a result, and finally
// performing a full comparison between the two.
SeedRandomStateForTest(SeedRand::ZEROS);
FuzzedDataProvider provider(buffer.data(), buffer.size());
/** Internal test RNG, used only for decisions which would require significant amount of data
* to be read from the provider, without realistically impacting test sensitivity. */
InsecureRandomContext rng(0xdecade2009added + buffer.size());
/** Variable used whenever an empty TxGraph::Ref is needed. */
TxGraph::Ref empty_ref;
// Decide the maximum number of transactions per cluster we will use in this simulation.
auto max_count = provider.ConsumeIntegralInRange<DepGraphIndex>(1, MAX_CLUSTER_COUNT_LIMIT);
// And the maximum combined size of transactions per cluster.
auto max_size = provider.ConsumeIntegralInRange<uint64_t>(1, 0x3fffff * MAX_CLUSTER_COUNT_LIMIT);
// Construct a real graph, and a vector of simulated graphs (main, and possibly staging).
auto real = MakeTxGraph(max_count, max_size);
std::vector<SimTxGraph> sims;
sims.reserve(2);
sims.emplace_back(max_count, max_size);
/** Struct encapsulating information about a BlockBuilder that's currently live. */
struct BlockBuilderData
{
/** BlockBuilder object from real. */
std::unique_ptr<TxGraph::BlockBuilder> builder;
/** The set of transactions marked as included in *builder. */
SimTxGraph::SetType done;
/** The last chunk feerate returned by *builder. IsEmpty() if none yet. */
FeePerWeight last_feerate;
BlockBuilderData(std::unique_ptr<TxGraph::BlockBuilder> builder_in) : builder(std::move(builder_in)) {}
};
/** Currently active block builders. */
std::vector<BlockBuilderData> block_builders;
/** Function to pick any Ref (for either sim in sims: from sim.simmap or sim.removed, or the
* empty Ref). */
auto pick_fn = [&]() noexcept -> TxGraph::Ref* {
size_t tx_count[2] = {sims[0].GetTransactionCount(), 0};
/** The number of possible choices. */
size_t choices = tx_count[0] + sims[0].removed.size() + 1;
if (sims.size() == 2) {
tx_count[1] = sims[1].GetTransactionCount();
choices += tx_count[1] + sims[1].removed.size();
}
/** Pick one of them. */
auto choice = provider.ConsumeIntegralInRange<size_t>(0, choices - 1);
// Consider both main and (if it exists) staging.
for (size_t level = 0; level < sims.size(); ++level) {
auto& sim = sims[level];
if (choice < tx_count[level]) {
// Return from graph.
for (auto i : sim.graph.Positions()) {
if (choice == 0) return sim.GetRef(i);
--choice;
}
assert(false);
} else {
choice -= tx_count[level];
}
if (choice < sim.removed.size()) {
// Return from removed.
return sim.removed[choice].get();
} else {
choice -= sim.removed.size();
}
}
// Return empty.
assert(choice == 0);
return &empty_ref;
};
/** Function to construct the full diagram for a simulated graph. This works by fetching the
* clusters and chunking them manually, so it works for both main and staging
* (GetMainChunkFeerate only works for main). */
auto get_diagram_fn = [&](bool main_only) -> std::vector<FeeFrac> {
int level = main_only ? 0 : sims.size() - 1;
auto& sim = sims[level];
// For every transaction in the graph, request its cluster, and throw them into a set.
std::set<std::vector<TxGraph::Ref*>> clusters;
for (auto i : sim.graph.Positions()) {
auto ref = sim.GetRef(i);
clusters.insert(real->GetCluster(*ref, main_only));
}
// Compute the chunkings of each (deduplicated) cluster.
size_t num_tx{0};
std::vector<FeeFrac> ret;
for (const auto& cluster : clusters) {
num_tx += cluster.size();
std::vector<SimTxGraph::Pos> linearization;
linearization.reserve(cluster.size());
for (auto refptr : cluster) linearization.push_back(sim.Find(refptr));
for (const FeeFrac& chunk_feerate : ChunkLinearization(sim.graph, linearization)) {
ret.push_back(chunk_feerate);
}
}
// Verify the number of transactions after deduplicating clusters. This implicitly verifies
// that GetCluster on each element of a cluster reports the cluster transactions in the same
// order.
assert(num_tx == sim.GetTransactionCount());
// Sort by feerate (we don't care about respecting ordering within clusters, as these are
// just feerates).
std::sort(ret.begin(), ret.end(), std::greater{});
return ret;
};
LIMITED_WHILE(provider.remaining_bytes() > 0, 200) {
// Read a one-byte command.
int command = provider.ConsumeIntegral<uint8_t>();
/** Use the bottom 2 bits of command to select an entry in the block_builders vector (if
* any). */
int builder_idx = block_builders.empty() ? -1 : int((command & 3) % block_builders.size());
// Treat the lowest bit of a command as a flag (which selects a variant of some of the
// operations), and the second-lowest bit as a way of selecting main vs. staging, and leave
// the rest of the bits in command.
bool alt = command & 1;
bool use_main = command & 2;
command >>= 2;
// Provide convenient aliases for the top simulated graph (main, or staging if it exists),
// one for the simulated graph selected based on use_main (for operations that can operate
// on both graphs), and one that always refers to the main graph.
auto& top_sim = sims.back();
auto& sel_sim = use_main ? sims[0] : top_sim;
auto& main_sim = sims[0];
// Keep decrementing command for each applicable operation, until one is hit. Multiple
// iterations may be necessary.
while (true) {
if ((block_builders.empty() || sims.size() > 1) && top_sim.GetTransactionCount() < SimTxGraph::MAX_TRANSACTIONS && command-- == 0) {
// AddTransaction.
int64_t fee;
int32_t size;
if (alt) {
// If alt is true, pick fee and size from the entire range.
fee = provider.ConsumeIntegralInRange<int64_t>(-0x8000000000000, 0x7ffffffffffff);
size = provider.ConsumeIntegralInRange<int32_t>(1, 0x3fffff);
} else {
// Otherwise, use smaller range which consume fewer fuzz input bytes, as just
// these are likely sufficient to trigger all interesting code paths already.
fee = provider.ConsumeIntegral<uint8_t>();
size = provider.ConsumeIntegralInRange<uint32_t>(1, 0xff);
}
FeePerWeight feerate{fee, size};
// Create a real TxGraph::Ref.
auto ref = real->AddTransaction(feerate);
// Create a shared_ptr place in the simulation to put the Ref in.
auto ref_loc = top_sim.AddTransaction(feerate);
// Move it in place.
*ref_loc = std::move(ref);
break;
} else if ((block_builders.empty() || sims.size() > 1) && top_sim.GetTransactionCount() + top_sim.removed.size() > 1 && command-- == 0) {
// AddDependency.
auto par = pick_fn();
auto chl = pick_fn();
auto pos_par = top_sim.Find(par);
auto pos_chl = top_sim.Find(chl);
if (pos_par != SimTxGraph::MISSING && pos_chl != SimTxGraph::MISSING) {
// Determine if adding this would introduce a cycle (not allowed by TxGraph),
// and if so, skip.
if (top_sim.graph.Ancestors(pos_par)[pos_chl]) break;
}
top_sim.AddDependency(par, chl);
real->AddDependency(*par, *chl);
break;
} else if ((block_builders.empty() || sims.size() > 1) && top_sim.removed.size() < 100 && command-- == 0) {
// RemoveTransaction. Either all its ancestors or all its descendants are also
// removed (if any), to make sure TxGraph's reordering of removals and dependencies
// has no effect.
std::vector<TxGraph::Ref*> to_remove;
to_remove.push_back(pick_fn());
top_sim.IncludeAncDesc(to_remove, alt);
// The order in which these ancestors/descendants are removed should not matter;
// randomly shuffle them.
std::shuffle(to_remove.begin(), to_remove.end(), rng);
for (TxGraph::Ref* ptr : to_remove) {
real->RemoveTransaction(*ptr);
top_sim.RemoveTransaction(ptr);
}
break;
} else if (sel_sim.removed.size() > 0 && command-- == 0) {
// ~Ref (of an already-removed transaction). Destroying a TxGraph::Ref has an
// observable effect on the TxGraph it refers to, so this simulation permits doing
// so separately from other actions on TxGraph.
// Pick a Ref of sel_sim.removed to destroy. Note that the same Ref may still occur
// in the other graph, and thus not actually trigger ~Ref yet (which is exactly
// what we want, as destroying Refs is only allowed when it does not refer to an
// existing transaction in either graph).
auto removed_pos = provider.ConsumeIntegralInRange<size_t>(0, sel_sim.removed.size() - 1);
if (removed_pos != sel_sim.removed.size() - 1) {
std::swap(sel_sim.removed[removed_pos], sel_sim.removed.back());
}
sel_sim.removed.pop_back();
break;
} else if (block_builders.empty() && command-- == 0) {
// ~Ref (of any transaction).
std::vector<TxGraph::Ref*> to_destroy;
to_destroy.push_back(pick_fn());
while (true) {
// Keep adding either the ancestors or descendants the already picked
// transactions have in both graphs (main and staging) combined. Destroying
// will trigger deletions in both, so to have consistent TxGraph behavior, the
// set must be closed under ancestors, or descendants, in both graphs.
auto old_size = to_destroy.size();
for (auto& sim : sims) sim.IncludeAncDesc(to_destroy, alt);
if (to_destroy.size() == old_size) break;
}
// The order in which these ancestors/descendants are destroyed should not matter;
// randomly shuffle them.
std::shuffle(to_destroy.begin(), to_destroy.end(), rng);
for (TxGraph::Ref* ptr : to_destroy) {
for (size_t level = 0; level < sims.size(); ++level) {
sims[level].DestroyTransaction(ptr, level == sims.size() - 1);
}
}
break;
} else if (block_builders.empty() && command-- == 0) {
// SetTransactionFee.
int64_t fee;
if (alt) {
fee = provider.ConsumeIntegralInRange<int64_t>(-0x8000000000000, 0x7ffffffffffff);
} else {
fee = provider.ConsumeIntegral<uint8_t>();
}
auto ref = pick_fn();
real->SetTransactionFee(*ref, fee);
for (auto& sim : sims) {
sim.SetTransactionFee(ref, fee);
}
break;
} else if (command-- == 0) {
// GetTransactionCount.
assert(real->GetTransactionCount(use_main) == sel_sim.GetTransactionCount());
break;
} else if (command-- == 0) {
// Exists.
auto ref = pick_fn();
bool exists = real->Exists(*ref, use_main);
bool should_exist = sel_sim.Find(ref) != SimTxGraph::MISSING;
assert(exists == should_exist);
break;
} else if (command-- == 0) {
// IsOversized.
assert(sel_sim.IsOversized() == real->IsOversized(use_main));
break;
} else if (command-- == 0) {
// GetIndividualFeerate.
auto ref = pick_fn();
auto feerate = real->GetIndividualFeerate(*ref);
bool found{false};
for (auto& sim : sims) {
auto simpos = sim.Find(ref);
if (simpos != SimTxGraph::MISSING) {
found = true;
assert(feerate == sim.graph.FeeRate(simpos));
}
}
if (!found) assert(feerate.IsEmpty());
break;
} else if (!main_sim.IsOversized() && command-- == 0) {
// GetMainChunkFeerate.
auto ref = pick_fn();
auto feerate = real->GetMainChunkFeerate(*ref);
auto simpos = main_sim.Find(ref);
if (simpos == SimTxGraph::MISSING) {
assert(feerate.IsEmpty());
} else {
// Just do some quick checks that the reported value is in range. A full
// recomputation of expected chunk feerates is done at the end.
assert(feerate.size >= main_sim.graph.FeeRate(simpos).size);
assert(feerate.size <= main_sim.SumAll().size);
}
break;
} else if (!sel_sim.IsOversized() && command-- == 0) {
// GetAncestors/GetDescendants.
auto ref = pick_fn();
auto result = alt ? real->GetDescendants(*ref, use_main)
: real->GetAncestors(*ref, use_main);
assert(result.size() <= max_count);
auto result_set = sel_sim.MakeSet(result);
assert(result.size() == result_set.Count());
auto expect_set = sel_sim.GetAncDesc(ref, alt);
assert(result_set == expect_set);
break;
} else if (!sel_sim.IsOversized() && command-- == 0) {
// GetAncestorsUnion/GetDescendantsUnion.
std::vector<TxGraph::Ref*> refs;
// Gather a list of up to 15 Ref pointers.
auto count = provider.ConsumeIntegralInRange<size_t>(0, 15);
refs.resize(count);
for (size_t i = 0; i < count; ++i) {
refs[i] = pick_fn();
}
// Their order should not matter, shuffle them.
std::shuffle(refs.begin(), refs.end(), rng);
// Invoke the real function, and convert to SimPos set.
auto result = alt ? real->GetDescendantsUnion(refs, use_main)
: real->GetAncestorsUnion(refs, use_main);
auto result_set = sel_sim.MakeSet(result);
assert(result.size() == result_set.Count());
// Compute the expected result.
SimTxGraph::SetType expect_set;
for (TxGraph::Ref* ref : refs) expect_set |= sel_sim.GetAncDesc(ref, alt);
// Compare.
assert(result_set == expect_set);
break;
} else if (!sel_sim.IsOversized() && command-- == 0) {
// GetCluster.
auto ref = pick_fn();
auto result = real->GetCluster(*ref, use_main);
// Check cluster count limit.
assert(result.size() <= max_count);
// Require the result to be topologically valid and not contain duplicates.
auto left = sel_sim.graph.Positions();
uint64_t total_size{0};
for (auto refptr : result) {
auto simpos = sel_sim.Find(refptr);
total_size += sel_sim.graph.FeeRate(simpos).size;
assert(simpos != SimTxGraph::MISSING);
assert(left[simpos]);
left.Reset(simpos);
assert(!sel_sim.graph.Ancestors(simpos).Overlaps(left));
}
// Check cluster size limit.
assert(total_size <= max_size);
// Require the set to be connected.
auto result_set = sel_sim.MakeSet(result);
assert(sel_sim.graph.IsConnected(result_set));
// If ref exists, the result must contain it. If not, it must be empty.
auto simpos = sel_sim.Find(ref);
if (simpos != SimTxGraph::MISSING) {
assert(result_set[simpos]);
} else {
assert(result_set.None());
}
// Require the set not to have ancestors or descendants outside of it.
for (auto i : result_set) {
assert(sel_sim.graph.Ancestors(i).IsSubsetOf(result_set));
assert(sel_sim.graph.Descendants(i).IsSubsetOf(result_set));
}
break;
} else if (command-- == 0) {
// HaveStaging.
assert((sims.size() == 2) == real->HaveStaging());
break;
} else if (sims.size() < 2 && command-- == 0) {
// StartStaging.
sims.emplace_back(sims.back());
real->StartStaging();
break;
} else if (block_builders.empty() && sims.size() > 1 && command-- == 0) {
// CommitStaging.
real->CommitStaging();
sims.erase(sims.begin());
break;
} else if (sims.size() > 1 && command-- == 0) {
// AbortStaging.
real->AbortStaging();
sims.pop_back();
// Reset the cached oversized value (if TxGraph::Ref destructions triggered
// removals of main transactions while staging was active, then aborting will
// cause it to be re-evaluated in TxGraph).
sims.back().oversized = std::nullopt;
break;
} else if (!main_sim.IsOversized() && command-- == 0) {
// CompareMainOrder.
auto ref_a = pick_fn();
auto ref_b = pick_fn();
auto sim_a = main_sim.Find(ref_a);
auto sim_b = main_sim.Find(ref_b);
// Both transactions must exist in the main graph.
if (sim_a == SimTxGraph::MISSING || sim_b == SimTxGraph::MISSING) break;
auto cmp = real->CompareMainOrder(*ref_a, *ref_b);
// Distinct transactions have distinct places.
if (sim_a != sim_b) assert(cmp != 0);
// Ancestors go before descendants.
if (main_sim.graph.Ancestors(sim_a)[sim_b]) assert(cmp >= 0);
if (main_sim.graph.Descendants(sim_a)[sim_b]) assert(cmp <= 0);
// Do not verify consistency with chunk feerates, as we cannot easily determine
// these here without making more calls to real, which could affect its internal
// state. A full comparison is done at the end.
break;
} else if (!sel_sim.IsOversized() && command-- == 0) {
// CountDistinctClusters.
std::vector<TxGraph::Ref*> refs;
// Gather a list of up to 15 (or up to 255) Ref pointers.
auto count = provider.ConsumeIntegralInRange<size_t>(0, alt ? 255 : 15);
refs.resize(count);
for (size_t i = 0; i < count; ++i) {
refs[i] = pick_fn();
}
// Their order should not matter, shuffle them.
std::shuffle(refs.begin(), refs.end(), rng);
// Invoke the real function.
auto result = real->CountDistinctClusters(refs, use_main);
// Build a vector with representatives of the clusters the Refs occur in in the
// simulated graph. For each, remember the lowest-index transaction SimPos in the
// cluster.
std::vector<DepGraphIndex> sim_reps;
for (auto ref : refs) {
// Skip Refs that do not occur in the simulated graph.
auto simpos = sel_sim.Find(ref);
if (simpos == SimTxGraph::MISSING) continue;
// Start with component equal to just the Ref's SimPos.
auto component = SimTxGraph::SetType::Singleton(simpos);
// Keep adding ancestors/descendants of all elements in component until it no
// longer changes.
while (true) {
auto old_component = component;
for (auto i : component) {
component |= sel_sim.graph.Ancestors(i);
component |= sel_sim.graph.Descendants(i);
}
if (component == old_component) break;
}
// Remember the lowest-index SimPos in component, as a representative for it.
assert(component.Any());
sim_reps.push_back(component.First());
}
// Remove duplicates from sim_reps.
std::sort(sim_reps.begin(), sim_reps.end());
sim_reps.erase(std::unique(sim_reps.begin(), sim_reps.end()), sim_reps.end());
// Compare the number of deduplicated representatives with the value returned by
// the real function.
assert(result == sim_reps.size());
break;
} else if (command-- == 0) {
// DoWork.
real->DoWork();
break;
} else if (sims.size() == 2 && !sims[0].IsOversized() && !sims[1].IsOversized() && command-- == 0) {
// GetMainStagingDiagrams()
auto [main_diagram, staged_diagram] = real->GetMainStagingDiagrams();
auto sum_main = std::accumulate(main_diagram.begin(), main_diagram.end(), FeeFrac{});
auto sum_staged = std::accumulate(staged_diagram.begin(), staged_diagram.end(), FeeFrac{});
auto diagram_gain = sum_staged - sum_main;
auto real_gain = sims[1].SumAll() - sims[0].SumAll();
// Just check that the total fee gained/lost and size gained/lost according to the
// diagram matches the difference in these values in the simulated graph. A more
// complete check of the GetMainStagingDiagrams result is performed at the end.
assert(diagram_gain == real_gain);
// Check that the feerates in each diagram are monotonically decreasing.
for (size_t i = 1; i < main_diagram.size(); ++i) {
assert(FeeRateCompare(main_diagram[i], main_diagram[i - 1]) <= 0);
}
for (size_t i = 1; i < staged_diagram.size(); ++i) {
assert(FeeRateCompare(staged_diagram[i], staged_diagram[i - 1]) <= 0);
}
break;
} else if (block_builders.size() < 4 && !main_sim.IsOversized() && command-- == 0) {
// GetBlockBuilder.
block_builders.emplace_back(real->GetBlockBuilder());
break;
} else if (!block_builders.empty() && command-- == 0) {
// ~BlockBuilder.
block_builders.erase(block_builders.begin() + builder_idx);
break;
} else if (!block_builders.empty() && *block_builders[builder_idx].builder && command-- == 0) {
// BlockBuilder::Include and BlockBuilder::Skip.
auto& builder_data = block_builders[builder_idx];
auto cur_feerate = builder_data.builder->GetCurrentChunkFeerate();
// Chunk feerates must be monotonously decreasing.
if (!builder_data.last_feerate.IsEmpty()) {
assert(!(cur_feerate >> builder_data.last_feerate));
}
builder_data.last_feerate = cur_feerate;
// Verify the contents of GetCurrentChunk.
auto new_done = builder_data.done;
FeePerWeight sum_feerate;
for (TxGraph::Ref* ref : builder_data.builder->GetCurrentChunk()) {
// Each transaction in the chunk must exist in the main graph.
auto simpos = main_sim.Find(ref);
assert(simpos != SimTxGraph::MISSING);
// Verify the claimed chunk feerate.
sum_feerate += main_sim.graph.FeeRate(simpos);
// Make sure no transaction is reported twice.
assert(!new_done[simpos]);
new_done.Set(simpos);
// The concatenation of all included transactions must be topologically valid.
assert(main_sim.graph.Ancestors(simpos).IsSubsetOf(new_done));
}
assert(sum_feerate == cur_feerate);
// Skip or Include.
if (alt) {
builder_data.builder->Skip();
} else {
builder_data.builder->Include();
builder_data.done = new_done;
}
break;
} else if (!main_sim.IsOversized() && command-- == 0) {
// GetWorstMainChunk.
auto [worst_chunk, worst_chunk_feerate] = real->GetWorstMainChunk();
// Just do some sanity checks here. Consistency with GetBlockBuilder is checked
// below.
if (main_sim.GetTransactionCount() == 0) {
assert(worst_chunk.empty());
} else {
assert(!worst_chunk.empty());
SimTxGraph::SetType done;
FeePerWeight sum;
for (TxGraph::Ref* ref : worst_chunk) {
// Each transaction in the chunk must exist in the main graph.
auto simpos = main_sim.Find(ref);
assert(simpos != SimTxGraph::MISSING);
sum += main_sim.graph.FeeRate(simpos);
// Make sure the chunk contains no duplicate transactions.
assert(!done[simpos]);
done.Set(simpos);
// All elements are preceded by all their descendants.
assert(main_sim.graph.Descendants(simpos).IsSubsetOf(done));
}
assert(sum == worst_chunk_feerate);
}
break;
} else if ((block_builders.empty() || sims.size() > 1) && command-- == 0) {
// Trim.
bool was_oversized = top_sim.IsOversized();
auto removed = real->Trim();
if (!was_oversized) {
assert(removed.empty());
break;
}
auto removed_set = top_sim.MakeSet(removed);
// The removed set must contain all its own descendants.
for (auto simpos : removed_set) {
assert(top_sim.graph.Descendants(simpos).IsSubsetOf(removed_set));
}
// Apply all removals to the simulation, and verify the result is no longer
// oversized. Don't query the real graph for oversizedness; it is compared
// against the simulation anyway later.
for (auto simpos : removed_set) {
top_sim.RemoveTransaction(top_sim.GetRef(simpos));
}
assert(!top_sim.IsOversized());
break;
}
}
}
// After running all modifications, perform an internal sanity check (before invoking
// inspectors that may modify the internal state).
real->SanityCheck();
if (!sims[0].IsOversized()) {
// If the main graph is not oversized, verify the total ordering implied by
// CompareMainOrder.
// First construct two distinct randomized permutations of the positions in sims[0].
std::vector<SimTxGraph::Pos> vec1;
for (auto i : sims[0].graph.Positions()) vec1.push_back(i);
std::shuffle(vec1.begin(), vec1.end(), rng);
auto vec2 = vec1;
std::shuffle(vec2.begin(), vec2.end(), rng);
if (vec1 == vec2) std::next_permutation(vec2.begin(), vec2.end());
// Sort both according to CompareMainOrder. By having randomized starting points, the order
// of CompareMainOrder invocations is somewhat randomized as well.
auto cmp = [&](SimTxGraph::Pos a, SimTxGraph::Pos b) noexcept {
return real->CompareMainOrder(*sims[0].GetRef(a), *sims[0].GetRef(b)) < 0;
};
std::sort(vec1.begin(), vec1.end(), cmp);
std::sort(vec2.begin(), vec2.end(), cmp);
// Verify the resulting orderings are identical. This could only fail if the ordering was
// not total.
assert(vec1 == vec2);
// Verify that the ordering is topological.
auto todo = sims[0].graph.Positions();
for (auto i : vec1) {
todo.Reset(i);
assert(!sims[0].graph.Ancestors(i).Overlaps(todo));
}
assert(todo.None());
// For every transaction in the total ordering, find a random one before it and after it,
// and compare their chunk feerates, which must be consistent with the ordering.
for (size_t pos = 0; pos < vec1.size(); ++pos) {
auto pos_feerate = real->GetMainChunkFeerate(*sims[0].GetRef(vec1[pos]));
if (pos > 0) {
size_t before = rng.randrange<size_t>(pos);
auto before_feerate = real->GetMainChunkFeerate(*sims[0].GetRef(vec1[before]));
assert(FeeRateCompare(before_feerate, pos_feerate) >= 0);
}
if (pos + 1 < vec1.size()) {
size_t after = pos + 1 + rng.randrange<size_t>(vec1.size() - 1 - pos);
auto after_feerate = real->GetMainChunkFeerate(*sims[0].GetRef(vec1[after]));
assert(FeeRateCompare(after_feerate, pos_feerate) <= 0);
}
}
// The same order should be obtained through a BlockBuilder, if nothing is skipped.
auto builder = real->GetBlockBuilder();
std::vector<SimTxGraph::Pos> vec_builder;
std::vector<TxGraph::Ref*> chunk;
FeePerWeight chunk_feerate;
while (*builder) {
FeePerWeight sum;
auto chunk_span = builder->GetCurrentChunk();
chunk.assign(chunk_span.begin(), chunk_span.end());
for (TxGraph::Ref* ref : chunk_span) {
// The reported chunk feerate must match the chunk feerate obtained by asking
// it for each of the chunk's transactions individually.
assert(real->GetMainChunkFeerate(*ref) == builder->GetCurrentChunkFeerate());
// Verify the chunk feerate matches the sum of the reported individual feerates.
sum += real->GetIndividualFeerate(*ref);
// Chunks must contain transactions that exist in the graph.
auto simpos = sims[0].Find(ref);
assert(simpos != SimTxGraph::MISSING);
vec_builder.push_back(simpos);
}
chunk_feerate = builder->GetCurrentChunkFeerate();
assert(sum == chunk_feerate);
builder->Include();
}
assert(vec_builder == vec1);
// The last chunk returned by the BlockBuilder must match GetWorstMainChunk, in reverse.
std::reverse(chunk.begin(), chunk.end());
auto [worst_chunk, worst_chunk_feerate] = real->GetWorstMainChunk();
assert(chunk == worst_chunk);
assert(chunk_feerate == worst_chunk_feerate);
// Check that the implied ordering gives rise to a combined diagram that matches the
// diagram constructed from the individual cluster linearization chunkings.
auto main_diagram = get_diagram_fn(true);
auto expected_main_diagram = ChunkLinearization(sims[0].graph, vec1);
assert(CompareChunks(main_diagram, expected_main_diagram) == 0);
if (sims.size() >= 2 && !sims[1].IsOversized()) {
// When the staging graph is not oversized as well, call GetMainStagingDiagrams, and
// fully verify the result.
auto [main_cmp_diagram, stage_cmp_diagram] = real->GetMainStagingDiagrams();
// Check that the feerates in each diagram are monotonically decreasing.
for (size_t i = 1; i < main_cmp_diagram.size(); ++i) {
assert(FeeRateCompare(main_cmp_diagram[i], main_cmp_diagram[i - 1]) <= 0);
}
for (size_t i = 1; i < stage_cmp_diagram.size(); ++i) {
assert(FeeRateCompare(stage_cmp_diagram[i], stage_cmp_diagram[i - 1]) <= 0);
}
// Apply total ordering on the feerate diagrams to make them comparable (the exact
// tie breaker among equal-feerate FeeFracs does not matter, but it has to be
// consistent with the one used in main_diagram and stage_diagram).
std::sort(main_cmp_diagram.begin(), main_cmp_diagram.end(), std::greater{});
std::sort(stage_cmp_diagram.begin(), stage_cmp_diagram.end(), std::greater{});
// Find the chunks that appear in main_diagram but are missing from main_cmp_diagram.
// This is allowed, because GetMainStagingDiagrams omits clusters in main unaffected
// by staging.
std::vector<FeeFrac> missing_main_cmp;
std::set_difference(main_diagram.begin(), main_diagram.end(),
main_cmp_diagram.begin(), main_cmp_diagram.end(),
std::inserter(missing_main_cmp, missing_main_cmp.end()),
std::greater{});
assert(main_cmp_diagram.size() + missing_main_cmp.size() == main_diagram.size());
// Do the same for chunks in stage_diagram missign from stage_cmp_diagram.
auto stage_diagram = get_diagram_fn(false);
std::vector<FeeFrac> missing_stage_cmp;
std::set_difference(stage_diagram.begin(), stage_diagram.end(),
stage_cmp_diagram.begin(), stage_cmp_diagram.end(),
std::inserter(missing_stage_cmp, missing_stage_cmp.end()),
std::greater{});
assert(stage_cmp_diagram.size() + missing_stage_cmp.size() == stage_diagram.size());
// The missing chunks must be equal across main & staging (otherwise they couldn't have
// been omitted).
assert(missing_main_cmp == missing_stage_cmp);
}
}
assert(real->HaveStaging() == (sims.size() > 1));
// Try to run a full comparison, for both main_only=false and main_only=true in TxGraph
// inspector functions that support both.
for (int main_only = 0; main_only < 2; ++main_only) {
auto& sim = main_only ? sims[0] : sims.back();
// Compare simple properties of the graph with the simulation.
assert(real->IsOversized(main_only) == sim.IsOversized());
assert(real->GetTransactionCount(main_only) == sim.GetTransactionCount());
// If the graph (and the simulation) are not oversized, perform a full comparison.
if (!sim.IsOversized()) {
auto todo = sim.graph.Positions();
// Iterate over all connected components of the resulting (simulated) graph, each of which
// should correspond to a cluster in the real one.
while (todo.Any()) {
auto component = sim.graph.FindConnectedComponent(todo);
todo -= component;
// Iterate over the transactions in that component.
for (auto i : component) {
// Check its individual feerate against simulation.
assert(sim.graph.FeeRate(i) == real->GetIndividualFeerate(*sim.GetRef(i)));
// Check its ancestors against simulation.
auto expect_anc = sim.graph.Ancestors(i);
auto anc = sim.MakeSet(real->GetAncestors(*sim.GetRef(i), main_only));
assert(anc.Count() <= max_count);
assert(anc == expect_anc);
// Check its descendants against simulation.
auto expect_desc = sim.graph.Descendants(i);
auto desc = sim.MakeSet(real->GetDescendants(*sim.GetRef(i), main_only));
assert(desc.Count() <= max_count);
assert(desc == expect_desc);
// Check the cluster the transaction is part of.
auto cluster = real->GetCluster(*sim.GetRef(i), main_only);
assert(cluster.size() <= max_count);
assert(sim.MakeSet(cluster) == component);
// Check that the cluster is reported in a valid topological order (its
// linearization).
std::vector<DepGraphIndex> simlin;
SimTxGraph::SetType done;
uint64_t total_size{0};
for (TxGraph::Ref* ptr : cluster) {
auto simpos = sim.Find(ptr);
assert(sim.graph.Descendants(simpos).IsSubsetOf(component - done));
done.Set(simpos);
assert(sim.graph.Ancestors(simpos).IsSubsetOf(done));
simlin.push_back(simpos);
total_size += sim.graph.FeeRate(simpos).size;
}
// Check cluster size.
assert(total_size <= max_size);
// Construct a chunking object for the simulated graph, using the reported cluster
// linearization as ordering, and compare it against the reported chunk feerates.
if (sims.size() == 1 || main_only) {
cluster_linearize::LinearizationChunking simlinchunk(sim.graph, simlin);
DepGraphIndex idx{0};
for (unsigned chunknum = 0; chunknum < simlinchunk.NumChunksLeft(); ++chunknum) {
auto chunk = simlinchunk.GetChunk(chunknum);
// Require that the chunks of cluster linearizations are connected (this must
// be the case as all linearizations inside are PostLinearized).
assert(sim.graph.IsConnected(chunk.transactions));
// Check the chunk feerates of all transactions in the cluster.
while (chunk.transactions.Any()) {
assert(chunk.transactions[simlin[idx]]);
chunk.transactions.Reset(simlin[idx]);
assert(chunk.feerate == real->GetMainChunkFeerate(*cluster[idx]));
++idx;
}
}
}
}
}
}
}
// Sanity check again (because invoking inspectors may modify internal unobservable state).
real->SanityCheck();
}

50
src/test/util/cluster_linearize.h
View File

@ -23,18 +23,6 @@ using namespace cluster_linearize;
using TestBitSet = BitSet<32>;
/** Check if a graph is acyclic. */
template<typename SetType>
bool IsAcyclic(const DepGraph<SetType>& depgraph) noexcept
{
for (ClusterIndex i : depgraph.Positions()) {
if ((depgraph.Ancestors(i) & depgraph.Descendants(i)) != SetType::Singleton(i)) {
return false;
}
}
return true;
}
/** A formatter for a bespoke serialization for acyclic DepGraph objects.
*
* The serialization format outputs information about transactions in a topological order (parents
@ -134,10 +122,10 @@ struct DepGraphFormatter
static void Ser(Stream& s, const DepGraph<SetType>& depgraph)
{
/** Construct a topological order to serialize the transactions in. */
std::vector<ClusterIndex> topo_order;
std::vector<DepGraphIndex> topo_order;
topo_order.reserve(depgraph.TxCount());
for (auto i : depgraph.Positions()) topo_order.push_back(i);
std::sort(topo_order.begin(), topo_order.end(), [&](ClusterIndex a, ClusterIndex b) {
std::sort(topo_order.begin(), topo_order.end(), [&](DepGraphIndex a, DepGraphIndex b) {
auto anc_a = depgraph.Ancestors(a).Count(), anc_b = depgraph.Ancestors(b).Count();
if (anc_a != anc_b) return anc_a < anc_b;
return a < b;
@ -148,9 +136,9 @@ struct DepGraphFormatter
SetType done;
// Loop over the transactions in topological order.
for (ClusterIndex topo_idx = 0; topo_idx < topo_order.size(); ++topo_idx) {
for (DepGraphIndex topo_idx = 0; topo_idx < topo_order.size(); ++topo_idx) {
/** Which depgraph index we are currently writing. */
ClusterIndex idx = topo_order[topo_idx];
DepGraphIndex idx = topo_order[topo_idx];
// Write size, which must be larger than 0.
s << VARINT_MODE(depgraph.FeeRate(idx).size, VarIntMode::NONNEGATIVE_SIGNED);
// Write fee, encoded as an unsigned varint (odd=negative, even=non-negative).
@ -158,9 +146,9 @@ struct DepGraphFormatter
// Write dependency information.
SetType written_parents;
uint64_t diff = 0; //!< How many potential parent/child relations we have skipped over.
for (ClusterIndex dep_dist = 0; dep_dist < topo_idx; ++dep_dist) {
for (DepGraphIndex dep_dist = 0; dep_dist < topo_idx; ++dep_dist) {
/** Which depgraph index we are currently considering as parent of idx. */
ClusterIndex dep_idx = topo_order[topo_idx - 1 - dep_dist];
DepGraphIndex dep_idx = topo_order[topo_idx - 1 - dep_dist];
// Ignore transactions which are already known to be ancestors.
if (depgraph.Descendants(dep_idx).Overlaps(written_parents)) continue;
if (depgraph.Ancestors(idx)[dep_idx]) {
@ -203,9 +191,9 @@ struct DepGraphFormatter
DepGraph<SetType> topo_depgraph;
/** Mapping from serialization order to cluster order, used later to reconstruct the
* cluster order. */
std::vector<ClusterIndex> reordering;
std::vector<DepGraphIndex> reordering;
/** How big the entries vector in the reconstructed depgraph will be (including holes). */
ClusterIndex total_size{0};
DepGraphIndex total_size{0};
// Read transactions in topological order.
while (true) {
@ -229,9 +217,9 @@ struct DepGraphFormatter
// Read dependency information.
auto topo_idx = reordering.size();
s >> VARINT(diff);
for (ClusterIndex dep_dist = 0; dep_dist < topo_idx; ++dep_dist) {
for (DepGraphIndex dep_dist = 0; dep_dist < topo_idx; ++dep_dist) {
/** Which topo_depgraph index we are currently considering as parent of topo_idx. */
ClusterIndex dep_topo_idx = topo_idx - 1 - dep_dist;
DepGraphIndex dep_topo_idx = topo_idx - 1 - dep_dist;
// Ignore transactions which are already known ancestors of topo_idx.
if (new_ancestors[dep_topo_idx]) continue;
if (diff == 0) {
@ -298,9 +286,9 @@ template<typename SetType>
void SanityCheck(const DepGraph<SetType>& depgraph)
{
// Verify Positions and PositionRange consistency.
ClusterIndex num_positions{0};
ClusterIndex position_range{0};
for (ClusterIndex i : depgraph.Positions()) {
DepGraphIndex num_positions{0};
DepGraphIndex position_range{0};
for (DepGraphIndex i : depgraph.Positions()) {
++num_positions;
position_range = i + 1;
}
@ -309,7 +297,7 @@ void SanityCheck(const DepGraph<SetType>& depgraph)
assert(position_range >= num_positions);
assert(position_range <= SetType::Size());
// Consistency check between ancestors internally.
for (ClusterIndex i : depgraph.Positions()) {
for (DepGraphIndex i : depgraph.Positions()) {
// Transactions include themselves as ancestors.
assert(depgraph.Ancestors(i)[i]);
// If a is an ancestor of b, then b's ancestors must include all of a's ancestors.
@ -318,8 +306,8 @@ void SanityCheck(const DepGraph<SetType>& depgraph)
}
}
// Consistency check between ancestors and descendants.
for (ClusterIndex i : depgraph.Positions()) {
for (ClusterIndex j : depgraph.Positions()) {
for (DepGraphIndex i : depgraph.Positions()) {
for (DepGraphIndex j : depgraph.Positions()) {
assert(depgraph.Ancestors(i)[j] == depgraph.Descendants(j)[i]);
}
// No transaction is a parent or child of itself.
@ -337,7 +325,7 @@ void SanityCheck(const DepGraph<SetType>& depgraph)
assert((depgraph.Descendants(child) & children).IsSubsetOf(SetType::Singleton(child)));
}
}
if (IsAcyclic(depgraph)) {
if (depgraph.IsAcyclic()) {
// If DepGraph is acyclic, serialize + deserialize must roundtrip.
std::vector<unsigned char> ser;
VectorWriter writer(ser, 0);
@ -360,7 +348,7 @@ void SanityCheck(const DepGraph<SetType>& depgraph)
// In acyclic graphs, the union of parents with parents of parents etc. yields the
// full ancestor set (and similar for children and descendants).
std::vector<SetType> parents(depgraph.PositionRange()), children(depgraph.PositionRange());
for (ClusterIndex i : depgraph.Positions()) {
for (DepGraphIndex i : depgraph.Positions()) {
parents[i] = depgraph.GetReducedParents(i);
children[i] = depgraph.GetReducedChildren(i);
}
@ -392,7 +380,7 @@ void SanityCheck(const DepGraph<SetType>& depgraph)
/** Perform a sanity check on a linearization. */
template<typename SetType>
void SanityCheck(const DepGraph<SetType>& depgraph, Span<const ClusterIndex> linearization)
void SanityCheck(const DepGraph<SetType>& depgraph, Span<const DepGraphIndex> linearization)
{
// Check completeness.
assert(linearization.size() == depgraph.TxCount());

2795
src/txgraph.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

221
src/txgraph.h Normal file
View File

@ -0,0 +1,221 @@
// Copyright (c) The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <compare>
#include <memory>
#include <optional>
#include <stdint.h>
#include <vector>
#include <utility>
#include <util/feefrac.h>
#ifndef BITCOIN_TXGRAPH_H
#define BITCOIN_TXGRAPH_H
static constexpr unsigned MAX_CLUSTER_COUNT_LIMIT{64};
/** Data structure to encapsulate fees, sizes, and dependencies for a set of transactions. */
class TxGraph
{
public:
/** Internal identifier for a transaction within a TxGraph. */
using GraphIndex = uint32_t;
/** Data type used to reference transactions within a TxGraph.
*
* Every transaction within a TxGraph has exactly one corresponding TxGraph::Ref, held by users
* of the class. Destroying the TxGraph::Ref removes the corresponding transaction.
*
* Users of the class can inherit from TxGraph::Ref. If all Refs are inherited this way, the
* Ref* pointers returned by TxGraph functions can be used as this inherited type.
*/
class Ref
{
// Allow TxGraph's GetRefGraph and GetRefIndex to access internals.
friend class TxGraph;
/** Which Graph the Entry lives in. nullptr if this Ref is empty. */
TxGraph* m_graph = nullptr;
/** Index into the Graph's m_entries. Only used if m_graph != nullptr. */
GraphIndex m_index = GraphIndex(-1);
public:
/** Construct an empty Ref. Non-empty Refs can only be created using
* TxGraph::AddTransaction. */
Ref() noexcept = default;
/** Destroy this Ref. If it is not empty, the corresponding transaction is removed (in both
* main and staging, if it exists). */
virtual ~Ref();
// Support moving a Ref.
Ref& operator=(Ref&& other) noexcept;
Ref(Ref&& other) noexcept;
// Do not permit copy constructing or copy assignment. A TxGraph entry can have at most one
// Ref pointing to it.
Ref& operator=(const Ref&) = delete;
Ref(const Ref&) = delete;
};
/** Interface returned by GetBlockBuilder. */
class BlockBuilder
{
protected:
/** The next chunk, in topological order plus feerate, or std::nullopt if done. */
std::optional<std::pair<std::span<Ref*>, FeePerWeight>> m_current_chunk;
/** Make constructor non-public (use TxGraph::GetBlockBuilder()). */
BlockBuilder() noexcept = default;
public:
/** Support safe inheritance. */
virtual ~BlockBuilder() = default;
/** Determine whether there are more transactions to be included. */
explicit operator bool() noexcept { return m_current_chunk.has_value(); }
/** Get the chunk that is currently suggested to be included. */
const std::span<Ref*>& GetCurrentChunk() noexcept { return m_current_chunk->first; }
/** Get the feerate of the currently suggested chunk. */
const FeePerWeight& GetCurrentChunkFeerate() noexcept { return m_current_chunk->second; }
/** Mark the current chunk as included, and progress to the next one. */
virtual void Include() noexcept = 0;
/** Mark the current chunk as skipped, and progress to the next one. */
virtual void Skip() noexcept = 0;
};
protected:
// Allow TxGraph::Ref to call UpdateRef and UnlinkRef.
friend class TxGraph::Ref;
/** Inform the TxGraph implementation that a TxGraph::Ref has moved. */
virtual void UpdateRef(GraphIndex index, Ref& new_location) noexcept = 0;
/** Inform the TxGraph implementation that a TxGraph::Ref was destroyed. */
virtual void UnlinkRef(GraphIndex index) noexcept = 0;
// Allow TxGraph implementations (inheriting from it) to access Ref internals.
static TxGraph*& GetRefGraph(Ref& arg) noexcept { return arg.m_graph; }
static TxGraph* GetRefGraph(const Ref& arg) noexcept { return arg.m_graph; }
static GraphIndex& GetRefIndex(Ref& arg) noexcept { return arg.m_index; }
static GraphIndex GetRefIndex(const Ref& arg) noexcept { return arg.m_index; }
public:
/** Virtual destructor, so inheriting is safe. */
virtual ~TxGraph() = default;
/** Construct a new transaction with the specified feerate, and return a Ref to it.
* If a staging graph exists, the new transaction is only created there. In all further calls,
* only Refs created by AddTransaction() are allowed to be passed to this TxGraph object (or
* empty Ref objects). */
[[nodiscard]] virtual Ref AddTransaction(const FeePerWeight& feerate) noexcept = 0;
/** Remove the specified transaction. If a staging graph exists, the removal only happens
* there. This is a no-op if the transaction was already removed.
*
* TxGraph may internally reorder transaction removals with dependency additions for
* performance reasons. If together with any transaction removal all its descendants, or all
* its ancestors, are removed as well (which is what always happens in realistic scenarios),
* this reordering will not affect the behavior of TxGraph.
*
* As an example, imagine 3 transactions A,B,C where B depends on A. If a dependency of C on B
* is added, and then B is deleted, C will still depend on A. If the deletion of B is reordered
* before the C->B dependency is added, the dependency adding has no effect. If, together with
* the deletion of B also either A or C is deleted, there is no distinction between the
* original order case and the reordered case.
*/
virtual void RemoveTransaction(const Ref& arg) noexcept = 0;
/** Add a dependency between two specified transactions. If a staging graph exists, the
* dependency is only added there. Parent may not be a descendant of child already (but may
* be an ancestor of it already, in which case this is a no-op). If either transaction is
* already removed, this is a no-op. */
virtual void AddDependency(const Ref& parent, const Ref& child) noexcept = 0;
/** Modify the fee of the specified transaction, in both the main graph and the staging
* graph if it exists. Wherever the transaction does not exist (or was removed), this has no
* effect. */
virtual void SetTransactionFee(const Ref& arg, int64_t fee) noexcept = 0;
/** TxGraph is internally lazy, and will not compute many things until they are needed.
* Calling DoWork will compute everything now, so that future operations are fast. This can be
* invoked while oversized. */
virtual void DoWork() noexcept = 0;
/** Create a staging graph (which cannot exist already). This acts as if a full copy of
* the transaction graph is made, upon which further modifications are made. This copy can
* be inspected, and then either discarded, or the main graph can be replaced by it by
* commiting it. */
virtual void StartStaging() noexcept = 0;
/** Discard the existing active staging graph (which must exist). */
virtual void AbortStaging() noexcept = 0;
/** Replace the main graph with the staging graph (which must exist). */
virtual void CommitStaging() noexcept = 0;
/** Check whether a staging graph exists. */
virtual bool HaveStaging() const noexcept = 0;
/** Determine whether arg exists in the graph (i.e., was not removed). If main_only is false
* and a staging graph exists, it is queried; otherwise the main graph is queried. */
virtual bool Exists(const Ref& arg, bool main_only = false) noexcept = 0;
/** Determine whether the graph is oversized (contains a connected component of more than the
* configured maximum cluster count). If main_only is false and a staging graph exists, it is
* queried; otherwise the main graph is queried. Some of the functions below are not available
* for oversized graphs. The mutators above are always available. Removing a transaction by
* destroying its Ref while staging exists will not clear main's oversizedness until staging
* is aborted or committed. */
virtual bool IsOversized(bool main_only = false) noexcept = 0;
/** Get the feerate of the chunk which transaction arg is in the main graph. Returns the empty
* FeePerWeight if arg does not exist in the main graph. The main graph must not be
* oversized. */
virtual FeePerWeight GetMainChunkFeerate(const Ref& arg) noexcept = 0;
/** Get the individual transaction feerate of transaction arg. Returns the empty FeePerWeight
* if arg does not exist in either main or staging. This is available even for oversized
* graphs. */
virtual FeePerWeight GetIndividualFeerate(const Ref& arg) noexcept = 0;
/** Get pointers to all transactions in the connected component ("cluster") which arg is in.
* The transactions will be returned in a topologically-valid order of acceptable quality.
* If main_only is false and a staging graph exists, it is queried; otherwise the main graph
* is queried. The queried graph must not be oversized. Returns {} if arg does not exist in
* the queried graph. */
virtual std::vector<Ref*> GetCluster(const Ref& arg, bool main_only = false) noexcept = 0;
/** Get pointers to all ancestors of the specified transaction. If main_only is false and a
* staging graph exists, it is queried; otherwise the main graph is queried. The queried
* graph must not be oversized. Returns {} if arg does not exist in the queried graph. */
virtual std::vector<Ref*> GetAncestors(const Ref& arg, bool main_only = false) noexcept = 0;
/** Like GetAncestors, but return the Refs for all transactions in the union of the provided
* arguments' ancestors (each transaction is only reported once). */
virtual std::vector<Ref*> GetAncestorsUnion(std::span<const Ref* const> args, bool main_only = false) noexcept = 0;
/** Get pointers to all descendants of the specified transaction. If main_only is false and a
* staging graph exists, it is queried; otherwise the main graph is queried. The queried
* graph must not be oversized. Returns {} if arg does not exist in the queried graph. */
virtual std::vector<Ref*> GetDescendants(const Ref& arg, bool main_only = false) noexcept = 0;
/** Like GetDescendants, but return the Refs for all transactions in the union of the provided
* arguments' descendants (each transaction is only reported once). */
virtual std::vector<Ref*> GetDescendantsUnion(std::span<const Ref* const> args, bool main_only = false) noexcept = 0;
/** Get the total number of transactions in the graph. If main_only is false and a staging
* graph exists, it is queried; otherwise the main graph is queried. This is available even
* for oversized graphs. */
virtual GraphIndex GetTransactionCount(bool main_only = false) noexcept = 0;
/** Compare two transactions according to the total order in the main graph (topological, and
* from high to low chunk feerate). Both transactions must be in the main graph. The main
* graph must not be oversized. */
virtual std::strong_ordering CompareMainOrder(const Ref& a, const Ref& b) noexcept = 0;
/** Count the number of distinct clusters that the specified transactions belong to. If
* main_only is false and a staging graph exists, staging clusters are counted. Otherwise,
* main clusters are counted. Refs that do not exist in the graph are not counted. The
* queried graph must not be oversized. */
virtual GraphIndex CountDistinctClusters(std::span<const Ref* const>, bool main_only = false) noexcept = 0;
/** Get feerate diagrams for both main and staging (which must both exist and not be
* oversized), ignoring unmodified components in both. Use FeeFrac rather than FeePerWeight
* so CompareChunks is usable without type-conversion. */
virtual std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>> GetMainStagingDiagrams() noexcept = 0;
/** Trim all clusters (and would-be clusters) to the TxGraph's cluster count and size
* constraints. Returns the list of all removed transactions, which will always include all
* its own descendants. Applies to staging if it exists, and main otherwise. */
virtual std::vector<Ref*> Trim() noexcept = 0;
/** Construct a block builder, drawing from the main graph, which cannot be oversized. While
* the returned object exists, no mutators on the main graph are allowed. */
virtual std::unique_ptr<BlockBuilder> GetBlockBuilder() noexcept = 0;
/** Get the worst chunk overall in the main graph, i.e., the last chunk that would be returned
* by a BlockBuilder created now. The chunk is returned in reversed order, so every element is
* preceded by all its descendants. If the graph is empty, {} is returned. */
virtual std::pair<std::vector<Ref*>, FeePerWeight> GetWorstMainChunk() noexcept = 0;
/** Perform an internal consistency check on this object. */
virtual void SanityCheck() const = 0;
};
/** Construct a new TxGraph with the specified limit on transactions within a cluster, and the
* specified limit on the sum of transaction sizes within a cluster. max_cluster_count cannot
* exceed MAX_CLUSTER_COUNT_LIMIT. */
std::unique_ptr<TxGraph> MakeTxGraph(unsigned max_cluster_count, uint64_t max_cluster_size) noexcept;
#endif // BITCOIN_TXGRAPH_H

2
src/util/feefrac.cpp
View File

@ -36,7 +36,7 @@ std::partial_ordering CompareChunks(Span<const FeeFrac> chunks0, Span<const FeeF
// Let `P` be the next point on diagram unproc_side, and `A` and `B` the previous and next points
// on the other diagram. We want to know if P lies above or below the line AB. To determine this, we
// compute the slopes of line AB and of line AP, and compare them. These slopes are fee per size,
// and can thus be expressed as FeeFracs.
// and can thus be expressed as FeeRates.
const FeeFrac& point_p = next_point(unproc_side);
const FeeFrac& point_a = prev_point(!unproc_side);

22
src/util/feefrac.h
View File

@ -156,4 +156,26 @@ struct FeeFrac
*/
std::partial_ordering CompareChunks(Span<const FeeFrac> chunks0, Span<const FeeFrac> chunks1);
/** Tagged wrapper around FeeFrac to avoid unit confusion. */
template<typename Tag>
struct FeePerUnit : public FeeFrac
{
// Inherit FeeFrac constructors.
using FeeFrac::FeeFrac;
/** Convert a FeeFrac to a FeePerUnit. */
static FeePerUnit FromFeeFrac(const FeeFrac& feefrac) noexcept
{
return {feefrac.fee, feefrac.size};
}
};
// FeePerUnit instance for satoshi / vbyte.
struct VSizeTag {};
using FeePerVSize = FeePerUnit<VSizeTag>;
// FeePerUnit instance for satoshi / WU.
struct WeightTag {};
using FeePerWeight = FeePerUnit<WeightTag>;
#endif // BITCOIN_UTIL_FEEFRAC_H