diff --git a/src/test/fuzz/txgraph.cpp b/src/test/fuzz/txgraph.cpp index caba743738f..09fc85bfe2f 100644 --- a/src/test/fuzz/txgraph.cpp +++ b/src/test/fuzz/txgraph.cpp @@ -269,6 +269,24 @@ FUZZ_TARGET(txgraph) sims.reserve(2); sims.emplace_back(max_count); + /** Struct encapsulating information about a BlockBuilder that's currently live. */ + struct BlockBuilderData + { + /** BlockBuilder object from real. */ + std::unique_ptr builder; + /** The set of transactions marked as included in *builder. */ + SimTxGraph::SetType included; + /** The set of transactions marked as included or skipped in *builder. */ + SimTxGraph::SetType done; + /** The last chunk feerate returned by *builder. IsEmpty() if none yet. */ + FeePerWeight last_feerate; + + BlockBuilderData(std::unique_ptr builder_in) : builder(std::move(builder_in)) {} + }; + + /** Currently active block builders. */ + std::vector block_builders; + /** Function to pick any Ref (for either sim in sims: from sim.simmap or sim.removed, or the * empty Ref). */ auto pick_fn = [&]() noexcept -> TxGraph::Ref* { @@ -342,6 +360,8 @@ FUZZ_TARGET(txgraph) LIMITED_WHILE(provider.remaining_bytes() > 0, 200) { // Read a one-byte command. int command = provider.ConsumeIntegral(); + int orig_command = command; + // Treat the lowest bit of a command as a flag (which selects a variant of some of the // operations), and the second-lowest bit as a way of selecting main vs. staging, and leave // the rest of the bits in command. @@ -349,6 +369,11 @@ FUZZ_TARGET(txgraph) bool use_main = command & 2; command >>= 2; + /** Use the bottom 2 bits of command to select an entry in the block_builders vector (if + * any). These use the same bits as alt/use_main, so don't use those in actions below + * where builder_idx is used as well. */ + int builder_idx = block_builders.empty() ? -1 : int((orig_command & 3) % block_builders.size()); + // Provide convenient aliases for the top simulated graph (main, or staging if it exists), // one for the simulated graph selected based on use_main (for operations that can operate // on both graphs), and one that always refers to the main graph. @@ -359,7 +384,7 @@ FUZZ_TARGET(txgraph) // Keep decrementing command for each applicable operation, until one is hit. Multiple // iterations may be necessary. while (true) { - if (top_sim.GetTransactionCount() < SimTxGraph::MAX_TRANSACTIONS && command-- == 0) { + if ((block_builders.empty() || sims.size() > 1) && top_sim.GetTransactionCount() < SimTxGraph::MAX_TRANSACTIONS && command-- == 0) { // AddTransaction. int64_t fee; int32_t size; @@ -381,7 +406,7 @@ FUZZ_TARGET(txgraph) // Move it in place. *ref_loc = std::move(ref); break; - } else if (top_sim.GetTransactionCount() + top_sim.removed.size() > 1 && command-- == 0) { + } else if ((block_builders.empty() || sims.size() > 1) && top_sim.GetTransactionCount() + top_sim.removed.size() > 1 && command-- == 0) { // AddDependency. auto par = pick_fn(); auto chl = pick_fn(); @@ -395,7 +420,7 @@ FUZZ_TARGET(txgraph) top_sim.AddDependency(par, chl); real->AddDependency(*par, *chl); break; - } else if (top_sim.removed.size() < 100 && command-- == 0) { + } else if ((block_builders.empty() || sims.size() > 1) && top_sim.removed.size() < 100 && command-- == 0) { // RemoveTransaction. Either all its ancestors or all its descendants are also // removed (if any), to make sure TxGraph's reordering of removals and dependencies // has no effect. @@ -425,7 +450,7 @@ FUZZ_TARGET(txgraph) } sel_sim.removed.pop_back(); break; - } else if (command-- == 0) { + } else if (block_builders.empty() && command-- == 0) { // ~Ref (of any transaction). std::vector to_destroy; to_destroy.push_back(pick_fn()); @@ -447,7 +472,7 @@ FUZZ_TARGET(txgraph) } } break; - } else if (command-- == 0) { + } else if (block_builders.empty() && command-- == 0) { // SetTransactionFee. int64_t fee; if (alt) { @@ -578,7 +603,7 @@ FUZZ_TARGET(txgraph) sims.back().modified = SimTxGraph::SetType{}; real->StartStaging(); break; - } else if (sims.size() > 1 && command-- == 0) { + } else if (block_builders.empty() && sims.size() > 1 && command-- == 0) { // CommitStaging. real->CommitStaging(); sims.erase(sims.begin()); @@ -664,6 +689,65 @@ FUZZ_TARGET(txgraph) assert(FeeRateCompare(real_staged_diagram[i], real_staged_diagram[i - 1]) <= 0); } break; + } else if (block_builders.size() < 4 && !main_sim.IsOversized() && command-- == 0) { + // GetBlockBuilder. + block_builders.emplace_back(real->GetBlockBuilder()); + break; + } else if (!block_builders.empty() && command-- == 0) { + // ~BlockBuilder. + block_builders.erase(block_builders.begin() + builder_idx); + break; + } else if (!block_builders.empty() && command-- == 0) { + // BlockBuilder::GetCurrentChunk, followed by Include/Skip. + auto& builder_data = block_builders[builder_idx]; + auto new_included = builder_data.included; + auto new_done = builder_data.done; + auto chunk = builder_data.builder->GetCurrentChunk(); + if (chunk) { + // Chunk feerates must be monotonously decreasing. + if (!builder_data.last_feerate.IsEmpty()) { + assert(!(chunk->second >> builder_data.last_feerate)); + } + builder_data.last_feerate = chunk->second; + // Verify the contents of GetCurrentChunk. + FeePerWeight sum_feerate; + for (TxGraph::Ref* ref : chunk->first) { + // Each transaction in the chunk must exist in the main graph. + auto simpos = main_sim.Find(ref); + assert(simpos != SimTxGraph::MISSING); + // Verify the claimed chunk feerate. + sum_feerate += main_sim.graph.FeeRate(simpos); + // Make sure no transaction is reported twice. + assert(!new_done[simpos]); + new_done.Set(simpos); + // The concatenation of all included transactions must be topologically valid. + new_included.Set(simpos); + assert(main_sim.graph.Ancestors(simpos).IsSubsetOf(new_included)); + } + assert(sum_feerate == chunk->second); + } else { + // When we reach the end, if nothing was skipped, the entire graph should have + // been reported. + if (builder_data.done == builder_data.included) { + assert(builder_data.done.Count() == main_sim.GetTransactionCount()); + } + } + // Possibly invoke GetCurrentChunk() again, which should give the same result. + if ((orig_command % 7) >= 5) { + auto chunk2 = builder_data.builder->GetCurrentChunk(); + assert(chunk == chunk2); + } + // Skip or include. + if ((orig_command % 5) >= 3) { + // Skip. + builder_data.builder->Skip(); + } else { + // Include. + builder_data.builder->Include(); + builder_data.included = new_included; + } + builder_data.done = new_done; + break; } } } @@ -718,6 +802,28 @@ FUZZ_TARGET(txgraph) } } + // The same order should be obtained through a BlockBuilder as implied by CompareMainOrder, + // if nothing is skipped. + auto builder = real->GetBlockBuilder(); + std::vector vec_builder; + while (auto chunk = builder->GetCurrentChunk()) { + FeePerWeight sum; + for (TxGraph::Ref* ref : chunk->first) { + // The reported chunk feerate must match the chunk feerate obtained by asking + // it for each of the chunk's transactions individually. + assert(real->GetMainChunkFeerate(*ref) == chunk->second); + // Verify the chunk feerate matches the sum of the reported individual feerates. + sum += real->GetIndividualFeerate(*ref); + // Chunks must contain transactions that exist in the graph. + auto simpos = sims[0].Find(ref); + assert(simpos != SimTxGraph::MISSING); + vec_builder.push_back(simpos); + } + assert(sum == chunk->second); + builder->Include(); + } + assert(vec_builder == vec1); + // Check that the implied ordering gives rise to a combined diagram that matches the // diagram constructed from the individual cluster linearization chunkings. auto main_real_diagram = get_diagram_fn(/*main_only=*/true); @@ -848,6 +954,8 @@ FUZZ_TARGET(txgraph) // Sanity check again (because invoking inspectors may modify internal unobservable state). real->SanityCheck(); + // Kill the block builders. + block_builders.clear(); // Kill the TxGraph object. real.reset(); // Kill the simulated graphs, with all remaining Refs in it. If any, this verifies that Refs diff --git a/src/txgraph.cpp b/src/txgraph.cpp index 8da1556909f..e88af40c25c 100644 --- a/src/txgraph.cpp +++ b/src/txgraph.cpp @@ -191,6 +191,7 @@ public: class TxGraphImpl final : public TxGraph { friend class Cluster; + friend class BlockBuilderImpl; private: /** Internal RNG. */ FastRandomContext m_rng; @@ -319,7 +320,7 @@ private: /** Index of ChunkData objects, indexing the last transaction in each chunk in the main * graph. */ ChunkIndex m_main_chunkindex; - /** Number of index-observing objects in existence. */ + /** Number of index-observing objects in existence (BlockBuilderImpls). */ size_t m_main_chunkindex_observers{0}; /** A Locator that describes whether, where, and in which Cluster an Entry appears. @@ -543,6 +544,8 @@ public: GraphIndex CountDistinctClusters(std::span refs, bool main_only = false) noexcept final; std::pair, std::vector> GetMainStagingDiagrams() noexcept final; + std::unique_ptr GetBlockBuilder() noexcept final; + void SanityCheck() const final; }; @@ -562,6 +565,34 @@ const TxGraphImpl::ClusterSet& TxGraphImpl::GetClusterSet(int level) const noexc return *m_staging_clusterset; } +/** Implementation of the TxGraph::BlockBuilder interface. */ +class BlockBuilderImpl final : public TxGraph::BlockBuilder +{ + /** Which TxGraphImpl this object is doing block building for. It will have its + * m_main_chunkindex_observers incremented as long as this BlockBuilderImpl exists. */ + TxGraphImpl* const m_graph; + /** Clusters which we're not including further transactions from. */ + std::set m_excluded_clusters; + /** Iterator to the current chunk in the chunk index. end() if nothing further remains. */ + TxGraphImpl::ChunkIndex::const_iterator m_cur_iter; + /** Which cluster the current chunk belongs to, so we can exclude further transactions from it + * when that chunk is skipped. */ + Cluster* m_cur_cluster; + + // Move m_cur_iter / m_cur_cluster to the next acceptable chunk. + void Next() noexcept; + +public: + /** Construct a new BlockBuilderImpl to build blocks for the provided graph. */ + BlockBuilderImpl(TxGraphImpl& graph) noexcept; + + // Implement the public interface. + ~BlockBuilderImpl() final; + std::optional, FeePerWeight>> GetCurrentChunk() noexcept final; + void Include() noexcept final; + void Skip() noexcept final; +}; + void TxGraphImpl::ClearLocator(int level, GraphIndex idx) noexcept { auto& entry = m_entries[idx]; @@ -2266,6 +2297,88 @@ void TxGraphImpl::DoWork() noexcept } } +void BlockBuilderImpl::Next() noexcept +{ + // Don't do anything if we're already done. + if (m_cur_iter == m_graph->m_main_chunkindex.end()) return; + while (true) { + // Advance the pointer, and stop if we reach the end. + ++m_cur_iter; + m_cur_cluster = nullptr; + if (m_cur_iter == m_graph->m_main_chunkindex.end()) break; + // Find the cluster pointed to by m_cur_iter. + const auto& chunk_data = *m_cur_iter; + const auto& chunk_end_entry = m_graph->m_entries[chunk_data.m_graph_index]; + m_cur_cluster = chunk_end_entry.m_locator[0].cluster; + // If we previously skipped a chunk from this cluster we cannot include more from it. + if (!m_excluded_clusters.contains(m_cur_cluster)) break; + } +} + +std::optional, FeePerWeight>> BlockBuilderImpl::GetCurrentChunk() noexcept +{ + std::optional, FeePerWeight>> ret; + // Populate the return value if we are not done. + if (m_cur_iter != m_graph->m_main_chunkindex.end()) { + ret.emplace(); + const auto& chunk_data = *m_cur_iter; + const auto& chunk_end_entry = m_graph->m_entries[chunk_data.m_graph_index]; + ret->first.resize(chunk_data.m_chunk_count); + auto start_pos = chunk_end_entry.m_main_lin_index + 1 - chunk_data.m_chunk_count; + Assume(m_cur_cluster); + m_cur_cluster->GetClusterRefs(*m_graph, ret->first, start_pos); + ret->second = chunk_end_entry.m_main_chunk_feerate; + } + return ret; +} + +BlockBuilderImpl::BlockBuilderImpl(TxGraphImpl& graph) noexcept : m_graph(&graph) +{ + // Make sure all clusters in main are up to date, and acceptable. + m_graph->MakeAllAcceptable(0); + // There cannot remain any inapplicable dependencies (only possible if main is oversized). + Assume(m_graph->m_main_clusterset.m_deps_to_add.empty()); + // Remember that this object is observing the graph's index, so that we can detect concurrent + // modifications. + ++m_graph->m_main_chunkindex_observers; + // Find the first chunk. + m_cur_iter = m_graph->m_main_chunkindex.begin(); + m_cur_cluster = nullptr; + if (m_cur_iter != m_graph->m_main_chunkindex.end()) { + // Find the cluster pointed to by m_cur_iter. + const auto& chunk_data = *m_cur_iter; + const auto& chunk_end_entry = m_graph->m_entries[chunk_data.m_graph_index]; + m_cur_cluster = chunk_end_entry.m_locator[0].cluster; + } +} + +BlockBuilderImpl::~BlockBuilderImpl() +{ + Assume(m_graph->m_main_chunkindex_observers > 0); + // Permit modifications to the main graph again after destroying the BlockBuilderImpl. + --m_graph->m_main_chunkindex_observers; +} + +void BlockBuilderImpl::Include() noexcept +{ + // The actual inclusion of the chunk is done by the calling code. All we have to do is switch + // to the next chunk. + Next(); +} + +void BlockBuilderImpl::Skip() noexcept +{ + // When skipping a chunk we need to not include anything more of the cluster, as that could make + // the result topologically invalid. + if (m_cur_cluster != nullptr) m_excluded_clusters.insert(m_cur_cluster); + Next(); +} + +std::unique_ptr TxGraphImpl::GetBlockBuilder() noexcept +{ + return std::make_unique(*this); +} + } // namespace TxGraph::Ref::~Ref() diff --git a/src/txgraph.h b/src/txgraph.h index 05a84680ad0..0aeff5af281 100644 --- a/src/txgraph.h +++ b/src/txgraph.h @@ -3,9 +3,11 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include #include +#include +#include #include +#include #include @@ -168,6 +170,29 @@ public: * usable without type-conversion. */ virtual std::pair, std::vector> GetMainStagingDiagrams() noexcept = 0; + /** Interface returned by GetBlockBuilder. */ + class BlockBuilder + { + protected: + /** Make constructor non-public (use TxGraph::GetBlockBuilder()). */ + BlockBuilder() noexcept = default; + public: + /** Support safe inheritance. */ + virtual ~BlockBuilder() = default; + /** Get the chunk that is currently suggested to be included, plus its feerate, if any. */ + virtual std::optional, FeePerWeight>> GetCurrentChunk() noexcept = 0; + /** Mark the current chunk as included, and progress to the next one. */ + virtual void Include() noexcept = 0; + /** Mark the current chunk as skipped, and progress to the next one. Further chunks from + * the same cluster as the current one will not be reported anymore. */ + virtual void Skip() noexcept = 0; + }; + + /** Construct a block builder, drawing chunks in order, from the main graph, which cannot be + * oversized. While the returned object exists, no mutators on the main graph are allowed. + * The BlockBuilder object must not outlive the TxGraph it was created with. */ + virtual std::unique_ptr GetBlockBuilder() noexcept = 0; + /** Perform an internal consistency check on this object. */ virtual void SanityCheck() const = 0;