diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index b02f502fc38..7babf5adb87 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -4,7 +4,9 @@ #include +#include #include +#include #include #include #include @@ -14,17 +16,23 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include +#include #include #include #include #include +#include +#include #include #include #include @@ -32,6 +40,18 @@ std::unique_ptr g_txindex; +namespace { +SipHasher13UJ ReadOrCreateTxidHasher(CDBWrapper& db) +{ + std::pair salt; + if (!db.Read(txindex::DB_TXID_HASH_SALT, salt)) { + FastRandomContext rng{}; + salt = {rng.rand64(), rng.rand64()}; + db.Write(txindex::DB_TXID_HASH_SALT, salt, /*fSync=*/true); + } + return SipHasher13UJ{salt.first, salt.second}; +} +} // namespace /** Access to the txindex database (indexes/txindex/) */ class TxIndex::DB : public BaseIndex::DB @@ -42,12 +62,16 @@ public: /// Write a block of transaction positions to the DB. void WriteTxs(const interfaces::BlockInfo& block); + /// Used to hash the txid to compute the prefix. + const SipHasher13UJ m_hasher; + CBlockLocator ReadBestBlock() const override; void WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator) override; }; TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) : - BaseIndex::DB(gArgs.GetDataDirNet() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) + BaseIndex::DB(gArgs.GetDataDirNet() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe), + m_hasher{ReadOrCreateTxidHasher(*this)} {} CBlockLocator TxIndex::DB::ReadBestBlock() const @@ -67,11 +91,24 @@ void TxIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator) void TxIndex::DB::WriteTxs(const interfaces::BlockInfo& block) { + // A block may be submitted again after it was already indexed, e.g. when it + // reconnects after a reorg or is re-processed after an unclean shutdown. It + // keeps its original sequence number, so skip it to avoid duplicate entries. + if (Exists(txindex::BlockHashKey{block.hash})) return; + + uint32_t block_seq{0}; + Read(txindex::DB_NEXT_BLOCK_SEQ, block_seq); + CDBBatch batch(*this); - CDiskTxPos pos({block.file_number, block.data_pos}, GetSizeOfCompactSize(block.data->vtx.size())); + batch.Write(txindex::BlockHashKey{block.hash}, block_seq); + batch.Write(txindex::BlockSeqKey{block_seq}, block.hash); + batch.Write(txindex::DB_NEXT_BLOCK_SEQ, block_seq + 1); + uint32_t tx_offset_in_block{txindex::BLOCK_HEADER_SIZE + GetSizeOfCompactSize(block.data->vtx.size())}; for (const auto& tx : block.data->vtx) { - batch.Write(std::make_pair(txindex::DB_TXINDEX, tx->GetHash().ToUint256()), pos); - pos.nTxOffset += ::GetSerializeSize(TX_WITH_WITNESS(*tx)); + const txindex::DBKey key{txindex::CreateKeyPrefix(m_hasher, tx->GetHash()), + txindex::BlockTxPosition{block_seq, tx_offset_in_block}}; + batch.Write(key, txindex::EMPTY_VALUE); + tx_offset_in_block += tx->ComputeTotalSize(); } WriteBatch(batch); } @@ -95,6 +132,67 @@ bool TxIndex::CustomAppend(const interfaces::BlockInfo& block) BaseIndex::DB& TxIndex::GetDB() const { return *m_db; } std::optional TxIndex::FindTx(const Txid& tx_hash) const +{ + struct Candidate { + FlatFilePos tx_position; + uint256 block_hash; + uint32_t block_seq; + //! Whether this candidate's block is currently in the active chain. + //! Active chain candidates are attempted first, so duplicate entries + //! in both active and stale blocks will always return the active block hash. + bool in_active_chain; + }; + std::vector candidates; + { + std::unique_ptr it{m_db->NewIterator()}; + const txindex::TxHashKeyPrefix prefix{txindex::CreateKeyPrefix(m_db->m_hasher, tx_hash)}; + txindex::DBKey key{prefix, {}}; + for (it->Seek(key); it->Valid() && it->GetKey(key) && key.hash_prefix == prefix; it->Next()) { + uint256 candidate_block_hash; + if (!m_db->Read(txindex::BlockSeqKey{key.pos.block_seq}, candidate_block_hash)) { + LogWarning("Block sequence %u not found for txid %s", key.pos.block_seq, tx_hash.ToString()); + continue; + } + LOCK(cs_main); + const CBlockIndex* block_index{m_chainstate->m_blockman.LookupBlockIndex(candidate_block_hash)}; + if (!block_index) { + LogWarning("Block index entry %s not found for txid %s", candidate_block_hash.ToString(), tx_hash.ToString()); + continue; + } + if (!(block_index->nStatus & BLOCK_HAVE_DATA)) continue; + const FlatFilePos tx_position{block_index->nFile, block_index->nDataPos + key.pos.tx_offset_in_block}; + candidates.emplace_back(tx_position, candidate_block_hash, key.pos.block_seq, m_chainstate->m_chain.Contains(*block_index)); + } + } + + // Prefer active-chain matches, then later-connected blocks. + std::ranges::sort(candidates, std::greater{}, [](const Candidate& c) { + return std::pair{c.in_active_chain, c.block_seq}; + }); + + for (const auto& candidate : candidates) { + AutoFile file{m_chainstate->m_blockman.OpenBlockFile(candidate.tx_position, /*fReadOnly=*/true)}; + if (file.IsNull()) { + LogWarning("OpenBlockFile failed for txid %s", tx_hash.ToString()); + continue; + } + CTransactionRef tx; + try { + file >> TX_WITH_WITNESS(tx); + } catch (const std::exception& e) { + LogWarning("Deserialize or I/O error - %s", e.what()); + continue; + } + if (tx->GetHash() == tx_hash) { + return TxIndexResult{candidate.block_hash, std::move(tx)}; + } + } + // Fall back to legacy if no hashed entry matched. This makes misses pay an + // extra lookup, but keeps existing full-txid entries readable after upgrade. + return FindLegacyTx(tx_hash); +} + +std::optional TxIndex::FindLegacyTx(const Txid& tx_hash) const { CDiskTxPos postx; if (!m_db->Read(txindex::LegacyTxKey(tx_hash), postx)) { diff --git a/src/index/txindex.h b/src/index/txindex.h index b4648817443..618f3bf0f07 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -27,8 +27,8 @@ struct TxIndexResult { /** * TxIndex is used to look up transactions included in the blockchain by hash. - * The index is written to a LevelDB database and records the filesystem - * location of each transaction by transaction hash. + * The index is written to a LevelDB database and records the block sequence + * number and serialized block offset of each transaction by transaction hash. */ class TxIndex final : public BaseIndex { @@ -40,6 +40,9 @@ private: bool AllowPrune() const override { return false; } + /// Look up a transaction among the legacy (full-txid) entries. + std::optional FindLegacyTx(const Txid& tx_hash) const; + protected: bool CustomAppend(const interfaces::BlockInfo& block) override; diff --git a/src/index/txindex_key.h b/src/index/txindex_key.h index 395092e49ad..6ad9b81ecd6 100644 --- a/src/index/txindex_key.h +++ b/src/index/txindex_key.h @@ -5,10 +5,16 @@ #ifndef BITCOIN_INDEX_TXINDEX_KEY_H #define BITCOIN_INDEX_TXINDEX_KEY_H +#include +#include #include +#include #include +#include +#include #include +#include #include #include @@ -16,15 +22,101 @@ namespace txindex { /* * Database layout: * + * ['x', hash prefix, block seq, tx offset] -> (empty) + * ['s', block seq] -> block hash + * ['h', block hash] -> block seq + * ["next_block_seq"] -> next block seq to assign + * ["txid_hash_salt"] -> txid hasher salt * ["best_block_v2"] -> current sync locator * ['t', txid] -> legacy CDiskTxPos * ['B'] -> legacy sync locator */ +constexpr uint8_t DB_TXINDEX_HASHED{'x'}; +constexpr uint8_t DB_BLOCK_SEQ{'s'}; +constexpr uint8_t DB_BLOCK_HASH{'h'}; +inline const std::string DB_NEXT_BLOCK_SEQ{"next_block_seq"}; +inline const std::string DB_TXID_HASH_SALT{"txid_hash_salt"}; inline const std::string DB_BEST_BLOCK_V2{"best_block_v2"}; //! Prefix of a legacy (pre-hashing) txindex row. constexpr uint8_t DB_TXINDEX{'t'}; +//! Empty value of a hashed txindex row, whose position is encoded in its key. +inline constexpr std::array EMPTY_VALUE{}; + +//! Serialized size of a block header, the offset of the first byte after it. +constexpr uint32_t BLOCK_HEADER_SIZE{80}; + +//! The location of a transaction: the sequence number of the block that contains it +//! and the transaction's serialized byte offset from the start of that block +//! (including the header), so the on-disk position is simply +//! block_data_pos + tx_offset_in_block. +//! +struct BlockTxPosition { + uint32_t block_seq{0}; + uint32_t tx_offset_in_block{0}; + + friend bool operator==(const BlockTxPosition&, const BlockTxPosition&) = default; + + // tx_offset is encoded in 3-byte big-endian integer. + // This can hold up to 16,777,216, which is >4x the maximum 4 million block weight position + static constexpr uint32_t TX_OFFSET_SIZE{3}; + static_assert(MAX_BLOCK_SERIALIZED_SIZE <= BigEndianFormatter::MAX); + + SERIALIZE_METHODS(BlockTxPosition, obj) + { + READWRITE(VARINT(obj.block_seq), + Using>(obj.tx_offset_in_block)); + } +}; + +//! Key for looking up the hash of the block with the given sequence number. +struct BlockSeqKey { + uint32_t block_seq{0}; + + SERIALIZE_METHODS(BlockSeqKey, obj) + { + uint8_t prefix{DB_BLOCK_SEQ}; + READWRITE(prefix); + if (ser_action.ForRead() && prefix != DB_BLOCK_SEQ) throw std::ios_base::failure("Invalid format for txindex block seq key"); + READWRITE(VARINT(obj.block_seq)); + } +}; + +//! Key for looking up the sequence number assigned to the block with the given hash. +struct BlockHashKey { + uint256 block_hash; + + SERIALIZE_METHODS(BlockHashKey, obj) + { + uint8_t prefix{DB_BLOCK_HASH}; + READWRITE(prefix); + if (ser_action.ForRead() && prefix != DB_BLOCK_HASH) throw std::ios_base::failure("Invalid format for txindex block hash key"); + READWRITE(obj.block_hash); + } +}; + +constexpr int HASH_PREFIX_SIZE{5}; +using TxHashKeyPrefix = uint64_t; + +inline TxHashKeyPrefix CreateKeyPrefix(const SipHasher13UJ& hasher, const Txid& txid) +{ + return hasher.Hash(txid.ToUint256()) >> (8 * (sizeof(TxHashKeyPrefix) - HASH_PREFIX_SIZE)); +} + +struct DBKey { + TxHashKeyPrefix hash_prefix{0}; + BlockTxPosition pos; + + SERIALIZE_METHODS(DBKey, obj) + { + uint8_t prefix{DB_TXINDEX_HASHED}; + READWRITE(prefix); + if (ser_action.ForRead() && prefix != DB_TXINDEX_HASHED) throw std::ios_base::failure("Invalid format for txindex DB key"); + READWRITE(Using>(obj.hash_prefix), obj.pos); + } +}; + //! Key of a legacy (pre-hashing) txindex row: the full txid under the 't' prefix. inline std::pair LegacyTxKey(const Txid& txid) {