highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2026-09-11 21:20:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

txindex: hash key prefixes and pack block positions

Browse Source 
Use a 5-byte salted siphash to key txindex entries,
instead of the full 32-byte txid. Store the block sequence and tx position
after the hash in the key, so an iterator can scan
through any collisions and return the correct tx.

Fall back to the legacy key lookup if the tx is not found.

Co-authored-by: Pieter Wuille <pieter@wuille.net>
Co-authored-by: l0rinc <pap.lorinc@gmail.com>
Co-authored-by: Anthony Towns <aj@erisian.com.au>
This commit is contained in:
Andrew Toth
2026-08-08 12:03:04 -04:00
parent 5a255970fd
commit 004d7c098c
3 changed files with 199 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
src/index/txindex.cpp
View File

@@ -4,7 +4,9 @@
#include <index/txindex.h>
#include <chain.h>
#include <common/args.h>
#include <crypto/siphash.h>
#include <dbwrapper.h>
#include <flatfile.h>
#include <index/base.h>
@@ -14,17 +16,23 @@
#include <node/blockstorage.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <random.h>
#include <serialize.h>
#include <streams.h>
#include <sync.h>
#include <uint256.h>
#include <util/fs.h>
#include <util/log.h>
#include <validation.h>
#include <algorithm>
#include <array>
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <exception>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <utility>
@@ -32,6 +40,18 @@
std::unique_ptr<TxIndex> g_txindex;
namespace {
SipHasher13UJ ReadOrCreateTxidHasher(CDBWrapper& db)
{
std::pair<uint64_t, uint64_t> salt;
if (!db.Read(txindex::DB_TXID_HASH_SALT, salt)) {
FastRandomContext rng{};
salt = {rng.rand64(), rng.rand64()};
db.Write(txindex::DB_TXID_HASH_SALT, salt, /*fSync=*/true);
}
return SipHasher13UJ{salt.first, salt.second};
}
} // namespace
/** Access to the txindex database (indexes/txindex/) */
class TxIndex::DB : public BaseIndex::DB
@@ -42,12 +62,16 @@ public:
/// Write a block of transaction positions to the DB.
void WriteTxs(const interfaces::BlockInfo& block);
/// Used to hash the txid to compute the prefix.
const SipHasher13UJ m_hasher;
CBlockLocator ReadBestBlock() const override;
void WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator) override;
};
TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
BaseIndex::DB(gArgs.GetDataDirNet() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
BaseIndex::DB(gArgs.GetDataDirNet() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe),
m_hasher{ReadOrCreateTxidHasher(*this)}
{}
CBlockLocator TxIndex::DB::ReadBestBlock() const
@@ -67,11 +91,24 @@ void TxIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator)
void TxIndex::DB::WriteTxs(const interfaces::BlockInfo& block)
{
// A block may be submitted again after it was already indexed, e.g. when it
// reconnects after a reorg or is re-processed after an unclean shutdown. It
// keeps its original sequence number, so skip it to avoid duplicate entries.
if (Exists(txindex::BlockHashKey{block.hash})) return;
uint32_t block_seq{0};
Read(txindex::DB_NEXT_BLOCK_SEQ, block_seq);
CDBBatch batch(*this);
CDiskTxPos pos({block.file_number, block.data_pos}, GetSizeOfCompactSize(block.data->vtx.size()));
batch.Write(txindex::BlockHashKey{block.hash}, block_seq);
batch.Write(txindex::BlockSeqKey{block_seq}, block.hash);
batch.Write(txindex::DB_NEXT_BLOCK_SEQ, block_seq + 1);
uint32_t tx_offset_in_block{txindex::BLOCK_HEADER_SIZE + GetSizeOfCompactSize(block.data->vtx.size())};
for (const auto& tx : block.data->vtx) {
batch.Write(std::make_pair(txindex::DB_TXINDEX, tx->GetHash().ToUint256()), pos);
pos.nTxOffset += ::GetSerializeSize(TX_WITH_WITNESS(*tx));
const txindex::DBKey key{txindex::CreateKeyPrefix(m_hasher, tx->GetHash()),
txindex::BlockTxPosition{block_seq, tx_offset_in_block}};
batch.Write(key, txindex::EMPTY_VALUE);
tx_offset_in_block += tx->ComputeTotalSize();
}
WriteBatch(batch);
}
@@ -95,6 +132,67 @@ bool TxIndex::CustomAppend(const interfaces::BlockInfo& block)
BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
std::optional<TxIndexResult> TxIndex::FindTx(const Txid& tx_hash) const
{
struct Candidate {
FlatFilePos tx_position;
uint256 block_hash;
uint32_t block_seq;
//! Whether this candidate's block is currently in the active chain.
//! Active chain candidates are attempted first, so duplicate entries
//! in both active and stale blocks will always return the active block hash.
bool in_active_chain;
};
std::vector<Candidate> candidates;
{
std::unique_ptr<CDBIterator> it{m_db->NewIterator()};
const txindex::TxHashKeyPrefix prefix{txindex::CreateKeyPrefix(m_db->m_hasher, tx_hash)};
txindex::DBKey key{prefix, {}};
for (it->Seek(key); it->Valid() && it->GetKey(key) && key.hash_prefix == prefix; it->Next()) {
uint256 candidate_block_hash;
if (!m_db->Read(txindex::BlockSeqKey{key.pos.block_seq}, candidate_block_hash)) {
LogWarning("Block sequence %u not found for txid %s", key.pos.block_seq, tx_hash.ToString());
continue;
}
LOCK(cs_main);
const CBlockIndex* block_index{m_chainstate->m_blockman.LookupBlockIndex(candidate_block_hash)};
if (!block_index) {
LogWarning("Block index entry %s not found for txid %s", candidate_block_hash.ToString(), tx_hash.ToString());
continue;
}
if (!(block_index->nStatus & BLOCK_HAVE_DATA)) continue;
const FlatFilePos tx_position{block_index->nFile, block_index->nDataPos + key.pos.tx_offset_in_block};
candidates.emplace_back(tx_position, candidate_block_hash, key.pos.block_seq, m_chainstate->m_chain.Contains(*block_index));
}
}
// Prefer active-chain matches, then later-connected blocks.
std::ranges::sort(candidates, std::greater{}, [](const Candidate& c) {
return std::pair{c.in_active_chain, c.block_seq};
});
for (const auto& candidate : candidates) {
AutoFile file{m_chainstate->m_blockman.OpenBlockFile(candidate.tx_position, /*fReadOnly=*/true)};
if (file.IsNull()) {
LogWarning("OpenBlockFile failed for txid %s", tx_hash.ToString());
continue;
}
CTransactionRef tx;
try {
file >> TX_WITH_WITNESS(tx);
} catch (const std::exception& e) {
LogWarning("Deserialize or I/O error - %s", e.what());
continue;
}
if (tx->GetHash() == tx_hash) {
return TxIndexResult{candidate.block_hash, std::move(tx)};
}
}
// Fall back to legacy if no hashed entry matched. This makes misses pay an
// extra lookup, but keeps existing full-txid entries readable after upgrade.
return FindLegacyTx(tx_hash);
}
std::optional<TxIndexResult> TxIndex::FindLegacyTx(const Txid& tx_hash) const
{
CDiskTxPos postx;
if (!m_db->Read(txindex::LegacyTxKey(tx_hash), postx)) {

7
src/index/txindex.h
View File

@@ -27,8 +27,8 @@ struct TxIndexResult {
/**
* TxIndex is used to look up transactions included in the blockchain by hash.
* The index is written to a LevelDB database and records the filesystem
* location of each transaction by transaction hash.
* The index is written to a LevelDB database and records the block sequence
* number and serialized block offset of each transaction by transaction hash.
*/
class TxIndex final : public BaseIndex
{
@@ -40,6 +40,9 @@ private:
bool AllowPrune() const override { return false; }
/// Look up a transaction among the legacy (full-txid) entries.
std::optional<TxIndexResult> FindLegacyTx(const Txid& tx_hash) const;
protected:
bool CustomAppend(const interfaces::BlockInfo& block) override;

92
src/index/txindex_key.h
View File

@@ -5,10 +5,16 @@
#ifndef BITCOIN_INDEX_TXINDEX_KEY_H
#define BITCOIN_INDEX_TXINDEX_KEY_H
#include <consensus/consensus.h>
#include <crypto/siphash.h>
#include <primitives/transaction_identifier.h>
#include <serialize.h>
#include <uint256.h>
#include <array>
#include <cstddef>
#include <cstdint>
#include <ios>
#include <string>
#include <utility>
@@ -16,15 +22,101 @@ namespace txindex {
/*
* Database layout:
*
* ['x', hash prefix, block seq, tx offset] -> (empty)
* ['s', block seq] -> block hash
* ['h', block hash] -> block seq
* ["next_block_seq"] -> next block seq to assign
* ["txid_hash_salt"] -> txid hasher salt
* ["best_block_v2"] -> current sync locator
* ['t', txid] -> legacy CDiskTxPos
* ['B'] -> legacy sync locator
*/
constexpr uint8_t DB_TXINDEX_HASHED{'x'};
constexpr uint8_t DB_BLOCK_SEQ{'s'};
constexpr uint8_t DB_BLOCK_HASH{'h'};
inline const std::string DB_NEXT_BLOCK_SEQ{"next_block_seq"};
inline const std::string DB_TXID_HASH_SALT{"txid_hash_salt"};
inline const std::string DB_BEST_BLOCK_V2{"best_block_v2"};
//! Prefix of a legacy (pre-hashing) txindex row.
constexpr uint8_t DB_TXINDEX{'t'};
//! Empty value of a hashed txindex row, whose position is encoded in its key.
inline constexpr std::array<std::byte, 0> EMPTY_VALUE{};
//! Serialized size of a block header, the offset of the first byte after it.
constexpr uint32_t BLOCK_HEADER_SIZE{80};
//! The location of a transaction: the sequence number of the block that contains it
//! and the transaction's serialized byte offset from the start of that block
//! (including the header), so the on-disk position is simply
//! block_data_pos + tx_offset_in_block.
//!
struct BlockTxPosition {
uint32_t block_seq{0};
uint32_t tx_offset_in_block{0};
friend bool operator==(const BlockTxPosition&, const BlockTxPosition&) = default;
// tx_offset is encoded in 3-byte big-endian integer.
// This can hold up to 16,777,216, which is >4x the maximum 4 million block weight position
static constexpr uint32_t TX_OFFSET_SIZE{3};
static_assert(MAX_BLOCK_SERIALIZED_SIZE <= BigEndianFormatter<TX_OFFSET_SIZE>::MAX);
SERIALIZE_METHODS(BlockTxPosition, obj)
{
READWRITE(VARINT(obj.block_seq),
Using<BigEndianFormatter<TX_OFFSET_SIZE>>(obj.tx_offset_in_block));
}
};
//! Key for looking up the hash of the block with the given sequence number.
struct BlockSeqKey {
uint32_t block_seq{0};
SERIALIZE_METHODS(BlockSeqKey, obj)
{
uint8_t prefix{DB_BLOCK_SEQ};
READWRITE(prefix);
if (ser_action.ForRead() && prefix != DB_BLOCK_SEQ) throw std::ios_base::failure("Invalid format for txindex block seq key");
READWRITE(VARINT(obj.block_seq));
}
};
//! Key for looking up the sequence number assigned to the block with the given hash.
struct BlockHashKey {
uint256 block_hash;
SERIALIZE_METHODS(BlockHashKey, obj)
{
uint8_t prefix{DB_BLOCK_HASH};
READWRITE(prefix);
if (ser_action.ForRead() && prefix != DB_BLOCK_HASH) throw std::ios_base::failure("Invalid format for txindex block hash key");
READWRITE(obj.block_hash);
}
};
constexpr int HASH_PREFIX_SIZE{5};
using TxHashKeyPrefix = uint64_t;
inline TxHashKeyPrefix CreateKeyPrefix(const SipHasher13UJ& hasher, const Txid& txid)
{
return hasher.Hash(txid.ToUint256()) >> (8 * (sizeof(TxHashKeyPrefix) - HASH_PREFIX_SIZE));
}
struct DBKey {
TxHashKeyPrefix hash_prefix{0};
BlockTxPosition pos;
SERIALIZE_METHODS(DBKey, obj)
{
uint8_t prefix{DB_TXINDEX_HASHED};
READWRITE(prefix);
if (ser_action.ForRead() && prefix != DB_TXINDEX_HASHED) throw std::ios_base::failure("Invalid format for txindex DB key");
READWRITE(Using<BigEndianFormatter<HASH_PREFIX_SIZE>>(obj.hash_prefix), obj.pos);
}
};
//! Key of a legacy (pre-hashing) txindex row: the full txid under the 't' prefix.
inline std::pair<uint8_t, uint256> LegacyTxKey(const Txid& txid)
{