highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2026-01-21 07:39:08 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge #10195: Switch chainstate db and cache to per-txout model

Browse Source 
589827975 scripted-diff: various renames for per-utxo consistency (Pieter Wuille)
a5e02bc7f Increase travis unit test timeout (Pieter Wuille)
73de2c1ff Rename CCoinsCacheEntry::coins to coin (Pieter Wuille)
119e552f7 Merge CCoinsViewCache's GetOutputFor and AccessCoin (Pieter Wuille)
580b02309 [MOVEONLY] Move old CCoins class to txdb.cpp (Pieter Wuille)
8b25d2c0c Upgrade from per-tx database to per-txout (Pieter Wuille)
b2af357f3 Reduce reserved memory space for flushing (Pieter Wuille)
41aa5b79a Pack Coin more tightly (Pieter Wuille)
97072d668 Remove unused CCoins methods (Pieter Wuille)
ce23efaa5 Extend coins_tests (Pieter Wuille)
508307968 Switch CCoinsView and chainstate db from per-txid to per-txout (Pieter Wuille)
4ec0d9e79 Refactor GetUTXOStats in preparation for per-COutPoint iteration (Pieter Wuille)
13870b56f Replace CCoins-based CTxMemPool::pruneSpent with isSpent (Pieter Wuille)
05293f3cb Remove ModifyCoins/ModifyNewCoins (Pieter Wuille)
961e48397 Switch tests from ModifyCoins to AddCoin/SpendCoin (Pieter Wuille)
8b3868c1b Switch CScriptCheck to use Coin instead of CCoins (Pieter Wuille)
c87b957a3 Only pass things committed to by tx's witness hash to CScriptCheck (Matt Corallo)
f68cdfe92 Switch from per-tx to per-txout CCoinsViewCache methods in some places (Pieter Wuille)
000391132 Introduce new per-txout CCoinsViewCache functions (Pieter Wuille)
bd83111a0 Optimization: Coin&& to ApplyTxInUndo (Pieter Wuille)
cb2c7fdac Replace CTxInUndo with Coin (Pieter Wuille)
422634e2f Introduce Coin, a single unspent output (Pieter Wuille)
7d991b55d Store/allow tx metadata in all undo records (Pieter Wuille)
c3aa0c119 Report on-disk size in gettxoutsetinfo (Pieter Wuille)
d34242430 Remove/ignore tx version in utxo and undo (Pieter Wuille)
7e0032290 Add specialization of SipHash for 256 + 32 bit data (Pieter Wuille)
e484652fc Introduce CHashVerifier to hash read data (Pieter Wuille)
f54580e7e error() in disconnect for disk corruption, not inconsistency (Pieter Wuille)
e66dbde6d Add SizeEstimate to CDBBatch (Pieter Wuille)

Tree-SHA512: ce1fb1e40c77d38915cd02189fab7a8b125c7f44d425c85579d872c3bede3a437760997907c99d7b3017ced1c2de54b2ac7223d99d83a6658fe5ef61edef1de3
This commit is contained in:
Pieter Wuille
2017-06-01 15:47:58 -07:00
parent 39039b12a7 589827975f
commit 1088b02f0c
31 changed files with 1118 additions and 1064 deletions
Download Patch File Download Diff File Expand all files Collapse all files

237
src/validation.cpp
View File

@@ -268,15 +268,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool
prevheights.resize(tx.vin.size());
for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) {
const CTxIn& txin = tx.vin[txinIndex];
CCoins coins;
if (!viewMemPool.GetCoins(txin.prevout.hash, coins)) {
Coin coin;
if (!viewMemPool.GetCoin(txin.prevout, coin)) {
return error("%s: Missing input", __func__);
}
if (coins.nHeight == MEMPOOL_HEIGHT) {
if (coin.nHeight == MEMPOOL_HEIGHT) {
// Assume all mempool transaction confirm in the next block
prevheights[txinIndex] = tip->nHeight + 1;
} else {
prevheights[txinIndex] = coins.nHeight;
prevheights[txinIndex] = coin.nHeight;
}
}
lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index);
@@ -315,9 +315,9 @@ void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) {
LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired);
}
std::vector<uint256> vNoSpendsRemaining;
std::vector<COutPoint> vNoSpendsRemaining;
pool.TrimToSize(limit, &vNoSpendsRemaining);
BOOST_FOREACH(const uint256& removed, vNoSpendsRemaining)
BOOST_FOREACH(const COutPoint& removed, vNoSpendsRemaining)
pcoinsTip->Uncache(removed);
}
@@ -394,7 +394,7 @@ void UpdateMempoolForReorg(DisconnectedBlockTransactions &disconnectpool, bool f
bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree,
bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced,
bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache)
bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<COutPoint>& coins_to_uncache)
{
const CTransaction& tx = *ptx;
const uint256 hash = tx.GetHash();
@@ -487,29 +487,29 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C
view.SetBackend(viewMemPool);
// do we already have it?
bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(hash);
if (view.HaveCoins(hash)) {
if (!fHadTxInCache)
vHashTxnToUncache.push_back(hash);
return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known");
}
// do all inputs exist?
// Note that this does not check for the presence of actual outputs (see the next check for that),
// and only helps with filling in pfMissingInputs (to determine missing vs spent).
BOOST_FOREACH(const CTxIn txin, tx.vin) {
if (!pcoinsTip->HaveCoinsInCache(txin.prevout.hash))
vHashTxnToUncache.push_back(txin.prevout.hash);
if (!view.HaveCoins(txin.prevout.hash)) {
if (pfMissingInputs)
*pfMissingInputs = true;
return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid()
for (size_t out = 0; out < tx.vout.size(); out++) {
COutPoint outpoint(hash, out);
bool had_coin_in_cache = pcoinsTip->HaveCoinInCache(outpoint);
if (view.HaveCoin(outpoint)) {
if (!had_coin_in_cache) {
coins_to_uncache.push_back(outpoint);
}
return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known");
}
}
// are the actual inputs available?
if (!view.HaveInputs(tx))
return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent");
// do all inputs exist?
BOOST_FOREACH(const CTxIn txin, tx.vin) {
if (!pcoinsTip->HaveCoinInCache(txin.prevout)) {
coins_to_uncache.push_back(txin.prevout);
}
if (!view.HaveCoin(txin.prevout)) {
if (pfMissingInputs) {
*pfMissingInputs = true;
}
return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid()
}
}
// Bring the best block into scope
view.GetBestBlock();
@@ -548,8 +548,8 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C
// during reorgs to ensure COINBASE_MATURITY is still met.
bool fSpendsCoinbase = false;
BOOST_FOREACH(const CTxIn &txin, tx.vin) {
const CCoins *coins = view.AccessCoins(txin.prevout.hash);
if (coins->IsCoinBase()) {
const Coin &coin = view.AccessCoin(txin.prevout);
if (coin.IsCoinBase()) {
fSpendsCoinbase = true;
break;
}
@@ -813,10 +813,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const
bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced,
bool fOverrideMempoolLimit, const CAmount nAbsurdFee)
{
std::vector<uint256> vHashTxToUncache;
bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache);
std::vector<COutPoint> coins_to_uncache;
bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache);
if (!res) {
BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache)
BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache)
pcoinsTip->Uncache(hashTx);
}
// After we've (potentially) uncached entries, ensure our coins cache is still within its size limits
@@ -868,15 +868,8 @@ bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus
}
if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it
int nHeight = -1;
{
const CCoinsViewCache& view = *pcoinsTip;
const CCoins* coins = view.AccessCoins(hash);
if (coins)
nHeight = coins->nHeight;
}
if (nHeight > 0)
pindexSlow = chainActive[nHeight];
const Coin& coin = AccessByTxid(*pcoinsTip, hash);
if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight];
}
if (pindexSlow) {
@@ -1124,24 +1117,12 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund
if (!tx.IsCoinBase()) {
txundo.vprevout.reserve(tx.vin.size());
BOOST_FOREACH(const CTxIn &txin, tx.vin) {
CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash);
unsigned nPos = txin.prevout.n;
if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull())
assert(false);
// mark an outpoint spent, and construct undo information
txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos]));
coins->Spend(nPos);
if (coins->vout.size() == 0) {
CTxInUndo& undo = txundo.vprevout.back();
undo.nHeight = coins->nHeight;
undo.fCoinBase = coins->fCoinBase;
undo.nVersion = coins->nVersion;
}
txundo.vprevout.emplace_back();
inputs.SpendCoin(txin.prevout, &txundo.vprevout.back());
}
}
// add outputs
inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight);
AddCoins(inputs, tx, nHeight);
}
void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight)
@@ -1185,11 +1166,19 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi
if (fScriptChecks) {
for (unsigned int i = 0; i < tx.vin.size(); i++) {
const COutPoint &prevout = tx.vin[i].prevout;
const CCoins* coins = inputs.AccessCoins(prevout.hash);
assert(coins);
const Coin& coin = inputs.AccessCoin(prevout);
assert(!coin.IsSpent());
// We very carefully only pass in things to CScriptCheck which
// are clearly committed to by tx' witness hash. This provides
// a sanity check that our caching is not introducing consensus
// failures through additional data in, eg, the coins being
// spent being checked as a part of CScriptCheck.
const CScript& scriptPubKey = coin.out.scriptPubKey;
const CAmount amount = coin.out.nValue;
// Verify signature
CScriptCheck check(*coins, tx, i, flags, cacheStore, &txdata);
CScriptCheck check(scriptPubKey, amount, tx, i, flags, cacheStore, &txdata);
if (pvChecks) {
pvChecks->push_back(CScriptCheck());
check.swap(pvChecks->back());
@@ -1201,7 +1190,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi
// arguments; if so, don't trigger DoS protection to
// avoid splitting the network between upgraded and
// non-upgraded nodes.
CScriptCheck check2(*coins, tx, i,
CScriptCheck check2(scriptPubKey, amount, tx, i,
flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore, &txdata);
if (check2())
return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError())));
@@ -1260,8 +1249,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin
// Read block
uint256 hashChecksum;
CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data
try {
filein >> blockundo;
verifier << hashBlock;
verifier >> blockundo;
filein >> hashChecksum;
}
catch (const std::exception& e) {
@@ -1269,10 +1260,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin
}
// Verify checksum
CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION);
hasher << hashBlock;
hasher << blockundo;
if (hashChecksum != hasher.GetHash())
if (hashChecksum != verifier.GetHash())
return error("%s: Checksum mismatch", __func__);
return true;
@@ -1298,39 +1286,6 @@ bool AbortNode(CValidationState& state, const std::string& strMessage, const std
} // anon namespace
/**
* Apply the undo operation of a CTxInUndo to the given chain state.
* @param undo The undo object.
* @param view The coins view to which to apply the changes.
* @param out The out point that corresponds to the tx input.
* @return True on success.
*/
bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out)
{
bool fClean = true;
CCoinsModifier coins = view.ModifyCoins(out.hash);
if (undo.nHeight != 0) {
// undo data contains height: this is the last output of the prevout tx being spent
if (!coins->IsPruned())
fClean = fClean && error("%s: undo data overwriting existing transaction", __func__);
coins->Clear();
coins->fCoinBase = undo.fCoinBase;
coins->nHeight = undo.nHeight;
coins->nVersion = undo.nVersion;
} else {
if (coins->IsPruned())
fClean = fClean && error("%s: undo data adding output to missing transaction", __func__);
}
if (coins->IsAvailable(out.n))
fClean = fClean && error("%s: undo data overwriting existing output", __func__);
if (coins->vout.size() < out.n+1)
coins->vout.resize(out.n+1);
coins->vout[out.n] = undo.txout;
return fClean;
}
enum DisconnectResult
{
DISCONNECT_OK, // All good.
@@ -1338,6 +1293,36 @@ enum DisconnectResult
DISCONNECT_FAILED // Something else went wrong.
};
/**
* Restore the UTXO in a Coin at a given COutPoint
* @param undo The Coin to be restored.
* @param view The coins view to which to apply the changes.
* @param out The out point that corresponds to the tx input.
* @return A DisconnectResult as an int
*/
int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out)
{
bool fClean = true;
if (view.HaveCoin(out)) fClean = false; // overwriting transaction output
if (undo.nHeight == 0) {
// Missing undo metadata (height and coinbase). Older versions included this
// information only in undo records for the last spend of a transactions'
// outputs. This implies that it must be present for some other output of the same tx.
const Coin& alternate = AccessByTxid(view, out.hash);
if (!alternate.IsSpent()) {
undo.nHeight = alternate.nHeight;
undo.fCoinBase = alternate.fCoinBase;
} else {
return DISCONNECT_FAILED; // adding output for transaction without known metadata
}
}
view.AddCoin(out, std::move(undo), undo.fCoinBase);
return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN;
}
/** Undo the effects of this block (with given index) on the UTXO set represented by coins.
* When UNCLEAN or FAILED is returned, view is left in an indeterminate state. */
static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view)
@@ -1369,36 +1354,31 @@ static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex*
// Check that all outputs are available and match the outputs in the block itself
// exactly.
{
CCoinsModifier outs = view.ModifyCoins(hash);
outs->ClearUnspendable();
CCoins outsBlock(tx, pindex->nHeight);
// The CCoins serialization does not serialize negative numbers.
// No network rules currently depend on the version here, so an inconsistency is harmless
// but it must be corrected before txout nversion ever influences a network rule.
if (outsBlock.nVersion < 0)
outs->nVersion = outsBlock.nVersion;
if (*outs != outsBlock)
fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted");
// remove outputs
outs->Clear();
for (size_t o = 0; o < tx.vout.size(); o++) {
if (!tx.vout[o].scriptPubKey.IsUnspendable()) {
COutPoint out(hash, o);
Coin coin;
view.SpendCoin(out, &coin);
if (tx.vout[o] != coin.out) {
fClean = false; // transaction output mismatch
}
}
}
// restore inputs
if (i > 0) { // not coinbases
const CTxUndo &txundo = blockUndo.vtxundo[i-1];
CTxUndo &txundo = blockUndo.vtxundo[i-1];
if (txundo.vprevout.size() != tx.vin.size()) {
error("DisconnectBlock(): transaction and undo data inconsistent");
return DISCONNECT_FAILED;
}
for (unsigned int j = tx.vin.size(); j-- > 0;) {
const COutPoint &out = tx.vin[j].prevout;
const CTxInUndo &undo = txundo.vprevout[j];
if (!ApplyTxInUndo(undo, view, out))
fClean = false;
int res = ApplyTxInUndo(std::move(txundo.vprevout[j]), view, out);
if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED;
fClean = fClean && res != DISCONNECT_UNCLEAN;
}
// At this point, all of txundo.vprevout should have been moved out.
}
}
@@ -1579,10 +1559,12 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd
if (fEnforceBIP30) {
for (const auto& tx : block.vtx) {
const CCoins* coins = view.AccessCoins(tx->GetHash());
if (coins && !coins->IsPruned())
return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"),
REJECT_INVALID, "bad-txns-BIP30");
for (size_t o = 0; o < tx->vout.size(); o++) {
if (view.HaveCoin(COutPoint(tx->GetHash(), o))) {
return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"),
REJECT_INVALID, "bad-txns-BIP30");
}
}
}
}
@@ -1649,7 +1631,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd
// be in ConnectBlock because they require the UTXO set
prevheights.resize(tx.vin.size());
for (size_t j = 0; j < tx.vin.size(); j++) {
prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight;
prevheights[j] = view.AccessCoin(tx.vin[j].prevout).nHeight;
}
if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) {
@@ -1787,9 +1769,8 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n
int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000;
int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR;
int64_t nTotalSpace = nCoinCacheUsage + std::max<int64_t>(nMempoolSizeMax - nMempoolUsage, 0);
// The cache is large and we're within 10% and 200 MiB or 50% and 50MiB of the limit, but we have time now (not in the middle of a block processing).
bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::min(std::max(nTotalSpace / 2, nTotalSpace - MIN_BLOCK_COINSDB_USAGE * 1024 * 1024),
std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024));
// The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing).
bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024);
// The cache is over the limit, we have to write now.
bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace;
// It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash.
@@ -1830,12 +1811,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n
}
// Flush best chain related state. This can only be done if the blocks / block index write was also done.
if (fDoFullFlush) {
// Typical CCoins structures on disk are around 128 bytes in size.
// Typical Coin structures on disk are around 48 bytes in size.
// Pushing a new one to the database can cause it to be written
// twice (once in the log, and once in the tables). This is already
// an overestimation, as most will delete an existing entry or
// overwrite one. Still, use a conservative safety factor of 2.
if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize()))
if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize()))
return state.Error("out of disk space");
// Flush the chainstate (which may refer to block index entries).
if (!pcoinsTip->Flush())
@@ -1917,7 +1898,7 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) {
DoWarning(strWarning);
}
}
LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__,
LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__,
chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion,
log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx,
DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()),