From cdaf42797fc708d6b150d8e0745619bbf7451ad6 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 18 Jul 2024 16:44:00 +0800 Subject: [PATCH 1/7] Add fee to Cpfp API --- backend/src/api/bitcoin/bitcoin.routes.ts | 1 + backend/src/api/cpfp.ts | 2 ++ backend/src/mempool.interfaces.ts | 1 + 3 files changed, 4 insertions(+) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 742ffe242..cca920afc 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -160,6 +160,7 @@ class BitcoinRoutes { descendants: tx.descendants || null, effectiveFeePerVsize: tx.effectiveFeePerVsize || null, sigops: tx.sigops, + fee: tx.fee, adjustedVsize: tx.adjustedVsize, acceleration: tx.acceleration, acceleratedBy: tx.acceleratedBy || undefined, diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts index 604c1b3c9..eaae8224c 100644 --- a/backend/src/api/cpfp.ts +++ b/backend/src/api/cpfp.ts @@ -32,6 +32,7 @@ export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: descendants: tx.descendants || [], effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize, sigops: tx.sigops, + fee: tx.fee, adjustedVsize: tx.adjustedVsize, acceleration: tx.acceleration }; @@ -70,6 +71,7 @@ export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: descendants: tx.descendants || [], effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize, sigops: tx.sigops, + fee: tx.fee, adjustedVsize: tx.adjustedVsize, acceleration: tx.acceleration }; diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 5e8026d15..0ad60f4b9 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -223,6 +223,7 @@ export interface CpfpInfo { sigops?: number; adjustedVsize?: number, acceleration?: boolean, + fee?: number; } export interface TransactionStripped { From 27374bd13156997532745a72c86dc2a94ef62c24 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 4 Apr 2024 04:28:57 +0000 Subject: [PATCH 2/7] Refactor cpfp & single-block gbt code into mini-miner module --- backend/src/api/acceleration/acceleration.ts | 530 +----------------- backend/src/api/bitcoin/bitcoin.routes.ts | 4 +- backend/src/api/blocks.ts | 9 +- backend/src/api/common.ts | 90 --- backend/src/api/cpfp.ts | 291 ++++------ backend/src/api/mini-miner.ts | 515 +++++++++++++++++ backend/src/api/websocket-handler.ts | 6 +- .../repositories/AccelerationRepository.ts | 3 +- 8 files changed, 650 insertions(+), 798 deletions(-) create mode 100644 backend/src/api/mini-miner.ts diff --git a/backend/src/api/acceleration/acceleration.ts b/backend/src/api/acceleration/acceleration.ts index 2dbaa8b07..f26805ff2 100644 --- a/backend/src/api/acceleration/acceleration.ts +++ b/backend/src/api/acceleration/acceleration.ts @@ -1,15 +1,14 @@ import logger from '../../logger'; import { MempoolTransactionExtended } from '../../mempool.interfaces'; -import { IEsploraApi } from '../bitcoin/esplora-api.interface'; +import { GraphTx, getSameBlockRelatives, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from '../mini-miner'; const BLOCK_WEIGHT_UNITS = 4_000_000; -const BLOCK_SIGOPS = 80_000; const MAX_RELATIVE_GRAPH_SIZE = 200; const BID_BOOST_WINDOW = 40_000; const BID_BOOST_MIN_OFFSET = 10_000; const BID_BOOST_MAX_OFFSET = 400_000; -type Acceleration = { +export type Acceleration = { txid: string; max_bid: number; }; @@ -28,31 +27,6 @@ export interface AccelerationInfo { cost: number; // additional cost to accelerate ((cost + txSummary.effectiveFee) / txSummary.effectiveVsize) >= targetFeeRate } -interface GraphTx { - txid: string; - vsize: number; - weight: number; - fees: { - base: number; // in sats - }; - depends: string[]; - spentby: string[]; -} - -interface MempoolTx extends GraphTx { - ancestorcount: number; - ancestorsize: number; - fees: { // in sats - base: number; - ancestor: number; - }; - - ancestors: Map, - ancestorRate: number; - individualRate: number; - score: number; -} - class AccelerationCosts { /** * Takes a list of accelerations and verbose block data @@ -61,7 +35,7 @@ class AccelerationCosts { * @param accelerationsx * @param verboseBlock */ - public calculateBoostRate(accelerations: Acceleration[], blockTxs: IEsploraApi.Transaction[]): number { + public calculateBoostRate(accelerations: Acceleration[], blockTxs: MempoolTransactionExtended[]): number { // Run GBT ourselves to calculate accurate effective fee rates // the list of transactions comes from a mined block, so we already know everything fits within consensus limits const template = makeBlockTemplate(blockTxs, accelerations, 1, Infinity, Infinity); @@ -170,108 +144,28 @@ class AccelerationCosts { /** * Takes an accelerated mined txid and a target rate * Returns the total vsize, fees and acceleration cost (in sats) of the tx and all same-block ancestors - * - * @param txid - * @param medianFeeRate + * + * @param txid + * @param medianFeeRate */ public getAccelerationInfo(tx: MempoolTransactionExtended, targetFeeRate: number, transactions: MempoolTransactionExtended[]): AccelerationInfo { // Get same-block transaction ancestors - const allRelatives = this.getSameBlockRelatives(tx, transactions); - const relativesMap = this.initializeRelatives(allRelatives); - const rootTx = relativesMap.get(tx.txid) as MempoolTx; + const allRelatives = getSameBlockRelatives(tx, transactions); + const relativesMap = initializeRelatives(allRelatives); + const rootTx = relativesMap.get(tx.txid) as GraphTx; // Calculate cost to boost return this.calculateAccelerationAncestors(rootTx, relativesMap, targetFeeRate); } - /** - * Takes a raw transaction, and builds a graph of same-block relatives, - * and returns as a MempoolTx - * - * @param tx - */ - private getSameBlockRelatives(tx: MempoolTransactionExtended, transactions: MempoolTransactionExtended[]): Map { - const blockTxs = new Map(); // map of txs in this block - const spendMap = new Map(); // map of outpoints to spending txids - for (const tx of transactions) { - blockTxs.set(tx.txid, tx); - for (const vin of tx.vin) { - spendMap.set(`${vin.txid}:${vin.vout}`, tx.txid); - } - } - - const relatives: Map = new Map(); - const stack: string[] = [tx.txid]; - - // build set of same-block ancestors - while (stack.length > 0) { - const nextTxid = stack.pop(); - const nextTx = nextTxid ? blockTxs.get(nextTxid) : null; - if (!nextTx || relatives.has(nextTx.txid)) { - continue; - } - - const mempoolTx = this.convertToGraphTx(nextTx); - - mempoolTx.fees.base = nextTx.fee || 0; - mempoolTx.depends = nextTx.vin.map(vin => vin.txid).filter(inTxid => inTxid && blockTxs.has(inTxid)) as string[]; - mempoolTx.spentby = nextTx.vout.map((vout, index) => spendMap.get(`${nextTx.txid}:${index}`)).filter(outTxid => outTxid && blockTxs.has(outTxid)) as string[]; - - for (const txid of [...mempoolTx.depends, ...mempoolTx.spentby]) { - if (txid) { - stack.push(txid); - } - } - - relatives.set(mempoolTx.txid, mempoolTx); - } - - return relatives; - } - - /** - * Takes a raw transaction and converts it to MempoolTx format - * fee and ancestor data is initialized with dummy/null values - * - * @param tx - */ - private convertToGraphTx(tx: MempoolTransactionExtended): GraphTx { - return { - txid: tx.txid, - vsize: Math.ceil(tx.weight / 4), - weight: tx.weight, - fees: { - base: 0, // dummy - }, - depends: [], // dummy - spentby: [], //dummy - }; - } - - private convertGraphToMempoolTx(tx: GraphTx): MempoolTx { - return { - ...tx, - fees: { - base: tx.fees.base, - ancestor: tx.fees.base, - }, - ancestorcount: 1, - ancestorsize: Math.ceil(tx.weight / 4), - ancestors: new Map(), - ancestorRate: 0, - individualRate: 0, - score: 0, - }; - } - /** * Given a root transaction, a list of in-mempool ancestors, and a target fee rate, * Calculate the minimum set of transactions to fee-bump, their total vsize + fees - * + * * @param tx * @param ancestors */ - private calculateAccelerationAncestors(tx: MempoolTx, relatives: Map, targetFeeRate: number): AccelerationInfo { + private calculateAccelerationAncestors(tx: GraphTx, relatives: Map, targetFeeRate: number): AccelerationInfo { // add root tx to the ancestor map relatives.set(tx.txid, tx); @@ -283,12 +177,12 @@ class AccelerationCosts { }); // Initialize individual & ancestor fee rates - relatives.forEach(entry => this.setAncestorScores(entry)); + relatives.forEach(entry => setAncestorScores(entry)); // Sort by descending ancestor score - let sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator); + let sortedRelatives = Array.from(relatives.values()).sort(mempoolComparator); - let includedInCluster: Map | null = null; + let includedInCluster: Map | null = null; // While highest score >= targetFeeRate let maxIterations = MAX_RELATIVE_GRAPH_SIZE; @@ -297,17 +191,17 @@ class AccelerationCosts { // Grab the highest scoring entry const best = sortedRelatives.shift(); if (best) { - const cluster = new Map(best.ancestors?.entries() || []); + const cluster = new Map(best.ancestors?.entries() || []); if (best.ancestors.has(tx.txid)) { includedInCluster = cluster; } cluster.set(best.txid, best); // Remove this cluster (it already pays over the target rate, so doesn't need to be boosted) // and update scores, ancestor totals and dependencies for the survivors - this.removeAncestors(cluster, relatives); + removeAncestors(cluster, relatives); // re-sort - sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator); + sortedRelatives = Array.from(relatives.values()).sort(mempoolComparator); } } @@ -345,394 +239,6 @@ class AccelerationCosts { nextBlockFee: Math.ceil(tx.ancestorsize * targetFeeRate), }; } - - /** - * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors - * for each transaction. - * - * @param tx - * @param all - */ - private setAncestors(tx: MempoolTx, all: Map, visited: Map>, depth: number = 0): Map { - // sanity check for infinite recursion / too many ancestors (should never happen) - if (depth >= 100) { - logger.warn('acceleration dependency calculation failed: setAncestors reached depth of 100, unable to proceed', `Accelerator`); - throw new Error('invalid_tx_dependencies'); - } - - // initialize the ancestor map for this tx - tx.ancestors = new Map(); - tx.depends.forEach(parentId => { - const parent = all.get(parentId); - if (parent) { - // add the parent - tx.ancestors?.set(parentId, parent); - // check for a cached copy of this parent's ancestors - let ancestors = visited.get(parent.txid); - if (!ancestors) { - // recursively fetch the parent's ancestors - ancestors = this.setAncestors(parent, all, visited, depth + 1); - } - // and add to this tx's map - ancestors.forEach((ancestor, ancestorId) => { - tx.ancestors?.set(ancestorId, ancestor); - }); - } - }); - visited.set(tx.txid, tx.ancestors); - - return tx.ancestors; - } - - /** - * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph - * by running setAncestors on each leaf, and caching intermediate results. - * then initializes ancestor data for each transaction - * - * @param all - */ - private initializeRelatives(all: Map): Map { - const mempoolTxs = new Map(); - all.forEach(entry => { - mempoolTxs.set(entry.txid, this.convertGraphToMempoolTx(entry)); - }); - const visited: Map> = new Map(); - const leaves: MempoolTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0); - for (const leaf of leaves) { - this.setAncestors(leaf, mempoolTxs, visited); - } - mempoolTxs.forEach(entry => { - entry.ancestors?.forEach(ancestor => { - entry.ancestorcount++; - entry.ancestorsize += ancestor.vsize; - entry.fees.ancestor += ancestor.fees.base; - }); - this.setAncestorScores(entry); - }); - return mempoolTxs; - } - - /** - * Remove a cluster of transactions from an in-mempool dependency graph - * and update the survivors' scores and ancestors - * - * @param cluster - * @param ancestors - */ - private removeAncestors(cluster: Map, all: Map): void { - // remove - cluster.forEach(tx => { - all.delete(tx.txid); - }); - - // update survivors - all.forEach(tx => { - cluster.forEach(remove => { - if (tx.ancestors?.has(remove.txid)) { - // remove as dependency - tx.ancestors.delete(remove.txid); - tx.depends = tx.depends.filter(parent => parent !== remove.txid); - // update ancestor sizes and fees - tx.ancestorsize -= remove.vsize; - tx.fees.ancestor -= remove.fees.base; - } - }); - // recalculate fee rates - this.setAncestorScores(tx); - }); - } - - /** - * Take a mempool transaction, and set the fee rates and ancestor score - * - * @param tx - */ - private setAncestorScores(tx: MempoolTx): void { - tx.individualRate = tx.fees.base / tx.vsize; - tx.ancestorRate = tx.fees.ancestor / tx.ancestorsize; - tx.score = Math.min(tx.individualRate, tx.ancestorRate); - } - - // Sort by descending score - private mempoolComparator(a, b): number { - return b.score - a.score; - } } -export default new AccelerationCosts; - -interface TemplateTransaction { - txid: string; - order: number; - weight: number; - adjustedVsize: number; // sigop-adjusted vsize, rounded up to the nearest integer - sigops: number; - fee: number; - feeDelta: number; - ancestors: string[]; - cluster: string[]; - effectiveFeePerVsize: number; -} - -interface MinerTransaction extends TemplateTransaction { - inputs: string[]; - feePerVsize: number; - relativesSet: boolean; - ancestorMap: Map; - children: Set; - ancestorFee: number; - ancestorVsize: number; - ancestorSigops: number; - score: number; - used: boolean; - modified: boolean; - dependencyRate: number; -} - -/* -* Build a block using an approximation of the transaction selection algorithm from Bitcoin Core -* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) -*/ -export function makeBlockTemplate(candidates: IEsploraApi.Transaction[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] { - const auditPool: Map = new Map(); - const mempoolArray: MinerTransaction[] = []; - - candidates.forEach(tx => { - // initializing everything up front helps V8 optimize property access later - const adjustedVsize = Math.ceil(Math.max(tx.weight / 4, 5 * (tx.sigops || 0))); - const feePerVsize = (tx.fee / adjustedVsize); - auditPool.set(tx.txid, { - txid: tx.txid, - order: txidToOrdering(tx.txid), - fee: tx.fee, - feeDelta: 0, - weight: tx.weight, - adjustedVsize, - feePerVsize: feePerVsize, - effectiveFeePerVsize: feePerVsize, - dependencyRate: feePerVsize, - sigops: tx.sigops || 0, - inputs: (tx.vin?.map(vin => vin.txid) || []) as string[], - relativesSet: false, - ancestors: [], - cluster: [], - ancestorMap: new Map(), - children: new Set(), - ancestorFee: 0, - ancestorVsize: 0, - ancestorSigops: 0, - score: 0, - used: false, - modified: false, - }); - mempoolArray.push(auditPool.get(tx.txid) as MinerTransaction); - }); - - // set accelerated effective fee - for (const acceleration of accelerations) { - const tx = auditPool.get(acceleration.txid); - if (tx) { - tx.feeDelta = acceleration.max_bid; - tx.feePerVsize = ((tx.fee + tx.feeDelta) / tx.adjustedVsize); - tx.effectiveFeePerVsize = tx.feePerVsize; - tx.dependencyRate = tx.feePerVsize; - } - } - - // Build relatives graph & calculate ancestor scores - for (const tx of mempoolArray) { - if (!tx.relativesSet) { - setRelatives(tx, auditPool); - } - } - - // Sort by descending ancestor score - mempoolArray.sort(priorityComparator); - - // Build blocks by greedily choosing the highest feerate package - // (i.e. the package rooted in the transaction with the best ancestor score) - const blocks: number[][] = []; - let blockWeight = 0; - let blockSigops = 0; - const transactions: MinerTransaction[] = []; - let modified: MinerTransaction[] = []; - const overflow: MinerTransaction[] = []; - let failures = 0; - while (mempoolArray.length || modified.length) { - // skip invalid transactions - while (mempoolArray[0].used || mempoolArray[0].modified) { - mempoolArray.shift(); - } - - // Select best next package - let nextTx; - const nextPoolTx = mempoolArray[0]; - const nextModifiedTx = modified[0]; - if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) { - nextTx = nextPoolTx; - mempoolArray.shift(); - } else { - modified.shift(); - if (nextModifiedTx) { - nextTx = nextModifiedTx; - } - } - - if (nextTx && !nextTx?.used) { - // Check if the package fits into this block - if (blocks.length >= (maxBlocks - 1) || ((blockWeight + (4 * nextTx.ancestorVsize) < weightLimit) && (blockSigops + nextTx.ancestorSigops <= sigopLimit))) { - const ancestors: MinerTransaction[] = Array.from(nextTx.ancestorMap.values()); - // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) - const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; - const clusterTxids = sortedTxSet.map(tx => tx.txid); - const effectiveFeeRate = Math.min(nextTx.dependencyRate || Infinity, nextTx.ancestorFee / nextTx.ancestorVsize); - const used: MinerTransaction[] = []; - while (sortedTxSet.length) { - const ancestor = sortedTxSet.pop(); - if (!ancestor) { - continue; - } - ancestor.used = true; - ancestor.usedBy = nextTx.txid; - // update this tx with effective fee rate & relatives data - if (ancestor.effectiveFeePerVsize !== effectiveFeeRate) { - ancestor.effectiveFeePerVsize = effectiveFeeRate; - } - ancestor.cluster = clusterTxids; - transactions.push(ancestor); - blockWeight += ancestor.weight; - blockSigops += ancestor.sigops; - used.push(ancestor); - } - - // remove these as valid package ancestors for any descendants remaining in the mempool - if (used.length) { - used.forEach(tx => { - modified = updateDescendants(tx, auditPool, modified, effectiveFeeRate); - }); - } - - failures = 0; - } else { - // hold this package in an overflow list while we check for smaller options - overflow.push(nextTx); - failures++; - } - } - - // this block is full - const exceededPackageTries = failures > 1000 && blockWeight > (weightLimit - 4000); - const queueEmpty = !mempoolArray.length && !modified.length; - - if (exceededPackageTries || queueEmpty) { - break; - } - } - - for (const tx of transactions) { - tx.ancestors = Object.values(tx.ancestorMap); - } - - return transactions; -} - -// traverse in-mempool ancestors -// recursion unavoidable, but should be limited to depth < 25 by mempool policy -function setRelatives( - tx: MinerTransaction, - mempool: Map, -): void { - for (const parent of tx.inputs) { - const parentTx = mempool.get(parent); - if (parentTx && !tx.ancestorMap?.has(parent)) { - tx.ancestorMap.set(parent, parentTx); - parentTx.children.add(tx); - // visit each node only once - if (!parentTx.relativesSet) { - setRelatives(parentTx, mempool); - } - parentTx.ancestorMap.forEach((ancestor) => { - tx.ancestorMap.set(ancestor.txid, ancestor); - }); - } - }; - tx.ancestorFee = (tx.fee + tx.feeDelta); - tx.ancestorVsize = tx.adjustedVsize || 0; - tx.ancestorSigops = tx.sigops || 0; - tx.ancestorMap.forEach((ancestor) => { - tx.ancestorFee += (ancestor.fee + ancestor.feeDelta); - tx.ancestorVsize += ancestor.adjustedVsize; - tx.ancestorSigops += ancestor.sigops; - }); - tx.score = tx.ancestorFee / tx.ancestorVsize; - tx.relativesSet = true; -} - -// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score -// avoids recursion to limit call stack depth -function updateDescendants( - rootTx: MinerTransaction, - mempool: Map, - modified: MinerTransaction[], - clusterRate: number, -): MinerTransaction[] { - const descendantSet: Set = new Set(); - // stack of nodes left to visit - const descendants: MinerTransaction[] = []; - let descendantTx: MinerTransaction | undefined; - rootTx.children.forEach(childTx => { - if (!descendantSet.has(childTx)) { - descendants.push(childTx); - descendantSet.add(childTx); - } - }); - while (descendants.length) { - descendantTx = descendants.pop(); - if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) { - // remove tx as ancestor - descendantTx.ancestorMap.delete(rootTx.txid); - descendantTx.ancestorFee -= (rootTx.fee + rootTx.feeDelta); - descendantTx.ancestorVsize -= rootTx.adjustedVsize; - descendantTx.ancestorSigops -= rootTx.sigops; - descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorVsize; - descendantTx.dependencyRate = descendantTx.dependencyRate ? Math.min(descendantTx.dependencyRate, clusterRate) : clusterRate; - - if (!descendantTx.modified) { - descendantTx.modified = true; - modified.push(descendantTx); - } - - // add this node's children to the stack - descendantTx.children.forEach(childTx => { - // visit each node only once - if (!descendantSet.has(childTx)) { - descendants.push(childTx); - descendantSet.add(childTx); - } - }); - } - } - // return new, resorted modified list - return modified.sort(priorityComparator); -} - -// Used to sort an array of MinerTransactions by descending ancestor score -function priorityComparator(a: MinerTransaction, b: MinerTransaction): number { - if (b.score === a.score) { - // tie-break by txid for stability - return a.order - b.order; - } else { - return b.score - a.score; - } -} - -// returns the most significant 4 bytes of the txid as an integer -function txidToOrdering(txid: string): number { - return parseInt( - txid.substring(62, 64) + - txid.substring(60, 62) + - txid.substring(58, 60) + - txid.substring(56, 58), - 16 - ); -} +export default new AccelerationCosts; \ No newline at end of file diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 742ffe242..69fb5168c 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -19,7 +19,7 @@ import bitcoinClient from './bitcoin-client'; import difficultyAdjustment from '../difficulty-adjustment'; import transactionRepository from '../../repositories/TransactionRepository'; import rbfCache from '../rbf-cache'; -import { calculateCpfp } from '../cpfp'; +import { calculateMempoolTxCpfp } from '../cpfp'; class BitcoinRoutes { public initRoutes(app: Application) { @@ -168,7 +168,7 @@ class BitcoinRoutes { return; } - const cpfpInfo = calculateCpfp(tx, mempool.getMempool()); + const cpfpInfo = calculateMempoolTxCpfp(tx, mempool.getMempool()); res.json(cpfpInfo); return; diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 9cc9233d5..5a1b35547 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -30,6 +30,7 @@ import redisCache from './redis-cache'; import rbfCache from './rbf-cache'; import { calcBitsDifference } from './difficulty-adjustment'; import AccelerationRepository from '../repositories/AccelerationRepository'; +import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp'; class Blocks { private blocks: BlockExtended[] = []; @@ -607,7 +608,7 @@ class Blocks { // fetch transactions txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)) || []; // add CPFP - const cpfpSummary = Common.calculateCpfp(height, txs, true); + const cpfpSummary = calculateFastBlockCpfp(height, txs, true); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions); await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1); @@ -636,7 +637,7 @@ class Blocks { } templateTxs.push(tx || templateTx); } - const cpfpSummary = Common.calculateCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as TransactionExtended[], true); + const cpfpSummary = calculateFastBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as TransactionExtended[], true); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions); const classifiedTxMap: { [txid: string]: TransactionClassified } = {}; @@ -890,7 +891,7 @@ class Blocks { } } - const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions); + const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions); const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions); this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`); @@ -1399,7 +1400,7 @@ class Blocks { } if (transactions?.length != null) { - const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); + const summary = calculateFastBlockCpfp(height, transactions as TransactionExtended[]); await this.$saveCpfp(hash, height, summary); diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 1d3b68541..dffabce0a 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -806,96 +806,6 @@ export class Common { } } - static calculateCpfp(height: number, transactions: TransactionExtended[], saveRelatives: boolean = false): CpfpSummary { - const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block - const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster - let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster - let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root - const txMap: { [txid: string]: TransactionExtended } = {}; - // initialize the txMap - for (const tx of transactions) { - txMap[tx.txid] = tx; - } - // reverse pass to identify CPFP clusters - for (let i = transactions.length - 1; i >= 0; i--) { - const tx = transactions[i]; - if (!ancestors[tx.txid]) { - let totalFee = 0; - let totalVSize = 0; - clusterTxs.forEach(tx => { - totalFee += tx?.fee || 0; - totalVSize += (tx.weight / 4); - }); - const effectiveFeePerVsize = totalFee / totalVSize; - let cluster: CpfpCluster; - if (clusterTxs.length > 1) { - cluster = { - root: clusterTxs[0].txid, - height, - txs: clusterTxs.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }), - effectiveFeePerVsize, - }; - clusters.push(cluster); - } - clusterTxs.forEach(tx => { - txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; - if (cluster) { - clusterMap[tx.txid] = cluster; - } - }); - // reset working vars - clusterTxs = []; - ancestors = {}; - } - clusterTxs.push(tx); - tx.vin.forEach(vin => { - ancestors[vin.txid] = true; - }); - } - // forward pass to enforce ancestor rate caps - for (const tx of transactions) { - let minAncestorRate = tx.effectiveFeePerVsize; - for (const vin of tx.vin) { - if (txMap[vin.txid]?.effectiveFeePerVsize) { - minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize); - } - } - // check rounded values to skip cases with almost identical fees - const roundedMinAncestorRate = Math.ceil(minAncestorRate); - const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize); - if (roundedMinAncestorRate < roundedEffectiveFeeRate) { - tx.effectiveFeePerVsize = minAncestorRate; - if (!clusterMap[tx.txid]) { - // add a single-tx cluster to record the dependent rate - const cluster = { - root: tx.txid, - height, - txs: [{ txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }], - effectiveFeePerVsize: minAncestorRate, - }; - clusterMap[tx.txid] = cluster; - clusters.push(cluster); - } else { - // update the existing cluster with the dependent rate - clusterMap[tx.txid].effectiveFeePerVsize = minAncestorRate; - } - } - } - if (saveRelatives) { - for (const cluster of clusters) { - cluster.txs.forEach((member, index) => { - txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse(); - txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse(); - txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize; - }); - } - } - return { - transactions, - clusters, - }; - } - static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats { const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts index 604c1b3c9..1b0873b7a 100644 --- a/backend/src/api/cpfp.ts +++ b/backend/src/api/cpfp.ts @@ -1,29 +1,109 @@ -import { CpfpInfo, MempoolTransactionExtended } from '../mempool.interfaces'; +import { CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces'; +import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner'; import memPool from './mempool'; const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction -const MAX_GRAPH_SIZE = 50; // the maximum number of in-mempool relatives to consider +const MAX_CLUSTER_ITERATIONS = 100; -interface GraphTx extends MempoolTransactionExtended { - depends: string[]; - spentby: string[]; - ancestorMap: Map; - fees: { - base: number; - ancestor: number; +export function calculateFastBlockCpfp(height: number, transactions: TransactionExtended[], saveRelatives: boolean = false): CpfpSummary { + const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block + const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster + let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster + let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root + const txMap: { [txid: string]: TransactionExtended } = {}; + // initialize the txMap + for (const tx of transactions) { + txMap[tx.txid] = tx; + } + // reverse pass to identify CPFP clusters + for (let i = transactions.length - 1; i >= 0; i--) { + const tx = transactions[i]; + if (!ancestors[tx.txid]) { + let totalFee = 0; + let totalVSize = 0; + clusterTxs.forEach(tx => { + totalFee += tx?.fee || 0; + totalVSize += (tx.weight / 4); + }); + const effectiveFeePerVsize = totalFee / totalVSize; + let cluster: CpfpCluster; + if (clusterTxs.length > 1) { + cluster = { + root: clusterTxs[0].txid, + height, + txs: clusterTxs.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }), + effectiveFeePerVsize, + }; + clusters.push(cluster); + } + clusterTxs.forEach(tx => { + txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; + if (cluster) { + clusterMap[tx.txid] = cluster; + } + }); + // reset working vars + clusterTxs = []; + ancestors = {}; + } + clusterTxs.push(tx); + tx.vin.forEach(vin => { + ancestors[vin.txid] = true; + }); + } + // forward pass to enforce ancestor rate caps + for (const tx of transactions) { + let minAncestorRate = tx.effectiveFeePerVsize; + for (const vin of tx.vin) { + if (txMap[vin.txid]?.effectiveFeePerVsize) { + minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize); + } + } + // check rounded values to skip cases with almost identical fees + const roundedMinAncestorRate = Math.ceil(minAncestorRate); + const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize); + if (roundedMinAncestorRate < roundedEffectiveFeeRate) { + tx.effectiveFeePerVsize = minAncestorRate; + if (!clusterMap[tx.txid]) { + // add a single-tx cluster to record the dependent rate + const cluster = { + root: tx.txid, + height, + txs: [{ txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }], + effectiveFeePerVsize: minAncestorRate, + }; + clusterMap[tx.txid] = cluster; + clusters.push(cluster); + } else { + // update the existing cluster with the dependent rate + clusterMap[tx.txid].effectiveFeePerVsize = minAncestorRate; + } + } + } + if (saveRelatives) { + for (const cluster of clusters) { + cluster.txs.forEach((member, index) => { + txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse(); + txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse(); + txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize; + }); + } + } + return { + transactions, + clusters, }; - ancestorcount: number; - ancestorsize: number; - ancestorRate: number; - individualRate: number; - score: number; +} + +export function calculateGoodBlockCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary { + return calculateFastBlockCpfp(height, transactions, true); } /** * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for * that transaction (and all others in the same cluster) */ -export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo { +export function calculateMempoolTxCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo { if (tx.cpfpUpdated && Date.now() < (tx.cpfpUpdated + CPFP_UPDATE_INTERVAL)) { tx.cpfpDirty = false; return { @@ -38,24 +118,24 @@ export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: } const ancestorMap = new Map(); - const graphTx = mempoolToGraphTx(tx); + const graphTx = convertToGraphTx(tx, memPool.getSpendMap()); ancestorMap.set(tx.txid, graphTx); - const allRelatives = expandRelativesGraph(mempool, ancestorMap); + const allRelatives = expandRelativesGraph(mempool, ancestorMap, memPool.getSpendMap()); const relativesMap = initializeRelatives(allRelatives); const cluster = calculateCpfpCluster(tx.txid, relativesMap); let totalVsize = 0; let totalFee = 0; for (const tx of cluster.values()) { - totalVsize += tx.adjustedVsize; - totalFee += tx.fee; + totalVsize += tx.vsize; + totalFee += tx.fees.base; } const effectiveFeePerVsize = totalFee / totalVsize; for (const tx of cluster.values()) { mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; - mempool[tx.txid].ancestors = Array.from(tx.ancestorMap.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee })); - mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestorMap.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee })); + mempool[tx.txid].ancestors = Array.from(tx.ancestors.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base })); + mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestors.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base })); mempool[tx.txid].bestDescendant = null; mempool[tx.txid].cpfpChecked = true; mempool[tx.txid].cpfpDirty = true; @@ -75,83 +155,6 @@ export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: }; } -function mempoolToGraphTx(tx: MempoolTransactionExtended): GraphTx { - return { - ...tx, - depends: tx.vin.map(v => v.txid), - spentby: tx.vout.map((v, i) => memPool.getFromSpendMap(tx.txid, i)).map(tx => tx?.txid).filter(txid => txid != null) as string[], - ancestorMap: new Map(), - fees: { - base: tx.fee, - ancestor: tx.fee, - }, - ancestorcount: 1, - ancestorsize: tx.adjustedVsize, - ancestorRate: 0, - individualRate: 0, - score: 0, - }; -} - -/** - * Takes a map of transaction ancestors, and expands it into a full graph of up to MAX_GRAPH_SIZE in-mempool relatives - */ -function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map): Map { - const relatives: Map = new Map(); - const stack: GraphTx[] = Array.from(ancestors.values()); - while (stack.length > 0) { - if (relatives.size > MAX_GRAPH_SIZE) { - return relatives; - } - - const nextTx = stack.pop(); - if (!nextTx) { - continue; - } - relatives.set(nextTx.txid, nextTx); - - for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) { - if (relatives.has(relativeTxid)) { - // already processed this tx - continue; - } - let mempoolTx = ancestors.get(relativeTxid); - if (!mempoolTx && mempool[relativeTxid]) { - mempoolTx = mempoolToGraphTx(mempool[relativeTxid]); - } - if (mempoolTx) { - stack.push(mempoolTx); - } - } - } - - return relatives; -} - - /** - * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph - * by running setAncestors on each leaf, and caching intermediate results. - * then initializes ancestor data for each transaction - * - * @param all - */ - function initializeRelatives(mempoolTxs: Map): Map { - const visited: Map> = new Map(); - const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0); - for (const leaf of leaves) { - setAncestors(leaf, mempoolTxs, visited); - } - mempoolTxs.forEach(entry => { - entry.ancestorMap?.forEach(ancestor => { - entry.ancestorcount++; - entry.ancestorsize += ancestor.adjustedVsize; - entry.fees.ancestor += ancestor.fees.base; - }); - setAncestorScores(entry); - }); - return mempoolTxs; -} - /** * Given a root transaction and a list of in-mempool ancestors, * Calculate the CPFP cluster @@ -172,10 +175,10 @@ function calculateCpfpCluster(txid: string, graph: Map): Map(best?.ancestorMap?.entries() || []); - while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestorMap.has(tx.txid)) && maxIterations > 0) { + let bestCluster = new Map(best?.ancestors?.entries() || []); + while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestors.has(tx.txid)) && maxIterations > 0) { maxIterations--; if ((best && best.txid === tx.txid) || (bestCluster && bestCluster.has(tx.txid))) { break; @@ -190,7 +193,7 @@ function calculateCpfpCluster(txid: string, graph: Map): Map(best?.ancestorMap?.entries() || []); + bestCluster = new Map(best?.ancestors?.entries() || []); bestCluster.set(best?.txid, best); } } @@ -199,88 +202,4 @@ function calculateCpfpCluster(txid: string, graph: Map): Map, all: Map): void { - // remove - cluster.forEach(tx => { - all.delete(tx.txid); - }); - - // update survivors - all.forEach(tx => { - cluster.forEach(remove => { - if (tx.ancestorMap?.has(remove.txid)) { - // remove as dependency - tx.ancestorMap.delete(remove.txid); - tx.depends = tx.depends.filter(parent => parent !== remove.txid); - // update ancestor sizes and fees - tx.ancestorsize -= remove.adjustedVsize; - tx.fees.ancestor -= remove.fees.base; - } - }); - // recalculate fee rates - setAncestorScores(tx); - }); -} - -/** - * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors - * for each transaction. - * - * @param tx - * @param all - */ -function setAncestors(tx: GraphTx, all: Map, visited: Map>, depth: number = 0): Map { - // sanity check for infinite recursion / too many ancestors (should never happen) - if (depth > MAX_GRAPH_SIZE) { - return tx.ancestorMap; - } - - // initialize the ancestor map for this tx - tx.ancestorMap = new Map(); - tx.depends.forEach(parentId => { - const parent = all.get(parentId); - if (parent) { - // add the parent - tx.ancestorMap?.set(parentId, parent); - // check for a cached copy of this parent's ancestors - let ancestors = visited.get(parent.txid); - if (!ancestors) { - // recursively fetch the parent's ancestors - ancestors = setAncestors(parent, all, visited, depth + 1); - } - // and add to this tx's map - ancestors.forEach((ancestor, ancestorId) => { - tx.ancestorMap?.set(ancestorId, ancestor); - }); - } - }); - visited.set(tx.txid, tx.ancestorMap); - - return tx.ancestorMap; -} - -/** - * Take a mempool transaction, and set the fee rates and ancestor score - * - * @param tx - */ -function setAncestorScores(tx: GraphTx): GraphTx { - tx.individualRate = (tx.fees.base * 100_000_000) / tx.adjustedVsize; - tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize; - tx.score = Math.min(tx.individualRate, tx.ancestorRate); - return tx; -} - -// Sort by descending score -function mempoolComparator(a: GraphTx, b: GraphTx): number { - return b.score - a.score; } \ No newline at end of file diff --git a/backend/src/api/mini-miner.ts b/backend/src/api/mini-miner.ts new file mode 100644 index 000000000..28d9de3f7 --- /dev/null +++ b/backend/src/api/mini-miner.ts @@ -0,0 +1,515 @@ +import { Acceleration } from './acceleration'; +import { MempoolTransactionExtended } from '../mempool.interfaces'; +import logger from '../logger'; + +const BLOCK_WEIGHT_UNITS = 4_000_000; +const BLOCK_SIGOPS = 80_000; +const MAX_RELATIVE_GRAPH_SIZE = 100; + +export interface GraphTx { + txid: string; + vsize: number; + weight: number; + depends: string[]; + spentby: string[]; + + ancestorcount: number; + ancestorsize: number; + fees: { // in sats + base: number; + ancestor: number; + }; + + ancestors: Map, + ancestorRate: number; + individualRate: number; + score: number; +} + +interface TemplateTransaction { + txid: string; + order: number; + weight: number; + adjustedVsize: number; // sigop-adjusted vsize, rounded up to the nearest integer + sigops: number; + fee: number; + feeDelta: number; + ancestors: string[]; + cluster: string[]; + effectiveFeePerVsize: number; +} + +interface MinerTransaction extends TemplateTransaction { + inputs: string[]; + feePerVsize: number; + relativesSet: boolean; + ancestorMap: Map; + children: Set; + ancestorFee: number; + ancestorVsize: number; + ancestorSigops: number; + score: number; + used: boolean; + modified: boolean; + dependencyRate: number; +} + +/** + * Takes a raw transaction, and builds a graph of same-block relatives, + * and returns as a GraphTx + * + * @param tx + */ +export function getSameBlockRelatives(tx: MempoolTransactionExtended, transactions: MempoolTransactionExtended[]): Map { + const blockTxs = new Map(); // map of txs in this block + const spendMap = new Map(); // map of outpoints to spending txids + for (const tx of transactions) { + blockTxs.set(tx.txid, tx); + for (const vin of tx.vin) { + spendMap.set(`${vin.txid}:${vin.vout}`, tx.txid); + } + } + + const relatives: Map = new Map(); + const stack: string[] = [tx.txid]; + + // build set of same-block ancestors + while (stack.length > 0) { + const nextTxid = stack.pop(); + const nextTx = nextTxid ? blockTxs.get(nextTxid) : null; + if (!nextTx || relatives.has(nextTx.txid)) { + continue; + } + + const mempoolTx = convertToGraphTx(nextTx, spendMap); + + for (const txid of [...mempoolTx.depends, ...mempoolTx.spentby]) { + if (txid) { + stack.push(txid); + } + } + + relatives.set(mempoolTx.txid, mempoolTx); + } + + return relatives; +} + +/** + * Takes a raw transaction and converts it to GraphTx format + * fee and ancestor data is initialized with dummy/null values + * + * @param tx + */ +export function convertToGraphTx(tx: MempoolTransactionExtended, spendMap?: Map): GraphTx { + return { + txid: tx.txid, + vsize: Math.max(tx.sigops * 5, Math.ceil(tx.weight / 4)), + weight: tx.weight, + fees: { + base: tx.fee || 0, + ancestor: tx.fee || 0, + }, + depends: (tx.vin.map(vin => vin.txid).filter(depend => depend) as string[]), + spentby: spendMap ? (tx.vout.map((vout, index) => { const spend = spendMap.get(`${tx.txid}:${index}`); return (spend?.['txid'] || spend); }).filter(spent => spent) as string[]) : [], + + ancestorcount: 1, + ancestorsize: Math.max(tx.sigops * 5, Math.ceil(tx.weight / 4)), + ancestors: new Map(), + ancestorRate: 0, + individualRate: 0, + score: 0, + }; +} + +/** + * Takes a map of transaction ancestors, and expands it into a full graph of up to MAX_GRAPH_SIZE in-mempool relatives + */ +export function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map, spendMap: Map): Map { + const relatives: Map = new Map(); + const stack: GraphTx[] = Array.from(ancestors.values()); + while (stack.length > 0) { + if (relatives.size > MAX_RELATIVE_GRAPH_SIZE) { + return relatives; + } + + const nextTx = stack.pop(); + if (!nextTx) { + continue; + } + relatives.set(nextTx.txid, nextTx); + + for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) { + if (relatives.has(relativeTxid)) { + // already processed this tx + continue; + } + let ancestorTx = ancestors.get(relativeTxid); + if (!ancestorTx && relativeTxid in mempool) { + const mempoolTx = mempool[relativeTxid]; + ancestorTx = convertToGraphTx(mempoolTx, spendMap); + } + if (ancestorTx) { + stack.push(ancestorTx); + } + } + } + + return relatives; +} + +/** + * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors + * for each transaction. + * + * @param tx + * @param all + */ +function setAncestors(tx: GraphTx, all: Map, visited: Map>, depth: number = 0): Map { + // sanity check for infinite recursion / too many ancestors (should never happen) + if (depth > MAX_RELATIVE_GRAPH_SIZE) { + logger.warn('cpfp dependency calculation failed: setAncestors reached depth of 100, unable to proceed'); + return tx.ancestors; + } + + // initialize the ancestor map for this tx + tx.ancestors = new Map(); + tx.depends.forEach(parentId => { + const parent = all.get(parentId); + if (parent) { + // add the parent + tx.ancestors?.set(parentId, parent); + // check for a cached copy of this parent's ancestors + let ancestors = visited.get(parent.txid); + if (!ancestors) { + // recursively fetch the parent's ancestors + ancestors = setAncestors(parent, all, visited, depth + 1); + } + // and add to this tx's map + ancestors.forEach((ancestor, ancestorId) => { + tx.ancestors?.set(ancestorId, ancestor); + }); + } + }); + visited.set(tx.txid, tx.ancestors); + + return tx.ancestors; +} + +/** + * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph + * by running setAncestors on each leaf, and caching intermediate results. + * then initializes ancestor data for each transaction + * + * @param all + */ +export function initializeRelatives(mempoolTxs: Map): Map { + const visited: Map> = new Map(); + const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0); + for (const leaf of leaves) { + setAncestors(leaf, mempoolTxs, visited); + } + mempoolTxs.forEach(entry => { + entry.ancestors?.forEach(ancestor => { + entry.ancestorcount++; + entry.ancestorsize += ancestor.vsize; + entry.fees.ancestor += ancestor.fees.base; + }); + setAncestorScores(entry); + }); + return mempoolTxs; +} + +/** + * Remove a cluster of transactions from an in-mempool dependency graph + * and update the survivors' scores and ancestors + * + * @param cluster + * @param ancestors + */ +export function removeAncestors(cluster: Map, all: Map): void { + // remove + cluster.forEach(tx => { + all.delete(tx.txid); + }); + + // update survivors + all.forEach(tx => { + cluster.forEach(remove => { + if (tx.ancestors?.has(remove.txid)) { + // remove as dependency + tx.ancestors.delete(remove.txid); + tx.depends = tx.depends.filter(parent => parent !== remove.txid); + // update ancestor sizes and fees + tx.ancestorsize -= remove.vsize; + tx.fees.ancestor -= remove.fees.base; + } + }); + // recalculate fee rates + setAncestorScores(tx); + }); +} + +/** + * Take a mempool transaction, and set the fee rates and ancestor score + * + * @param tx + */ +export function setAncestorScores(tx: GraphTx): void { + tx.individualRate = tx.fees.base / tx.vsize; + tx.ancestorRate = tx.fees.ancestor / tx.ancestorsize; + tx.score = Math.min(tx.individualRate, tx.ancestorRate); +} + +// Sort by descending score +export function mempoolComparator(a: GraphTx, b: GraphTx): number { + return b.score - a.score; +} + +/* +* Build a block using an approximation of the transaction selection algorithm from Bitcoin Core +* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) +*/ +export function makeBlockTemplate(candidates: MempoolTransactionExtended[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] { + const auditPool: Map = new Map(); + const mempoolArray: MinerTransaction[] = []; + + candidates.forEach(tx => { + // initializing everything up front helps V8 optimize property access later + const adjustedVsize = Math.ceil(Math.max(tx.weight / 4, 5 * (tx.sigops || 0))); + const feePerVsize = (tx.fee / adjustedVsize); + auditPool.set(tx.txid, { + txid: tx.txid, + order: txidToOrdering(tx.txid), + fee: tx.fee, + feeDelta: 0, + weight: tx.weight, + adjustedVsize, + feePerVsize: feePerVsize, + effectiveFeePerVsize: feePerVsize, + dependencyRate: feePerVsize, + sigops: tx.sigops || 0, + inputs: (tx.vin?.map(vin => vin.txid) || []) as string[], + relativesSet: false, + ancestors: [], + cluster: [], + ancestorMap: new Map(), + children: new Set(), + ancestorFee: 0, + ancestorVsize: 0, + ancestorSigops: 0, + score: 0, + used: false, + modified: false, + }); + mempoolArray.push(auditPool.get(tx.txid) as MinerTransaction); + }); + + // set accelerated effective fee + for (const acceleration of accelerations) { + const tx = auditPool.get(acceleration.txid); + if (tx) { + tx.feeDelta = acceleration.max_bid; + tx.feePerVsize = ((tx.fee + tx.feeDelta) / tx.adjustedVsize); + tx.effectiveFeePerVsize = tx.feePerVsize; + tx.dependencyRate = tx.feePerVsize; + } + } + + // Build relatives graph & calculate ancestor scores + for (const tx of mempoolArray) { + if (!tx.relativesSet) { + setRelatives(tx, auditPool); + } + } + + // Sort by descending ancestor score + mempoolArray.sort(priorityComparator); + + // Build blocks by greedily choosing the highest feerate package + // (i.e. the package rooted in the transaction with the best ancestor score) + const blocks: number[][] = []; + let blockWeight = 0; + let blockSigops = 0; + const transactions: MinerTransaction[] = []; + let modified: MinerTransaction[] = []; + const overflow: MinerTransaction[] = []; + let failures = 0; + while (mempoolArray.length || modified.length) { + // skip invalid transactions + while (mempoolArray[0].used || mempoolArray[0].modified) { + mempoolArray.shift(); + } + + // Select best next package + let nextTx; + const nextPoolTx = mempoolArray[0]; + const nextModifiedTx = modified[0]; + if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) { + nextTx = nextPoolTx; + mempoolArray.shift(); + } else { + modified.shift(); + if (nextModifiedTx) { + nextTx = nextModifiedTx; + } + } + + if (nextTx && !nextTx?.used) { + // Check if the package fits into this block + if (blocks.length >= (maxBlocks - 1) || ((blockWeight + (4 * nextTx.ancestorVsize) < weightLimit) && (blockSigops + nextTx.ancestorSigops <= sigopLimit))) { + const ancestors: MinerTransaction[] = Array.from(nextTx.ancestorMap.values()); + // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) + const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; + const clusterTxids = sortedTxSet.map(tx => tx.txid); + const effectiveFeeRate = Math.min(nextTx.dependencyRate || Infinity, nextTx.ancestorFee / nextTx.ancestorVsize); + const used: MinerTransaction[] = []; + while (sortedTxSet.length) { + const ancestor = sortedTxSet.pop(); + if (!ancestor) { + continue; + } + ancestor.used = true; + ancestor.usedBy = nextTx.txid; + // update this tx with effective fee rate & relatives data + if (ancestor.effectiveFeePerVsize !== effectiveFeeRate) { + ancestor.effectiveFeePerVsize = effectiveFeeRate; + } + ancestor.cluster = clusterTxids; + transactions.push(ancestor); + blockWeight += ancestor.weight; + blockSigops += ancestor.sigops; + used.push(ancestor); + } + + // remove these as valid package ancestors for any descendants remaining in the mempool + if (used.length) { + used.forEach(tx => { + modified = updateDescendants(tx, auditPool, modified, effectiveFeeRate); + }); + } + + failures = 0; + } else { + // hold this package in an overflow list while we check for smaller options + overflow.push(nextTx); + failures++; + } + } + + // this block is full + const exceededPackageTries = failures > 1000 && blockWeight > (weightLimit - 4000); + const queueEmpty = !mempoolArray.length && !modified.length; + + if (exceededPackageTries || queueEmpty) { + break; + } + } + + for (const tx of transactions) { + tx.ancestors = Object.values(tx.ancestorMap); + } + + return transactions; +} + +// traverse in-mempool ancestors +// recursion unavoidable, but should be limited to depth < 25 by mempool policy +function setRelatives( + tx: MinerTransaction, + mempool: Map, +): void { + for (const parent of tx.inputs) { + const parentTx = mempool.get(parent); + if (parentTx && !tx.ancestorMap?.has(parent)) { + tx.ancestorMap.set(parent, parentTx); + parentTx.children.add(tx); + // visit each node only once + if (!parentTx.relativesSet) { + setRelatives(parentTx, mempool); + } + parentTx.ancestorMap.forEach((ancestor) => { + tx.ancestorMap.set(ancestor.txid, ancestor); + }); + } + }; + tx.ancestorFee = (tx.fee + tx.feeDelta); + tx.ancestorVsize = tx.adjustedVsize || 0; + tx.ancestorSigops = tx.sigops || 0; + tx.ancestorMap.forEach((ancestor) => { + tx.ancestorFee += (ancestor.fee + ancestor.feeDelta); + tx.ancestorVsize += ancestor.adjustedVsize; + tx.ancestorSigops += ancestor.sigops; + }); + tx.score = tx.ancestorFee / tx.ancestorVsize; + tx.relativesSet = true; +} + +// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score +// avoids recursion to limit call stack depth +function updateDescendants( + rootTx: MinerTransaction, + mempool: Map, + modified: MinerTransaction[], + clusterRate: number, +): MinerTransaction[] { + const descendantSet: Set = new Set(); + // stack of nodes left to visit + const descendants: MinerTransaction[] = []; + let descendantTx: MinerTransaction | undefined; + rootTx.children.forEach(childTx => { + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); + } + }); + while (descendants.length) { + descendantTx = descendants.pop(); + if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) { + // remove tx as ancestor + descendantTx.ancestorMap.delete(rootTx.txid); + descendantTx.ancestorFee -= (rootTx.fee + rootTx.feeDelta); + descendantTx.ancestorVsize -= rootTx.adjustedVsize; + descendantTx.ancestorSigops -= rootTx.sigops; + descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorVsize; + descendantTx.dependencyRate = descendantTx.dependencyRate ? Math.min(descendantTx.dependencyRate, clusterRate) : clusterRate; + + if (!descendantTx.modified) { + descendantTx.modified = true; + modified.push(descendantTx); + } + + // add this node's children to the stack + descendantTx.children.forEach(childTx => { + // visit each node only once + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); + } + }); + } + } + // return new, resorted modified list + return modified.sort(priorityComparator); +} + +// Used to sort an array of MinerTransactions by descending ancestor score +function priorityComparator(a: MinerTransaction, b: MinerTransaction): number { + if (b.score === a.score) { + // tie-break by txid for stability + return a.order - b.order; + } else { + return b.score - a.score; + } +} + +// returns the most significant 4 bytes of the txid as an integer +function txidToOrdering(txid: string): number { + return parseInt( + txid.substring(62, 64) + + txid.substring(60, 62) + + txid.substring(58, 60) + + txid.substring(56, 58), + 16 + ); +} diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 8d1a85994..cf2c56763 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -33,7 +33,7 @@ interface AddressTransactions { removed: MempoolTransactionExtended[], } import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; -import { calculateCpfp } from './cpfp'; +import { calculateMempoolTxCpfp } from './cpfp'; // valid 'want' subscriptions const wantable = [ @@ -827,7 +827,7 @@ class WebsocketHandler { accelerationPositions: memPool.getAccelerationPositions(mempoolTx.txid), }; if (!mempoolTx.cpfpChecked && !mempoolTx.acceleration) { - calculateCpfp(mempoolTx, newMempool); + calculateMempoolTxCpfp(mempoolTx, newMempool); } if (mempoolTx.cpfpDirty) { positionData['cpfp'] = { @@ -866,7 +866,7 @@ class WebsocketHandler { acceleratedAt: mempoolTx.acceleratedAt || undefined, }; if (!mempoolTx.cpfpChecked) { - calculateCpfp(mempoolTx, newMempool); + calculateMempoolTxCpfp(mempoolTx, newMempool); } if (mempoolTx.cpfpDirty) { txInfo.cpfp = { diff --git a/backend/src/repositories/AccelerationRepository.ts b/backend/src/repositories/AccelerationRepository.ts index 0da66228c..1c3bdad29 100644 --- a/backend/src/repositories/AccelerationRepository.ts +++ b/backend/src/repositories/AccelerationRepository.ts @@ -1,4 +1,4 @@ -import { AccelerationInfo, makeBlockTemplate } from '../api/acceleration/acceleration'; +import { AccelerationInfo } from '../api/acceleration/acceleration'; import { RowDataPacket } from 'mysql2'; import DB from '../database'; import logger from '../logger'; @@ -11,6 +11,7 @@ import accelerationCosts from '../api/acceleration/acceleration'; import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; import transactionUtils from '../api/transaction-utils'; import { BlockExtended, MempoolTransactionExtended } from '../mempool.interfaces'; +import { makeBlockTemplate } from '../api/mini-miner'; export interface PublicAcceleration { txid: string, From 41c373c39d6f7f18228f3b673be790db0e036e2d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 6 Apr 2024 02:24:22 +0000 Subject: [PATCH 3/7] Mini-miner based block cpfp calculations --- backend/src/api/blocks.ts | 11 +++--- backend/src/api/cpfp.ts | 71 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 5a1b35547..a36f18200 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -31,6 +31,7 @@ import rbfCache from './rbf-cache'; import { calcBitsDifference } from './difficulty-adjustment'; import AccelerationRepository from '../repositories/AccelerationRepository'; import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp'; +import mempool from './mempool'; class Blocks { private blocks: BlockExtended[] = []; @@ -602,13 +603,13 @@ class Blocks { for (let height = currentBlockHeight; height >= 0; height--) { try { - let txs: TransactionExtended[] | null = null; + let txs: MempoolTransactionExtended[] | null = null; if (unclassifiedBlocks[height]) { const blockHash = unclassifiedBlocks[height]; // fetch transactions - txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)) || []; + txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendMempoolTransaction(tx)) || []; // add CPFP - const cpfpSummary = calculateFastBlockCpfp(height, txs, true); + const cpfpSummary = calculateGoodBlockCpfp(height, txs, []); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions); await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1); @@ -637,7 +638,7 @@ class Blocks { } templateTxs.push(tx || templateTx); } - const cpfpSummary = calculateFastBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as TransactionExtended[], true); + const cpfpSummary = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions); const classifiedTxMap: { [txid: string]: TransactionClassified } = {}; @@ -891,7 +892,7 @@ class Blocks { } } - const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions); + const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, Object.values(mempool.getAccelerations()).map(a => ({ txid: a.txid, max_bid: a.feeDelta }))); const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions); this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`); diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts index 1b0873b7a..9bf311ff4 100644 --- a/backend/src/api/cpfp.ts +++ b/backend/src/api/cpfp.ts @@ -1,6 +1,7 @@ -import { CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces'; -import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner'; +import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces'; +import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner'; import memPool from './mempool'; +import { Acceleration } from './acceleration'; const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction const MAX_CLUSTER_ITERATIONS = 100; @@ -95,8 +96,70 @@ export function calculateFastBlockCpfp(height: number, transactions: Transaction }; } -export function calculateGoodBlockCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary { - return calculateFastBlockCpfp(height, transactions, true); +export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary { + const txMap: { [txid: string]: MempoolTransactionExtended } = {}; + for (const tx of transactions) { + txMap[tx.txid] = tx; + } + const template = makeBlockTemplate(transactions, accelerations, 1, Infinity, Infinity); + const clusters = new Map(); + for (const tx of template) { + const cluster = tx.cluster || []; + const root = cluster.length ? cluster[cluster.length - 1] : null; + if (cluster.length > 1 && root && !clusters.has(root)) { + clusters.set(root, cluster); + } + txMap[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize; + } + + const clusterArray: CpfpCluster[] = []; + + for (const cluster of clusters.values()) { + for (const txid of cluster) { + const mempoolTx = txMap[txid]; + if (mempoolTx) { + const ancestors: Ancestor[] = []; + const descendants: Ancestor[] = []; + let matched = false; + cluster.forEach(relativeTxid => { + if (relativeTxid === txid) { + matched = true; + } else { + const relative = { + txid: relativeTxid, + fee: txMap[relativeTxid].fee, + weight: (txMap[relativeTxid].adjustedVsize * 4) || txMap[relativeTxid].weight, + }; + if (matched) { + descendants.push(relative); + } else { + ancestors.push(relative); + } + } + }); + if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) { + mempoolTx.cpfpDirty = true; + } + Object.assign(mempoolTx, { ancestors, descendants, bestDescendant: null, cpfpChecked: true }); + } + } + const root = cluster[cluster.length - 1]; + clusterArray.push({ + root: root, + height, + txs: cluster.reverse().map(txid => ({ + txid, + fee: txMap[txid].fee, + weight: (txMap[txid].adjustedVsize * 4) || txMap[txid].weight, + })), + effectiveFeePerVsize: txMap[root].effectiveFeePerVsize, + }); + } + + return { + transactions: transactions.map(tx => txMap[tx.txid]), + clusters: clusterArray, + }; } /** From 79eb9635c2abb66c6987e13c0efeff67358c1220 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 6 Apr 2024 07:59:18 +0000 Subject: [PATCH 4/7] Fix cpfp vsize rounding & goggles bugs --- backend/src/api/blocks.ts | 2 +- backend/src/api/common.ts | 3 +++ backend/src/api/transaction-utils.ts | 4 ++-- .../block-overview-tooltip.component.ts | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index a36f18200..97b5d2055 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -1151,7 +1151,7 @@ class Blocks { transactions: cpfpSummary.transactions.map(tx => { let flags: number = 0; try { - flags = tx.flags || Common.getTransactionFlags(tx); + flags = Common.getTransactionFlags(tx); } catch (e) { logger.warn('Failed to classify transaction: ' + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index dffabce0a..cba39a511 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -419,12 +419,15 @@ export class Common { let flags = tx.flags ? BigInt(tx.flags) : 0n; // Update variable flags (CPFP, RBF) + flags &= ~TransactionFlags.cpfp_child; if (tx.ancestors?.length) { flags |= TransactionFlags.cpfp_child; } + flags &= ~TransactionFlags.cpfp_parent; if (tx.descendants?.length) { flags |= TransactionFlags.cpfp_parent; } + flags &= ~TransactionFlags.replacement; if (tx.replacement) { flags |= TransactionFlags.replacement; } diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 9107f2ae7..b3077b935 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -103,7 +103,7 @@ class TransactionUtils { } const feePerVbytes = (transaction.fee || 0) / (transaction.weight / 4); const transactionExtended: TransactionExtended = Object.assign({ - vsize: Math.round(transaction.weight / 4), + vsize: transaction.weight / 4, feePerVsize: feePerVbytes, effectiveFeePerVsize: feePerVbytes, }, transaction); @@ -123,7 +123,7 @@ class TransactionUtils { const adjustedFeePerVsize = (transaction.fee || 0) / adjustedVsize; const transactionExtended: MempoolTransactionExtended = Object.assign(transaction, { order: this.txidToOrdering(transaction.txid), - vsize: Math.round(transaction.weight / 4), + vsize, adjustedVsize, sigops, feePerVsize: feePerVbytes, diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts index d59bfa87f..16cba08c0 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts @@ -68,7 +68,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { this.effectiveRate = this.tx.rate; const txFlags = BigInt(this.tx.flags) || 0n; this.acceleration = this.tx.acc || (txFlags & TransactionFlags.acceleration); - this.hasEffectiveRate = this.tx.acc || Math.abs((this.fee / this.vsize) - this.effectiveRate) > 0.05 + this.hasEffectiveRate = this.tx.acc || !(Math.abs((this.fee / this.vsize) - this.effectiveRate) <= 0.05 && Math.abs((this.fee / Math.ceil(this.vsize)) - this.effectiveRate) <= 0.05) || (txFlags && (txFlags & (TransactionFlags.cpfp_child | TransactionFlags.cpfp_parent)) > 0n); this.filters = this.tx.flags ? toFilters(txFlags).filter(f => f.tooltip) : []; this.activeFilters = {} From 9aac0ddce7744dfc3e8428aab28751ba35ff47e2 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 6 Apr 2024 09:05:27 +0000 Subject: [PATCH 5/7] Fix merge conflict --- backend/src/api/cpfp.ts | 2 +- backend/src/api/mini-miner.ts | 2 +- .../block-overview-tooltip/block-overview-tooltip.component.ts | 2 +- .../src/app/components/transaction/transaction.component.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts index 9bf311ff4..284c45349 100644 --- a/backend/src/api/cpfp.ts +++ b/backend/src/api/cpfp.ts @@ -1,7 +1,7 @@ import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces'; import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner'; import memPool from './mempool'; -import { Acceleration } from './acceleration'; +import { Acceleration } from './acceleration/acceleration'; const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction const MAX_CLUSTER_ITERATIONS = 100; diff --git a/backend/src/api/mini-miner.ts b/backend/src/api/mini-miner.ts index 28d9de3f7..4a4ef5daa 100644 --- a/backend/src/api/mini-miner.ts +++ b/backend/src/api/mini-miner.ts @@ -1,4 +1,4 @@ -import { Acceleration } from './acceleration'; +import { Acceleration } from './acceleration/acceleration'; import { MempoolTransactionExtended } from '../mempool.interfaces'; import logger from '../logger'; diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts index 16cba08c0..0a606983e 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts @@ -68,7 +68,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { this.effectiveRate = this.tx.rate; const txFlags = BigInt(this.tx.flags) || 0n; this.acceleration = this.tx.acc || (txFlags & TransactionFlags.acceleration); - this.hasEffectiveRate = this.tx.acc || !(Math.abs((this.fee / this.vsize) - this.effectiveRate) <= 0.05 && Math.abs((this.fee / Math.ceil(this.vsize)) - this.effectiveRate) <= 0.05) + this.hasEffectiveRate = this.tx.acc || !(Math.abs((this.fee / this.vsize) - this.effectiveRate) <= 0.1 && Math.abs((this.fee / Math.ceil(this.vsize)) - this.effectiveRate) <= 0.1) || (txFlags && (txFlags & (TransactionFlags.cpfp_child | TransactionFlags.cpfp_parent)) > 0n); this.filters = this.tx.flags ? toFilters(txFlags).filter(f => f.tooltip) : []; this.activeFilters = {} diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index fef98d414..f73a0b225 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -599,7 +599,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { bestDescendant: tx.bestDescendant, }); const hasRelatives = !!(tx.ancestors?.length || tx.bestDescendant); - this.hasEffectiveFeeRate = hasRelatives || (tx.effectiveFeePerVsize && (Math.abs(tx.effectiveFeePerVsize - tx.feePerVsize) > 0.01)); + this.hasEffectiveFeeRate = hasRelatives || (tx.effectiveFeePerVsize && (Math.abs(tx.effectiveFeePerVsize - tx.feePerVsize) >= 0.1)); } else { this.fetchCpfp$.next(this.tx.txid); } From 398593828fc5ac6c6cfd5168d8aae7643bced203 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 20 Jul 2024 11:59:18 +0000 Subject: [PATCH 6/7] Implement CPFP reindexing using mini-miner method (not activated) --- backend/src/api/blocks.ts | 18 ++++- .../repositories/BlocksSummariesRepository.ts | 37 +++++++++ backend/src/repositories/CpfpRepository.ts | 77 +++++++++++++++++++ 3 files changed, 129 insertions(+), 3 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 97b5d2055..762c81ff7 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -32,6 +32,7 @@ import { calcBitsDifference } from './difficulty-adjustment'; import AccelerationRepository from '../repositories/AccelerationRepository'; import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp'; import mempool from './mempool'; +import CpfpRepository from '../repositories/CpfpRepository'; class Blocks { private blocks: BlockExtended[] = []; @@ -569,8 +570,11 @@ class Blocks { const blockchainInfo = await bitcoinClient.getBlockchainInfo(); const currentBlockHeight = blockchainInfo.blocks; - const unclassifiedBlocksList = await BlocksSummariesRepository.$getSummariesWithVersion(0); - const unclassifiedTemplatesList = await BlocksSummariesRepository.$getTemplatesWithVersion(0); + const targetSummaryVersion: number = 1; + const targetTemplateVersion: number = 1; + + const unclassifiedBlocksList = await BlocksSummariesRepository.$getSummariesBelowVersion(targetSummaryVersion); + const unclassifiedTemplatesList = await BlocksSummariesRepository.$getTemplatesBelowVersion(targetTemplateVersion); // nothing to do if (!unclassifiedBlocksList?.length && !unclassifiedTemplatesList?.length) { @@ -612,7 +616,15 @@ class Blocks { const cpfpSummary = calculateGoodBlockCpfp(height, txs, []); // classify const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions); - await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1); + await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 2); + if (unclassifiedBlocks[height].version < 2 && targetSummaryVersion === 2) { + const cpfpClusters = await CpfpRepository.$getClustersAt(height); + if (!cpfpRepository.compareClusters(cpfpClusters, cpfpSummary.clusters)) { + // CPFP clusters changed - update the compact_cpfp tables + await CpfpRepository.$deleteClustersAt(height); + await this.$saveCpfp(blockHash, height, cpfpSummary); + } + } await Common.sleep$(250); } if (unclassifiedTemplates[height]) { diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 63ad5ddf2..0268424f2 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -114,6 +114,43 @@ class BlocksSummariesRepository { return []; } + public async $getSummariesBelowVersion(version: number): Promise<{ height: number, id: string, version: number }[]> { + try { + const [rows]: any[] = await DB.query(` + SELECT + height, + id, + version + FROM blocks_summaries + WHERE version < ? + ORDER BY height DESC;`, [version]); + return rows; + } catch (e) { + logger.err(`Cannot get block summaries below version. Reason: ` + (e instanceof Error ? e.message : e)); + } + + return []; + } + + public async $getTemplatesBelowVersion(version: number): Promise<{ height: number, id: string, version: number }[]> { + try { + const [rows]: any[] = await DB.query(` + SELECT + blocks_summaries.height as height, + blocks_templates.id as id, + blocks_templates.version as version + FROM blocks_templates + JOIN blocks_summaries ON blocks_templates.id = blocks_summaries.id + WHERE blocks_templates.version < ? + ORDER BY height DESC;`, [version]); + return rows; + } catch (e) { + logger.err(`Cannot get block summaries below version. Reason: ` + (e instanceof Error ? e.message : e)); + } + + return []; + } + /** * Get the fee percentiles if the block has already been indexed, [] otherwise * diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index b33ff1e4a..0242188df 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -91,6 +91,26 @@ class CpfpRepository { return; } + public async $getClustersAt(height: number): Promise { + const [clusterRows]: any = await DB.query( + ` + SELECT * + FROM compact_cpfp_clusters + WHERE height = ? + `, + [height] + ); + return clusterRows.map(cluster => { + if (cluster?.txs) { + cluster.effectiveFeePerVsize = cluster.fee_rate; + cluster.txs = this.unpack(cluster.txs); + return cluster; + } else { + return null; + } + }).filter(cluster => cluster !== null); + } + public async $deleteClustersFrom(height: number): Promise { logger.info(`Delete newer cpfp clusters from height ${height} from the database`); try { @@ -122,6 +142,37 @@ class CpfpRepository { } } + public async $deleteClustersAt(height: number): Promise { + logger.info(`Delete cpfp clusters at height ${height} from the database`); + try { + const [rows] = await DB.query( + ` + SELECT txs, height, root from compact_cpfp_clusters + WHERE height = ? + `, + [height] + ) as RowDataPacket[][]; + if (rows?.length) { + for (const clusterToDelete of rows) { + const txs = this.unpack(clusterToDelete?.txs); + for (const tx of txs) { + await transactionRepository.$removeTransaction(tx.txid); + } + } + } + await DB.query( + ` + DELETE from compact_cpfp_clusters + WHERE height = ? + `, + [height] + ); + } catch (e: any) { + logger.err(`Cannot delete cpfp clusters from db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + // insert a dummy row to mark that we've indexed as far as this block public async $insertProgressMarker(height: number): Promise { try { @@ -190,6 +241,32 @@ class CpfpRepository { return []; } } + + // returns `true` if two sets of CPFP clusters are deeply identical + public compareClusters(clustersA: CpfpCluster[], clustersB: CpfpCluster[]): boolean { + if (clustersA.length !== clustersB.length) { + return false; + } + + clustersA = clustersA.sort((a,b) => a.root.localeCompare(b.root)); + clustersB = clustersB.sort((a,b) => a.root.localeCompare(b.root)); + + for (let i = 0; i < clustersA.length; i++) { + if (clustersA[i].root !== clustersB[i].root) { + return false; + } + if (clustersA[i].txs.length !== clustersB[i].txs.length) { + return false; + } + for (let j = 0; j < clustersA[i].txs.length; j++) { + if (clustersA[i].txs[j].txid !== clustersB[i].txs[j].txid) { + return false; + } + } + } + + return true; + } } export default new CpfpRepository(); \ No newline at end of file From 8116b50d503293e203c31887d247a0895cde92c3 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 21 Jul 2024 20:34:20 +0800 Subject: [PATCH 7/7] fix spelling error --- frontend/src/app/components/address/address.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index 08eb841ee..76c64948b 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -201,7 +201,7 @@ Error loading address data.
- There many transactions on this address, more than your backend can handle. See more on setting up a stronger backend. + There are too many transactions on this address, more than your backend can handle. See more on setting up a stronger backend.

Consider viewing this address on the official Mempool website instead: