diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 3b416255a..b0b157c42 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -25,7 +25,8 @@ "AUTOMATIC_BLOCK_REINDEXING": false, "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", - "ADVANCED_TRANSACTION_SELECTION": false + "ADVANCED_TRANSACTION_SELECTION": false, + "TRANSACTION_INDEXING": false }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index ec6be20d8..8b368a43a 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -26,7 +26,8 @@ "INDEXING_BLOCKS_AMOUNT": 14, "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__POOLS_JSON_URL__", - "ADVANCED_TRANSACTION_SELECTION": "__ADVANCED_TRANSACTION_SELECTION__" + "ADVANCED_TRANSACTION_SELECTION": "__ADVANCED_TRANSACTION_SELECTION__", + "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 9bb06c58a..c95888cf2 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -39,6 +39,7 @@ describe('Mempool Backend Config', () => { POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', ADVANCED_TRANSACTION_SELECTION: false, + TRANSACTION_INDEXING: false, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index cdcc589fd..5d0a89787 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -17,13 +17,14 @@ import logger from '../../logger'; import blocks from '../blocks'; import bitcoinClient from './bitcoin-client'; import difficultyAdjustment from '../difficulty-adjustment'; +import transactionRepository from '../../repositories/TransactionRepository'; class BitcoinRoutes { public initRoutes(app: Application) { app .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes) .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends) - .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.getCpfpInfo) + .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo) .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange) .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees) .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks) @@ -188,29 +189,34 @@ class BitcoinRoutes { } } - private getCpfpInfo(req: Request, res: Response) { + private async $getCpfpInfo(req: Request, res: Response) { if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) { res.status(501).send(`Invalid transaction ID.`); return; } const tx = mempool.getMempool()[req.params.txId]; - if (!tx) { - res.status(404).send(`Transaction doesn't exist in the mempool.`); + if (tx) { + if (tx?.cpfpChecked) { + res.json({ + ancestors: tx.ancestors, + bestDescendant: tx.bestDescendant || null, + }); + return; + } + + const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool()); + + res.json(cpfpInfo); return; + } else { + const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId); + if (cpfpInfo) { + res.json(cpfpInfo); + return; + } } - - if (tx.cpfpChecked) { - res.json({ - ancestors: tx.ancestors, - bestDescendant: tx.bestDescendant || null, - }); - return; - } - - const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool()); - - res.json(cpfpInfo); + res.status(404).send(`Transaction has no CPFP info available.`); } private getBackendInfo(req: Request, res: Response) { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 411777393..eed362623 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -21,6 +21,8 @@ import fiatConversion from './fiat-conversion'; import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; +import cpfpRepository from '../repositories/CpfpRepository'; +import transactionRepository from '../repositories/TransactionRepository'; import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; import PricesRepository from '../repositories/PricesRepository'; @@ -260,7 +262,7 @@ class Blocks { /** * [INDEXING] Index all blocks summaries for the block txs visualization */ - public async $generateBlocksSummariesDatabase() { + public async $generateBlocksSummariesDatabase(): Promise { if (Common.blocksSummariesIndexingEnabled() === false) { return; } @@ -316,6 +318,56 @@ class Blocks { } } + /** + * [INDEXING] Index transaction CPFP data for all blocks + */ + public async $generateCPFPDatabase(): Promise { + if (Common.cpfpIndexingEnabled() === false) { + return; + } + + try { + // Get all indexed block hash + const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks(); + + if (!unindexedBlocks?.length) { + return; + } + + // Logging + let count = 0; + let countThisRun = 0; + let timer = new Date().getTime() / 1000; + const startedAt = new Date().getTime() / 1000; + + for (const block of unindexedBlocks) { + // Logging + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + if (elapsedSeconds > 5) { + const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + const blockPerSeconds = Math.max(1, countThisRun / elapsedSeconds); + const progress = Math.round(count / unindexedBlocks.length * 10000) / 100; + logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`); + timer = new Date().getTime() / 1000; + countThisRun = 0; + } + + await this.$indexCPFP(block.hash); // Calculate and save CPFP data for transactions in this block + + // Logging + count++; + } + if (count > 0) { + logger.notice(`CPFP indexing completed: indexed ${count} blocks`); + } else { + logger.debug(`CPFP indexing completed: indexed ${count} blocks`); + } + } catch (e) { + logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); + throw e; + } + } + /** * [INDEXING] Index all blocks metadata for the mining dashboard */ @@ -461,9 +513,13 @@ class Blocks { await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10); await HashratesRepository.$deleteLastEntries(); await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10); + await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10); for (let i = 10; i >= 0; --i) { const newBlock = await this.$indexBlock(lastBlock['height'] - i); await this.$getStrippedBlockTransactions(newBlock.id, true, true); + if (config.MEMPOOL.TRANSACTION_INDEXING) { + await this.$indexCPFP(newBlock.id); + } } await mining.$indexDifficultyAdjustments(); await DifficultyAdjustmentsRepository.$deleteLastAdjustment(); @@ -489,6 +545,9 @@ class Blocks { if (Common.blocksSummariesIndexingEnabled() === true) { await this.$getStrippedBlockTransactions(blockExtended.id, true); } + if (config.MEMPOOL.TRANSACTION_INDEXING) { + this.$indexCPFP(blockExtended.id); + } } } @@ -678,6 +737,38 @@ class Blocks { public getCurrentBlockHeight(): number { return this.currentBlockHeight; } + + public async $indexCPFP(hash: string): Promise { + const block = await bitcoinClient.getBlock(hash, 2); + const transactions = block.tx; + let cluster: IBitcoinApi.VerboseTransaction[] = []; + let ancestors: { [txid: string]: boolean } = {}; + for (let i = transactions.length - 1; i >= 0; i--) { + const tx = transactions[i]; + if (!ancestors[tx.txid]) { + let totalFee = 0; + let totalWeight = 0; + cluster.forEach(tx => { + totalFee += tx?.fee || 0; + totalWeight += tx.weight; + }); + const effectiveFeePerVsize = (totalFee * 100_000_000) / (totalWeight / 4); + if (cluster.length > 1) { + await cpfpRepository.$saveCluster(block.height, cluster.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: (tx.fee || 0) * 100_000_000 }; }), effectiveFeePerVsize); + for (const tx of cluster) { + await transactionRepository.$setCluster(tx.txid, cluster[0].txid); + } + } + cluster = []; + ancestors = {}; + } + cluster.push(tx); + tx.vin.forEach(vin => { + ancestors[vin.txid] = true; + }); + } + await blocksRepository.$setCPFPIndexed(hash); + } } export default new Blocks(); diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index b9cc1453c..621f021ba 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -187,6 +187,13 @@ export class Common { ); } + static cpfpIndexingEnabled(): boolean { + return ( + Common.indexingEnabled() && + config.MEMPOOL.TRANSACTION_INDEXING === true + ); + } + static setDateMidnight(date: Date): void { date.setUTCHours(0); date.setUTCMinutes(0); diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index d8e96d57d..e51a374f1 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 45; + private static currentVersion = 46; private queryTimeout = 900_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -369,6 +369,12 @@ class DatabaseMigration { if (databaseSchemaVersion < 45 && isBitcoin === true) { await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"'); } + + if (databaseSchemaVersion < 46 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks` ADD cpfp_indexed tinyint(1) DEFAULT 0'); + await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters')); + await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions')); + } } /** @@ -817,6 +823,25 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateCPFPTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS cpfp_clusters ( + root varchar(65) NOT NULL, + height int(10) NOT NULL, + txs JSON DEFAULT NULL, + fee_rate double unsigned NOT NULL, + PRIMARY KEY (root) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + + private getCreateTransactionsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS transactions ( + txid varchar(65) NOT NULL, + cluster varchar(65) DEFAULT NULL, + PRIMARY KEY (txid), + FOREIGN KEY (cluster) REFERENCES cpfp_clusters (root) ON DELETE SET NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates', 'prices']; diff --git a/backend/src/config.ts b/backend/src/config.ts index f7d1ee60a..808e1406b 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -30,6 +30,7 @@ interface IConfig { POOLS_JSON_URL: string, POOLS_JSON_TREE_URL: string, ADVANCED_TRANSACTION_SELECTION: boolean; + TRANSACTION_INDEXING: boolean; }; ESPLORA: { REST_API_URL: string; @@ -148,6 +149,7 @@ const defaults: IConfig = { 'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', 'ADVANCED_TRANSACTION_SELECTION': false, + 'TRANSACTION_INDEXING': false, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 26a407291..22f3ce319 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -77,6 +77,7 @@ class Indexer { await mining.$generateNetworkHashrateHistory(); await mining.$generatePoolHashrateHistory(); await blocks.$generateBlocksSummariesDatabase(); + await blocks.$generateCPFPDatabase(); } catch (e) { this.indexerRunning = false; logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 0e68d2ed5..01bc45742 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -119,7 +119,9 @@ interface BestDescendant { export interface CpfpInfo { ancestors: Ancestor[]; - bestDescendant: BestDescendant | null; + bestDescendant?: BestDescendant | null; + descendants?: Ancestor[]; + effectiveFeePerVsize?: number; } export interface TransactionStripped { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 590e9de37..78a8fcce2 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -662,6 +662,23 @@ class BlocksRepository { } } + /** + * Get a list of blocks that have not had CPFP data indexed + */ + public async $getCPFPUnindexedBlocks(): Promise { + try { + const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE cpfp_indexed = 0 ORDER BY height DESC`); + return rows; + } catch (e) { + logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + + public async $setCPFPIndexed(hash: string): Promise { + await DB.query(`UPDATE blocks SET cpfp_indexed = 1 WHERE hash = ?`, [hash]); + } + /** * Return the oldest block from a consecutive chain of block from the most recent one */ diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts new file mode 100644 index 000000000..563e6ede1 --- /dev/null +++ b/backend/src/repositories/CpfpRepository.ts @@ -0,0 +1,43 @@ +import DB from '../database'; +import logger from '../logger'; +import { Ancestor } from '../mempool.interfaces'; + +class CpfpRepository { + public async $saveCluster(height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise { + try { + const txsJson = JSON.stringify(txs); + await DB.query( + ` + INSERT INTO cpfp_clusters(root, height, txs, fee_rate) + VALUE (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + height = ?, + txs = ?, + fee_rate = ? + `, + [txs[0].txid, height, txsJson, effectiveFeePerVsize, height, txsJson, effectiveFeePerVsize, height] + ); + } catch (e: any) { + logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + public async $deleteClustersFrom(height: number): Promise { + logger.info(`Delete newer cpfp clusters from height ${height} from the database`); + try { + await DB.query( + ` + DELETE from cpfp_clusters + WHERE height >= ? + `, + [height] + ); + } catch (e: any) { + logger.err(`Cannot delete cpfp clusters from db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } +} + +export default new CpfpRepository(); \ No newline at end of file diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts new file mode 100644 index 000000000..1c6e3719f --- /dev/null +++ b/backend/src/repositories/TransactionRepository.ts @@ -0,0 +1,77 @@ +import DB from '../database'; +import logger from '../logger'; +import { Ancestor, CpfpInfo } from '../mempool.interfaces'; + +interface CpfpSummary { + txid: string; + cluster: string; + root: string; + txs: Ancestor[]; + height: number; + fee_rate: number; +} + +class TransactionRepository { + public async $setCluster(txid: string, cluster: string): Promise { + try { + await DB.query( + ` + INSERT INTO transactions + ( + txid, + cluster + ) + VALUE (?, ?) + ON DUPLICATE KEY UPDATE + cluster = ? + ;`, + [txid, cluster, cluster] + ); + } catch (e: any) { + logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + public async $getCpfpInfo(txid: string): Promise { + try { + let query = ` + SELECT * + FROM transactions + LEFT JOIN cpfp_clusters AS cluster ON cluster.root = transactions.cluster + WHERE transactions.txid = ? + `; + const [rows]: any = await DB.query(query, [txid]); + if (rows.length) { + rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[]; + return this.convertCpfp(rows[0]); + } + } catch (e) { + logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + + private convertCpfp(cpfp: CpfpSummary): CpfpInfo { + const descendants: Ancestor[] = []; + const ancestors: Ancestor[] = []; + let matched = false; + for (const tx of cpfp.txs) { + if (tx.txid === cpfp.txid) { + matched = true; + } else if (!matched) { + descendants.push(tx); + } else { + ancestors.push(tx); + } + } + return { + descendants, + ancestors, + effectiveFeePerVsize: cpfp.fee_rate + }; + } +} + +export default new TransactionRepository(); +