From f9d03b1bb47dafcaa2324f25502604657631c44e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 24 Jun 2024 06:15:01 +0000 Subject: [PATCH] Add coinbase_addresses to extended blocks & table --- .../bitcoin/bitcoin-api-abstract-factory.ts | 1 + backend/src/api/bitcoin/bitcoin-api.ts | 5 ++ backend/src/api/bitcoin/esplora-api.ts | 5 ++ backend/src/api/blocks.ts | 49 +++++++++++++++++++ backend/src/api/database-migration.ts | 7 ++- backend/src/indexer.ts | 1 + backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/BlocksRepository.ts | 46 ++++++++++++++++- 8 files changed, 112 insertions(+), 3 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 4ec50f4b3..a08f43238 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -28,6 +28,7 @@ export interface AbstractBitcoinApi { $getBatchedOutspends(txId: string[]): Promise; $getBatchedOutspendsInternal(txId: string[]): Promise; $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise; + $getCoinbaseTx(blockhash: string): Promise; startHealthChecks(): void; getHealthStatus(): HealthCheckHost[]; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 0724065ae..9248338ad 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -238,6 +238,11 @@ class BitcoinApi implements AbstractBitcoinApi { return outspends; } + async $getCoinbaseTx(blockhash: string): Promise { + const txids = await this.$getTxIdsForBlock(blockhash); + return this.$getRawTransaction(txids[0]); + } + $getEstimatedHashrate(blockHeight: number): Promise { // 120 is the default block span in Core return this.bitcoindClient.getNetworkHashPs(120, blockHeight); diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index a484e689e..b4ae35da9 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -352,6 +352,11 @@ class ElectrsApi implements AbstractBitcoinApi { return this.failoverRouter.$post('/internal/txs/outspends/by-outpoint', outpoints.map(out => `${out.txid}:${out.vout}`), 'json'); } + async $getCoinbaseTx(blockhash: string): Promise { + const txid = await this.failoverRouter.$get(`/block/${blockhash}/txid/0`); + return this.failoverRouter.$get('/tx/' + txid); + } + public startHealthChecks(): void { this.failoverRouter.startHealthChecks(); } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 42e15bf82..f258fefea 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -295,10 +295,12 @@ class Blocks { extras.virtualSize = block.weight / 4.0; if (coinbaseTx?.vout.length > 0) { extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null; + extras.coinbaseAddresses = [...new Set(...coinbaseTx.vout.map(v => v.scriptpubkey_address).filter(a => a) as string[])]; extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null; extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null; } else { extras.coinbaseAddress = null; + extras.coinbaseAddresses = null; extras.coinbaseSignature = null; extras.coinbaseSignatureAscii = null; } @@ -690,6 +692,52 @@ class Blocks { this.classifyingBlocks = false; } + /** + * [INDEXING] Index missing coinbase addresses for all blocks + */ + public async $indexCoinbaseAddresses(): Promise { + try { + // Get all indexed block hash + const unindexedBlocks = await blocksRepository.$getBlocksWithoutCoinbaseAddresses(); + + if (!unindexedBlocks?.length) { + return; + } + + logger.info(`Indexing missing coinbase addresses for ${unindexedBlocks.length} blocks`); + + // Logging + let count = 0; + let countThisRun = 0; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; + for (const { height, hash } of unindexedBlocks) { + // Logging + const elapsedSeconds = (Date.now() / 1000) - timer; + if (elapsedSeconds > 5) { + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = countThisRun / elapsedSeconds; + const progress = Math.round(count / unindexedBlocks.length * 10000) / 100; + logger.debug(`Indexing coinbase addresses for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`); + timer = Date.now() / 1000; + countThisRun = 0; + } + + const coinbaseTx = await bitcoinApi.$getCoinbaseTx(hash); + const addresses = new Set(coinbaseTx.vout.map(v => v.scriptpubkey_address).filter(a => a)); + await blocksRepository.$saveCoinbaseAddresses(hash, [...addresses]); + + // Logging + count++; + countThisRun++; + } + logger.notice(`coinbase addresses indexing completed: indexed ${count} blocks`); + } catch (e) { + logger.err(`coinbase addresses indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); + throw e; + } + } + /** * [INDEXING] Index all blocks metadata for the mining dashboard */ @@ -1259,6 +1307,7 @@ class Blocks { utxoset_size: block.extras.utxoSetSize ?? null, coinbase_raw: block.extras.coinbaseRaw ?? null, coinbase_address: block.extras.coinbaseAddress ?? null, + coinbase_addresses: block.extras.coinbaseAddresses ?? null, coinbase_signature: block.extras.coinbaseSignature ?? null, coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null, pool_slug: block.extras.pool.slug ?? null, diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index c6cd11866..70ff2d5bb 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 79; + private static currentVersion = 80; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -686,6 +686,11 @@ class DatabaseMigration { `); await this.updateToSchemaVersion(79); } + + if (databaseSchemaVersion < 80) { + await this.$executeQuery('ALTER TABLE `blocks` ADD coinbase_addresses JSON DEFAULT NULL'); + await this.updateToSchemaVersion(80); + } } /** diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index ab2e0678d..0dd1090b8 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -182,6 +182,7 @@ class Indexer { } this.runSingleTask('blocksPrices'); + await blocks.$indexCoinbaseAddresses(); await mining.$indexDifficultyAdjustments(); await mining.$generateNetworkHashrateHistory(); await mining.$generatePoolHashrateHistory(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a2951b4d6..2d3c7a7c0 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -287,6 +287,7 @@ export interface BlockExtension { coinbaseRaw: string; orphans: OrphanedBlock[] | null; coinbaseAddress: string | null; + coinbaseAddresses: string[] | null; coinbaseSignature: string | null; coinbaseSignatureAscii: string | null; virtualSize: number; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 0077a0b79..b2e4b0f11 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -40,6 +40,7 @@ interface DatabaseBlock { avgFeeRate: number; coinbaseRaw: string; coinbaseAddress: string; + coinbaseAddresses: string[]; coinbaseSignature: string; coinbaseSignatureAscii: string; avgTxSize: number; @@ -82,6 +83,7 @@ const BLOCK_DB_FIELDS = ` blocks.avg_fee_rate AS avgFeeRate, blocks.coinbase_raw AS coinbaseRaw, blocks.coinbase_address AS coinbaseAddress, + blocks.coinbase_addresses AS coinbaseAddresses, blocks.coinbase_signature AS coinbaseSignature, blocks.coinbase_signature_ascii AS coinbaseSignatureAscii, blocks.avg_tx_size AS avgTxSize, @@ -114,7 +116,7 @@ class BlocksRepository { pool_id, fees, fee_span, median_fee, reward, version, bits, nonce, merkle_root, previous_block_hash, avg_fee, avg_fee_rate, - median_timestamp, header, coinbase_address, + median_timestamp, header, coinbase_address, coinbase_addresses, coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, total_inputs, total_outputs, total_input_amt, total_output_amt, fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, @@ -125,7 +127,7 @@ class BlocksRepository { ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - FROM_UNIXTIME(?), ?, ?, + FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, @@ -161,6 +163,7 @@ class BlocksRepository { block.mediantime, block.extras.header, block.extras.coinbaseAddress, + block.extras.coinbaseAddresses, truncatedCoinbaseSignature, block.extras.utxoSetSize, block.extras.utxoSetChange, @@ -922,6 +925,25 @@ class BlocksRepository { } } + /** + * Get all indexed blocks with missing coinbase addresses + */ + public async $getBlocksWithoutCoinbaseAddresses(): Promise { + try { + const [blocks] = await DB.query(` + SELECT height, hash, coinbase_addresses + FROM blocks + WHERE coinbase_addresses IS NULL AND + coinbase_address IS NOT NULL + ORDER BY height DESC + `); + return blocks; + } catch (e) { + logger.err(`Cannot get blocks with missing coinbase addresses. Reason: ` + (e instanceof Error ? e.message : e)); + return []; + } + } + /** * Save indexed median fee to avoid recomputing it later * @@ -960,6 +982,25 @@ class BlocksRepository { } } + /** + * Save coinbase addresses + * + * @param id + * @param addresses + */ + public async $saveCoinbaseAddresses(id: string, addresses: string[]): Promise { + try { + await DB.query(` + UPDATE blocks SET coinbase_addresses = ? + WHERE hash = ?`, + [JSON.stringify(addresses), id] + ); + } catch (e) { + logger.err(`Cannot update block coinbase addresses. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** * Convert a mysql row block into a BlockExtended. Note that you * must provide the correct field into dbBlk object param @@ -999,6 +1040,7 @@ class BlocksRepository { extras.avgFeeRate = dbBlk.avgFeeRate; extras.coinbaseRaw = dbBlk.coinbaseRaw; extras.coinbaseAddress = dbBlk.coinbaseAddress; + extras.coinbaseAddresses = dbBlk.coinbaseAddresses; extras.coinbaseSignature = dbBlk.coinbaseSignature; extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii; extras.avgTxSize = dbBlk.avgTxSize;