diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index e621056ab..542074743 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -244,7 +244,7 @@ class Blocks { */ private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - + const blk: Partial = Object.assign({}, block); const extras: Partial = {}; @@ -268,15 +268,17 @@ class Blocks { extras.segwitTotalWeight = 0; } else { const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id); - let feeStats = { + const feeStats = { medianFee: stats.feerate_percentiles[2], // 50th percentiles feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(), }; - if (transactions?.length > 1) { - feeStats = Common.calcEffectiveFeeStatistics(transactions); - } extras.medianFee = feeStats.medianFee; extras.feeRange = feeStats.feeRange; + if (transactions?.length > 1) { + const effectiveFeeStats = Common.calcEffectiveFeeStatistics(transactions); + extras.effectiveMedianFee = effectiveFeeStats.effective_median; + extras.effectiveFeeRange = effectiveFeeStats.effective_range; + } extras.totalFees = stats.totalfee; extras.avgFee = stats.avgfee; extras.avgFeeRate = stats.avgfeerate; @@ -296,7 +298,7 @@ class Blocks { extras.medianFeeAmt = extras.feePercentiles[3]; } } - + extras.virtualSize = block.weight / 4.0; if (coinbaseTx?.vout.length > 0) { extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null; @@ -1316,6 +1318,8 @@ class Blocks { avg_fee_rate: block.extras.avgFeeRate ?? null, median_fee_rate: block.extras.medianFee ?? null, fee_rate_percentiles: block.extras.feeRange ?? null, + effective_median_fee_rate: block.extras.effectiveMedianFee ?? null, + effective_fee_rate_percentiles: block.extras.effectiveFeeRange ?? null, total_inputs: block.extras.totalInputs ?? null, total_input_amt: block.extras.totalInputAmt ?? null, total_outputs: block.extras.totalOutputs ?? null, @@ -1378,6 +1382,17 @@ class Blocks { 'perc_90': cleanBlock.fee_rate_percentiles[5], 'max': cleanBlock.fee_rate_percentiles[6], }; + if (cleanBlock.effective_fee_rate_percentiles) { + cleanBlock.effective_fee_rate_percentiles = { + 'min': cleanBlock.effective_fee_rate_percentiles[0], + 'perc_10': cleanBlock.effective_fee_rate_percentiles[1], + 'perc_25': cleanBlock.effective_fee_rate_percentiles[2], + 'perc_50': cleanBlock.effective_fee_rate_percentiles[3], + 'perc_75': cleanBlock.effective_fee_rate_percentiles[4], + 'perc_90': cleanBlock.effective_fee_rate_percentiles[5], + 'max': cleanBlock.effective_fee_rate_percentiles[6], + }; + } // Re-org can happen after indexing so we need to always get the // latest state from core diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 50de63afc..48987156d 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,6 +1,6 @@ import * as bitcoinjs from 'bitcoinjs-lib'; import { Request } from 'express'; -import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces'; +import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags, FeeStats } from '../mempool.interfaces'; import config from '../config'; import { NodeSocket } from '../repositories/NodesSocketsRepository'; import { isIP } from 'net'; @@ -856,6 +856,15 @@ export class Common { } } + static calcFeeStatistics(transactions: { txid: string, feePerVsize: number }[]): FeeStats { + // skip the coinbase, then sort the remaining fee rates + const sortedRates = transactions.slice(1).map(tx => tx.feePerVsize).sort((a, b) => a - b); + return { + median: Math.round(Common.getNthPercentile(50, sortedRates)), + range: [0, 10, 25, 50, 75, 90, 100].map(n => Math.round(Common.getNthPercentile(n, sortedRates))), + }; + } + static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats { const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); @@ -898,8 +907,8 @@ export class Common { ); return { - medianFee: medianFeeRate, - feeRange: [ + effective_median: medianFeeRate, + effective_range: [ minFee, [10,25,50,75,90].map(n => Common.getNthPercentile(n, sortedTxs).rate), maxFee, @@ -1150,16 +1159,16 @@ export class OnlineFeeStatsCalculator { } return { minFee: this.feeRange[0].min, - medianFee: this.feeRange[Math.floor(this.feeRange.length / 2)].avg, + effective_median: this.feeRange[Math.floor(this.feeRange.length / 2)].avg, maxFee: this.feeRange[this.feeRange.length - 1].max, - feeRange: this.feeRange.map(f => f.avg), + effective_range: this.feeRange.map(f => f.avg), }; } getFeeStats(): EffectiveFeeStats { const stats = this.getRawFeeStats(); - stats.feeRange[0] = stats.minFee; - stats.feeRange[stats.feeRange.length - 1] = stats.maxFee; + stats.effective_range[0] = stats.minFee; + stats.effective_range[stats.effective_range.length - 1] = stats.maxFee; return stats; } } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index dc8c7291a..05af613cf 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 94; + private static currentVersion = 95; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -1118,6 +1118,16 @@ class DatabaseMigration { } await this.updateToSchemaVersion(94); } + + if (databaseSchemaVersion < 95) { + // Version 95 + await this.$executeQuery(` + ALTER TABLE \`blocks\` + ADD \`effective_median_fee\` BIGINT UNSIGNED NOT NULL DEFAULT 0, + ADD \`effective_fee_span\` JSON DEFAULT NULL; + `); + await this.updateToSchemaVersion(95); + } } /** diff --git a/backend/src/api/fee-api.ts b/backend/src/api/fee-api.ts index 24fd25a4b..f120f8b8f 100644 --- a/backend/src/api/fee-api.ts +++ b/backend/src/api/fee-api.ts @@ -63,7 +63,8 @@ class FeeApi { } private optimizeMedianFee(pBlock: MempoolBlock, nextBlock: MempoolBlock | undefined, previousFee?: number): number { - const useFee = previousFee ? (pBlock.medianFee + previousFee) / 2 : pBlock.medianFee; + const medianFee = pBlock.effectiveMedianFee ?? pBlock.medianFee; + const useFee = previousFee ? (medianFee + previousFee) / 2 : medianFee; if (pBlock.blockVSize <= 500000) { return this.defaultFee; } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index ba4ce2ed0..8f43b1c94 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,6 +1,6 @@ import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt'; import logger from '../logger'; -import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates, PoolTag } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, FeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates, PoolTag, EffectiveFeeStats } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; @@ -33,6 +33,8 @@ class MempoolBlocks { totalFees: block.totalFees, medianFee: block.medianFee, feeRange: block.feeRange, + effectiveMedianFee: block.effectiveMedianFee, + effectiveFeeRange: block.effectiveFeeRange, }; }); } @@ -527,7 +529,7 @@ class MempoolBlocks { totalSize, totalWeight, totalFees, - (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined, + (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getFeeStats() : undefined, ); }; @@ -541,17 +543,20 @@ class MempoolBlocks { return mempoolBlocks; } - private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { - if (!feeStats) { - feeStats = Common.calcEffectiveFeeStatistics(transactions); + private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, effectiveFeeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { + const feeStats = Common.calcFeeStatistics(transactions); + if (!effectiveFeeStats) { + effectiveFeeStats = Common.calcEffectiveFeeStatistics(transactions); } return { blockSize: totalSize, blockVSize: (totalWeight / 4), // fractional vsize to avoid rounding errors nTx: transactionIds.length, totalFees: totalFees, - medianFee: feeStats.medianFee, // Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE), - feeRange: feeStats.feeRange, //Common.getFeesInRange(transactions, rangeLength), + medianFee: feeStats.median, + feeRange: feeStats.range, + effectiveMedianFee: effectiveFeeStats.effective_median, + effectiveFeeRange: effectiveFeeStats.effective_range, transactionIds: transactionIds, transactions: transactions.map((tx) => Common.classifyTransaction(tx)), }; diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index dc703af21..b86ddaa55 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -73,6 +73,8 @@ export interface MempoolBlock { medianFee: number; totalFees: number; feeRange: number[]; + effectiveMedianFee?: number; + effectiveFeeRange?: number[]; } export interface MempoolBlockWithTransactions extends MempoolBlock { @@ -288,8 +290,10 @@ export const TransactionFlags = { export interface BlockExtension { totalFees: number; - medianFee: number; // median fee rate - feeRange: number[]; // fee rate percentiles + medianFee: number; // core median fee rate + feeRange: number[]; // core fee rate percentiles + effectiveMedianFee?: number; // effective median fee rate + effectiveFeeRange?: number[]; // effective fee rate percentiles reward: number; matchRate: number | null; expectedFees: number | null; @@ -369,9 +373,18 @@ export interface MempoolStats { tx_count: number; } +// Core fee stats +// measured in individual sats/vbyte +export interface FeeStats { + median: number; // median core fee rate + range: number[]; // 0th, 10th, 25th, 50th, 75th, 90th, 100th percentiles +} + +// Mempool effective fee stats +// measured in effective sats/vbyte export interface EffectiveFeeStats { - medianFee: number; // median effective fee rate - feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles + effective_median: number; // median effective fee rate by weight + effective_range: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles } export interface WorkingEffectiveFeeStats extends EffectiveFeeStats { diff --git a/backend/src/repositories/AccelerationRepository.ts b/backend/src/repositories/AccelerationRepository.ts index 4c9896296..1e4609a72 100644 --- a/backend/src/repositories/AccelerationRepository.ts +++ b/backend/src/repositories/AccelerationRepository.ts @@ -315,12 +315,12 @@ class AccelerationRepository { Infinity ); const feeStats = Common.calcEffectiveFeeStatistics(template); - boostRate = feeStats.medianFee; + boostRate = feeStats.effective_median; } const accelerationSummaries = accelerations.map(acc => ({ ...acc, pools: acc.pools, - })) + })); for (const acc of accelerations) { if (blockTxs[acc.txid] && acc.pools.includes(block.extras.pool.id)) { const tx = blockTxs[acc.txid]; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 424a668c7..c6793cde3 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -33,6 +33,8 @@ interface DatabaseBlock { totalFees: number; medianFee: number; feeRange: string; + effectiveMedianFee?: number; + effectiveFeeRange?: string; reward: number; poolId: number; poolName: string; @@ -77,6 +79,8 @@ const BLOCK_DB_FIELDS = ` blocks.fees AS totalFees, blocks.median_fee AS medianFee, blocks.fee_span AS feeRange, + blocks.effective_median_fee AS effectiveMedianFee, + blocks.effective_fee_span AS effectiveFeeRange, blocks.reward, pools.unique_id AS poolId, pools.name AS poolName, @@ -108,7 +112,7 @@ class BlocksRepository { /** * Save indexed block data in the database */ - public async $saveBlockInDatabase(block: BlockExtended) { + public async $saveBlockInDatabase(block: BlockExtended): Promise { const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500); const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500); @@ -117,6 +121,7 @@ class BlocksRepository { height, hash, blockTimestamp, size, weight, tx_count, coinbase_raw, difficulty, pool_id, fees, fee_span, median_fee, + effective_fee_span, effective_median_fee, reward, version, bits, nonce, merkle_root, previous_block_hash, avg_fee, avg_fee_rate, median_timestamp, header, coinbase_address, coinbase_addresses, @@ -128,6 +133,7 @@ class BlocksRepository { ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?), ?, ?, ?, @@ -155,6 +161,8 @@ class BlocksRepository { block.extras.totalFees, JSON.stringify(block.extras.feeRange), block.extras.medianFee, + block.extras.effectiveFeeRange ? JSON.stringify(block.extras.effectiveFeeRange) : null, + block.extras.effectiveMedianFee, block.extras.reward, block.version, block.bits, @@ -968,16 +976,16 @@ class BlocksRepository { /** * Save indexed effective fee statistics - * - * @param id - * @param feeStats + * + * @param id + * @param feeStats */ public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise { try { await DB.query(` - UPDATE blocks SET median_fee = ?, fee_span = ? + UPDATE blocks SET effective_median_fee = ?, effective_fee_span = ? WHERE hash = ?`, - [feeStats.medianFee, JSON.stringify(feeStats.feeRange), id] + [feeStats.effective_median, JSON.stringify(feeStats.effective_range), id] ); } catch (e) { logger.err(`Cannot update block fee stats. Reason: ` + (e instanceof Error ? e.message : e)); @@ -1065,11 +1073,13 @@ class BlocksRepository { blk.weight = dbBlk.weight; blk.previousblockhash = dbBlk.previousblockhash; blk.mediantime = dbBlk.mediantime; - + // BlockExtension extras.totalFees = dbBlk.totalFees; extras.medianFee = dbBlk.medianFee; extras.feeRange = JSON.parse(dbBlk.feeRange); + extras.effectiveMedianFee = dbBlk.effectiveMedianFee; + extras.effectiveFeeRange = dbBlk.effectiveFeeRange ? JSON.parse(dbBlk.effectiveFeeRange) : null; extras.reward = dbBlk.reward; extras.pool = { id: dbBlk.poolId,