highperfocused/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2025-04-07 19:38:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add separate effective fee stats fields

Browse Source
This commit is contained in:
Mononaut 2024-12-16 22:36:11 +00:00
parent 3c84505579
commit 2befbec7a5
No known key found for this signature in database
GPG Key ID: A3F058E41374C04E
8 changed files with 98 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
backend/src/api/blocks.ts
View File

@ -244,7 +244,7 @@ class Blocks {
*/
private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
const blk: Partial<BlockExtended> = Object.assign({}, block);
const extras: Partial<BlockExtension> = {};
@ -268,15 +268,17 @@ class Blocks {
extras.segwitTotalWeight = 0;
} else {
const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
let feeStats = {
const feeStats = {
medianFee: stats.feerate_percentiles[2], // 50th percentiles
feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(),
};
if (transactions?.length > 1) {
feeStats = Common.calcEffectiveFeeStatistics(transactions);
}
extras.medianFee = feeStats.medianFee;
extras.feeRange = feeStats.feeRange;
if (transactions?.length > 1) {
const effectiveFeeStats = Common.calcEffectiveFeeStatistics(transactions);
extras.effectiveMedianFee = effectiveFeeStats.effective_median;
extras.effectiveFeeRange = effectiveFeeStats.effective_range;
}
extras.totalFees = stats.totalfee;
extras.avgFee = stats.avgfee;
extras.avgFeeRate = stats.avgfeerate;
@ -296,7 +298,7 @@ class Blocks {
extras.medianFeeAmt = extras.feePercentiles[3];
}
}
extras.virtualSize = block.weight / 4.0;
if (coinbaseTx?.vout.length > 0) {
extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
@ -1316,6 +1318,8 @@ class Blocks {
avg_fee_rate: block.extras.avgFeeRate ?? null,
median_fee_rate: block.extras.medianFee ?? null,
fee_rate_percentiles: block.extras.feeRange ?? null,
effective_median_fee_rate: block.extras.effectiveMedianFee ?? null,
effective_fee_rate_percentiles: block.extras.effectiveFeeRange ?? null,
total_inputs: block.extras.totalInputs ?? null,
total_input_amt: block.extras.totalInputAmt ?? null,
total_outputs: block.extras.totalOutputs ?? null,
@ -1378,6 +1382,17 @@ class Blocks {
'perc_90': cleanBlock.fee_rate_percentiles[5],
'max': cleanBlock.fee_rate_percentiles[6],
};
if (cleanBlock.effective_fee_rate_percentiles) {
cleanBlock.effective_fee_rate_percentiles = {
'min': cleanBlock.effective_fee_rate_percentiles[0],
'perc_10': cleanBlock.effective_fee_rate_percentiles[1],
'perc_25': cleanBlock.effective_fee_rate_percentiles[2],
'perc_50': cleanBlock.effective_fee_rate_percentiles[3],
'perc_75': cleanBlock.effective_fee_rate_percentiles[4],
'perc_90': cleanBlock.effective_fee_rate_percentiles[5],
'max': cleanBlock.effective_fee_rate_percentiles[6],
};
}
// Re-org can happen after indexing so we need to always get the
// latest state from core

23
backend/src/api/common.ts
View File

@ -1,6 +1,6 @@
import * as bitcoinjs from 'bitcoinjs-lib';
import { Request } from 'express';
import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags, FeeStats } from '../mempool.interfaces';
import config from '../config';
import { NodeSocket } from '../repositories/NodesSocketsRepository';
import { isIP } from 'net';
@ -856,6 +856,15 @@ export class Common {
}
}
static calcFeeStatistics(transactions: { txid: string, feePerVsize: number }[]): FeeStats {
// skip the coinbase, then sort the remaining fee rates
const sortedRates = transactions.slice(1).map(tx => tx.feePerVsize).sort((a, b) => a - b);
return {
median: Math.round(Common.getNthPercentile(50, sortedRates)),
range: [0, 10, 25, 50, 75, 90, 100].map(n => Math.round(Common.getNthPercentile(n, sortedRates))),
};
}
static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats {
const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
@ -898,8 +907,8 @@ export class Common {
);
return {
medianFee: medianFeeRate,
feeRange: [
effective_median: medianFeeRate,
effective_range: [
minFee,
[10,25,50,75,90].map(n => Common.getNthPercentile(n, sortedTxs).rate),
maxFee,
@ -1150,16 +1159,16 @@ export class OnlineFeeStatsCalculator {
}
return {
minFee: this.feeRange[0].min,
medianFee: this.feeRange[Math.floor(this.feeRange.length / 2)].avg,
effective_median: this.feeRange[Math.floor(this.feeRange.length / 2)].avg,
maxFee: this.feeRange[this.feeRange.length - 1].max,
feeRange: this.feeRange.map(f => f.avg),
effective_range: this.feeRange.map(f => f.avg),
};
}
getFeeStats(): EffectiveFeeStats {
const stats = this.getRawFeeStats();
stats.feeRange[0] = stats.minFee;
stats.feeRange[stats.feeRange.length - 1] = stats.maxFee;
stats.effective_range[0] = stats.minFee;
stats.effective_range[stats.effective_range.length - 1] = stats.maxFee;
return stats;
}
}

12
backend/src/api/database-migration.ts
View File

@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 94;
private static currentVersion = 95;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -1118,6 +1118,16 @@ class DatabaseMigration {
}
await this.updateToSchemaVersion(94);
}
if (databaseSchemaVersion < 95) {
// Version 95
await this.$executeQuery(`
ALTER TABLE \`blocks\`
ADD \`effective_median_fee\` BIGINT UNSIGNED NOT NULL DEFAULT 0,
ADD \`effective_fee_span\` JSON DEFAULT NULL;
`);
await this.updateToSchemaVersion(95);
}
}
/**

3
backend/src/api/fee-api.ts
View File

@ -63,7 +63,8 @@ class FeeApi {
}
private optimizeMedianFee(pBlock: MempoolBlock, nextBlock: MempoolBlock | undefined, previousFee?: number): number {
const useFee = previousFee ? (pBlock.medianFee + previousFee) / 2 : pBlock.medianFee;
const medianFee = pBlock.effectiveMedianFee ?? pBlock.medianFee;
const useFee = previousFee ? (medianFee + previousFee) / 2 : medianFee;
if (pBlock.blockVSize <= 500000) {
return this.defaultFee;
}

19
backend/src/api/mempool-blocks.ts
View File

@ -1,6 +1,6 @@
import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
import logger from '../logger';
import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates, PoolTag } from '../mempool.interfaces';
import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, FeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates, PoolTag, EffectiveFeeStats } from '../mempool.interfaces';
import { Common, OnlineFeeStatsCalculator } from './common';
import config from '../config';
import { Worker } from 'worker_threads';
@ -33,6 +33,8 @@ class MempoolBlocks {
totalFees: block.totalFees,
medianFee: block.medianFee,
feeRange: block.feeRange,
effectiveMedianFee: block.effectiveMedianFee,
effectiveFeeRange: block.effectiveFeeRange,
};
});
}
@ -527,7 +529,7 @@ class MempoolBlocks {
totalSize,
totalWeight,
totalFees,
(hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
(hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getFeeStats() : undefined,
);
};
@ -541,17 +543,20 @@ class MempoolBlocks {
return mempoolBlocks;
}
private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
if (!feeStats) {
feeStats = Common.calcEffectiveFeeStatistics(transactions);
private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, effectiveFeeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
const feeStats = Common.calcFeeStatistics(transactions);
if (!effectiveFeeStats) {
effectiveFeeStats = Common.calcEffectiveFeeStatistics(transactions);
}
return {
blockSize: totalSize,
blockVSize: (totalWeight / 4), // fractional vsize to avoid rounding errors
nTx: transactionIds.length,
totalFees: totalFees,
medianFee: feeStats.medianFee, // Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
feeRange: feeStats.feeRange, //Common.getFeesInRange(transactions, rangeLength),
medianFee: feeStats.median,
feeRange: feeStats.range,
effectiveMedianFee: effectiveFeeStats.effective_median,
effectiveFeeRange: effectiveFeeStats.effective_range,
transactionIds: transactionIds,
transactions: transactions.map((tx) => Common.classifyTransaction(tx)),
};

21
backend/src/mempool.interfaces.ts
View File

@ -73,6 +73,8 @@ export interface MempoolBlock {
medianFee: number;
totalFees: number;
feeRange: number[];
effectiveMedianFee?: number;
effectiveFeeRange?: number[];
}
export interface MempoolBlockWithTransactions extends MempoolBlock {
@ -288,8 +290,10 @@ export const TransactionFlags = {
export interface BlockExtension {
totalFees: number;
medianFee: number; // median fee rate
feeRange: number[]; // fee rate percentiles
medianFee: number; // core median fee rate
feeRange: number[]; // core fee rate percentiles
effectiveMedianFee?: number; // effective median fee rate
effectiveFeeRange?: number[]; // effective fee rate percentiles
reward: number;
matchRate: number | null;
expectedFees: number | null;
@ -369,9 +373,18 @@ export interface MempoolStats {
tx_count: number;
}
// Core fee stats
// measured in individual sats/vbyte
export interface FeeStats {
median: number; // median core fee rate
range: number[]; // 0th, 10th, 25th, 50th, 75th, 90th, 100th percentiles
}
// Mempool effective fee stats
// measured in effective sats/vbyte
export interface EffectiveFeeStats {
medianFee: number; // median effective fee rate
feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
effective_median: number; // median effective fee rate by weight
effective_range: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
}
export interface WorkingEffectiveFeeStats extends EffectiveFeeStats {

4
backend/src/repositories/AccelerationRepository.ts
View File

@ -315,12 +315,12 @@ class AccelerationRepository {
Infinity
);
const feeStats = Common.calcEffectiveFeeStatistics(template);
boostRate = feeStats.medianFee;
boostRate = feeStats.effective_median;
}
const accelerationSummaries = accelerations.map(acc => ({
...acc,
pools: acc.pools,
}))
}));
for (const acc of accelerations) {
if (blockTxs[acc.txid] && acc.pools.includes(block.extras.pool.id)) {
const tx = blockTxs[acc.txid];

24
backend/src/repositories/BlocksRepository.ts
View File

@ -33,6 +33,8 @@ interface DatabaseBlock {
totalFees: number;
medianFee: number;
feeRange: string;
effectiveMedianFee?: number;
effectiveFeeRange?: string;
reward: number;
poolId: number;
poolName: string;
@ -77,6 +79,8 @@ const BLOCK_DB_FIELDS = `
blocks.fees AS totalFees,
blocks.median_fee AS medianFee,
blocks.fee_span AS feeRange,
blocks.effective_median_fee AS effectiveMedianFee,
blocks.effective_fee_span AS effectiveFeeRange,
blocks.reward,
pools.unique_id AS poolId,
pools.name AS poolName,
@ -108,7 +112,7 @@ class BlocksRepository {
/**
* Save indexed block data in the database
*/
public async $saveBlockInDatabase(block: BlockExtended) {
public async $saveBlockInDatabase(block: BlockExtended): Promise<void> {
const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
@ -117,6 +121,7 @@ class BlocksRepository {
height, hash, blockTimestamp, size,
weight, tx_count, coinbase_raw, difficulty,
pool_id, fees, fee_span, median_fee,
effective_fee_span, effective_median_fee,
reward, version, bits, nonce,
merkle_root, previous_block_hash, avg_fee, avg_fee_rate,
median_timestamp, header, coinbase_address, coinbase_addresses,
@ -128,6 +133,7 @@ class BlocksRepository {
?, ?, FROM_UNIXTIME(?), ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
FROM_UNIXTIME(?), ?, ?, ?,
@ -155,6 +161,8 @@ class BlocksRepository {
block.extras.totalFees,
JSON.stringify(block.extras.feeRange),
block.extras.medianFee,
block.extras.effectiveFeeRange ? JSON.stringify(block.extras.effectiveFeeRange) : null,
block.extras.effectiveMedianFee,
block.extras.reward,
block.version,
block.bits,
@ -968,16 +976,16 @@ class BlocksRepository {
/**
* Save indexed effective fee statistics
*
* @param id
* @param feeStats
*
* @param id
* @param feeStats
*/
public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise<void> {
try {
await DB.query(`
UPDATE blocks SET median_fee = ?, fee_span = ?
UPDATE blocks SET effective_median_fee = ?, effective_fee_span = ?
WHERE hash = ?`,
[feeStats.medianFee, JSON.stringify(feeStats.feeRange), id]
[feeStats.effective_median, JSON.stringify(feeStats.effective_range), id]
);
} catch (e) {
logger.err(`Cannot update block fee stats. Reason: ` + (e instanceof Error ? e.message : e));
@ -1065,11 +1073,13 @@ class BlocksRepository {
blk.weight = dbBlk.weight;
blk.previousblockhash = dbBlk.previousblockhash;
blk.mediantime = dbBlk.mediantime;
// BlockExtension
extras.totalFees = dbBlk.totalFees;
extras.medianFee = dbBlk.medianFee;
extras.feeRange = JSON.parse(dbBlk.feeRange);
extras.effectiveMedianFee = dbBlk.effectiveMedianFee;
extras.effectiveFeeRange = dbBlk.effectiveFeeRange ? JSON.parse(dbBlk.effectiveFeeRange) : null;
extras.reward = dbBlk.reward;
extras.pool = {
id: dbBlk.poolId,