diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index eea96af69..e09234cdc 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -2,6 +2,7 @@ import config from '../config'; import logger from '../logger'; import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; import rbfCache from './rbf-cache'; +import transactionUtils from './transaction-utils'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners @@ -15,7 +16,8 @@ class Audit { const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const unseen: string[] = []; // present in the mined block, not in our mempool - const prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone + let prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone + let deprioritized: string[] = []; // lower in the block than would be expected by in-band feerate alone const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block const accelerated: string[] = []; // prioritized by the mempool accelerator @@ -133,23 +135,7 @@ class Audit { totalWeight += tx.weight; } - - // identify "prioritized" transactions - let lastEffectiveRate = 0; - // Iterate over the mined template from bottom to top (excluding the coinbase) - // Transactions should appear in ascending order of mining priority. - for (let i = transactions.length - 1; i > 0; i--) { - const blockTx = transactions[i]; - // If a tx has a lower in-band effective fee rate than the previous tx, - // it must have been prioritized out-of-band (in order to have a higher mining priority) - // so exclude from the analysis. - if ((blockTx.effectiveFeePerVsize || 0) < lastEffectiveRate) { - prioritized.push(blockTx.txid); - // accelerated txs may or may not have their prioritized fee rate applied, so don't use them as a reference - } else if (!isAccelerated[blockTx.txid]) { - lastEffectiveRate = blockTx.effectiveFeePerVsize || 0; - } - } + ({ prioritized, deprioritized } = transactionUtils.identifyPrioritizedTransactions(transactions, 'effectiveFeePerVsize')); // transactions missing from near the end of our template are probably not being censored let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index b3077b935..15d3e7110 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -338,6 +338,87 @@ class TransactionUtils { const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2; return witness[positionOfScript]; } + + // calculate the most parsimonious set of prioritizations given a list of block transactions + // (i.e. the most likely prioritizations and deprioritizations) + public identifyPrioritizedTransactions(transactions: any[], rateKey: string): { prioritized: string[], deprioritized: string[] } { + // find the longest increasing subsequence of transactions + // (adapted from https://en.wikipedia.org/wiki/Longest_increasing_subsequence#Efficient_algorithms) + // should be O(n log n) + const X = transactions.slice(1).reverse().map((tx) => ({ txid: tx.txid, rate: tx[rateKey] })); // standard block order is by *decreasing* effective fee rate, but we want to iterate in increasing order (and skip the coinbase) + if (X.length < 2) { + return { prioritized: [], deprioritized: [] }; + } + const N = X.length; + const P: number[] = new Array(N); + const M: number[] = new Array(N + 1); + M[0] = -1; // undefined so can be set to any value + + let L = 0; + for (let i = 0; i < N; i++) { + // Binary search for the smallest positive l ≤ L + // such that X[M[l]].effectiveFeePerVsize > X[i].effectiveFeePerVsize + let lo = 1; + let hi = L + 1; + while (lo < hi) { + const mid = lo + Math.floor((hi - lo) / 2); // lo <= mid < hi + if (X[M[mid]].rate > X[i].rate) { + hi = mid; + } else { // if X[M[mid]].effectiveFeePerVsize < X[i].effectiveFeePerVsize + lo = mid + 1; + } + } + + // After searching, lo == hi is 1 greater than the + // length of the longest prefix of X[i] + const newL = lo; + + // The predecessor of X[i] is the last index of + // the subsequence of length newL-1 + P[i] = M[newL - 1]; + M[newL] = i; + + if (newL > L) { + // If we found a subsequence longer than any we've + // found yet, update L + L = newL; + } + } + + // Reconstruct the longest increasing subsequence + // It consists of the values of X at the L indices: + // ..., P[P[M[L]]], P[M[L]], M[L] + const LIS: any[] = new Array(L); + let k = M[L]; + for (let j = L - 1; j >= 0; j--) { + LIS[j] = X[k]; + k = P[k]; + } + + const lisMap = new Map(); + LIS.forEach((tx, index) => lisMap.set(tx.txid, index)); + + const prioritized: string[] = []; + const deprioritized: string[] = []; + + let lastRate = X[0].rate; + + for (const tx of X) { + if (lisMap.has(tx.txid)) { + lastRate = tx.rate; + } else { + if (Math.abs(tx.rate - lastRate) < 0.1) { + // skip if the rate is almost the same as the previous transaction + } else if (tx.rate <= lastRate) { + prioritized.push(tx.txid); + } else { + deprioritized.push(tx.txid); + } + } + } + + return { prioritized, deprioritized }; + } } export default new TransactionUtils(); diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index ad24b26c3..f612368f4 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -33,7 +33,7 @@ export default class TxView implements TransactionStripped { flags: number; bigintFlags?: bigint | null = 0b00000100_00000000_00000000_00000000n; time?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'added_deprioritized' | 'deprioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; scene?: BlockScene; diff --git a/frontend/src/app/components/block-overview-graph/utils.ts b/frontend/src/app/components/block-overview-graph/utils.ts index 4f7c7ed5a..625029db0 100644 --- a/frontend/src/app/components/block-overview-graph/utils.ts +++ b/frontend/src/app/components/block-overview-graph/utils.ts @@ -142,6 +142,10 @@ export function defaultColorFunction( return auditColors.added_prioritized; case 'prioritized': return auditColors.prioritized; + case 'added_deprioritized': + return auditColors.added_prioritized; + case 'deprioritized': + return auditColors.prioritized; case 'selected': return colors.marginal[levelIndex] || colors.marginal[defaultMempoolFeeColors.length - 1]; case 'accelerated': diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 037229398..f1f5bb3d4 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -79,6 +79,11 @@ Added Prioritized + Deprioritized + + Added + Deprioritized + Marginal fee rate Conflict Accelerated diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 44328c591..5cba85e90 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -17,6 +17,7 @@ import { PriceService, Price } from '../../services/price.service'; import { CacheService } from '../../services/cache.service'; import { ServicesApiServices } from '../../services/services-api.service'; import { PreloadService } from '../../services/preload.service'; +import { identifyPrioritizedTransactions } from '../../shared/transaction.utils'; @Component({ selector: 'app-block', @@ -524,6 +525,7 @@ export class BlockComponent implements OnInit, OnDestroy { const isUnseen = {}; const isAdded = {}; const isPrioritized = {}; + const isDeprioritized = {}; const isCensored = {}; const isMissing = {}; const isSelected = {}; @@ -535,6 +537,17 @@ export class BlockComponent implements OnInit, OnDestroy { this.numUnexpected = 0; if (blockAudit?.template) { + // augment with locally calculated *de*prioritized transactions if possible + const { prioritized, deprioritized } = identifyPrioritizedTransactions(transactions); + // but if the local calculation produces returns unexpected results, don't use it + let useLocalDeprioritized = deprioritized.length < (transactions.length * 0.1); + for (const tx of prioritized) { + if (!isPrioritized[tx] && !isAccelerated[tx]) { + useLocalDeprioritized = false; + break; + } + } + for (const tx of blockAudit.template) { inTemplate[tx.txid] = true; if (tx.acc) { @@ -550,9 +563,14 @@ export class BlockComponent implements OnInit, OnDestroy { for (const txid of blockAudit.addedTxs) { isAdded[txid] = true; } - for (const txid of blockAudit.prioritizedTxs || []) { + for (const txid of blockAudit.prioritizedTxs) { isPrioritized[txid] = true; } + if (useLocalDeprioritized) { + for (const txid of deprioritized || []) { + isDeprioritized[txid] = true; + } + } for (const txid of blockAudit.missingTxs) { isCensored[txid] = true; } @@ -608,6 +626,12 @@ export class BlockComponent implements OnInit, OnDestroy { } else { tx.status = 'prioritized'; } + } else if (isDeprioritized[tx.txid]) { + if (isAdded[tx.txid] || (blockAudit.version > 0 && isUnseen[tx.txid])) { + tx.status = 'added_deprioritized'; + } else { + tx.status = 'deprioritized'; + } } else if (isAdded[tx.txid] && (blockAudit.version === 0 || isUnseen[tx.txid])) { tx.status = 'added'; } else if (inTemplate[tx.txid]) { diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 4d2ffc09a..3e38ff88b 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -239,7 +239,7 @@ export interface TransactionStripped { acc?: boolean; flags?: number | null; time?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'added_deprioritized' | 'deprioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/shared/transaction.utils.ts b/frontend/src/app/shared/transaction.utils.ts index 9d9cd801b..c13616c60 100644 --- a/frontend/src/app/shared/transaction.utils.ts +++ b/frontend/src/app/shared/transaction.utils.ts @@ -1,7 +1,7 @@ import { TransactionFlags } from './filters.utils'; import { getVarIntLength, opcodes, parseMultisigScript, isPoint } from './script.utils'; import { Transaction } from '../interfaces/electrs.interface'; -import { CpfpInfo, RbfInfo } from '../interfaces/node-api.interface'; +import { CpfpInfo, RbfInfo, TransactionStripped } from '../interfaces/node-api.interface'; // Bitcoin Core default policy settings const TX_MAX_STANDARD_VERSION = 2; @@ -458,4 +458,83 @@ export function getUnacceleratedFeeRate(tx: Transaction, accelerated: boolean): } else { return tx.effectiveFeePerVsize; } +} + +export function identifyPrioritizedTransactions(transactions: TransactionStripped[]): { prioritized: string[], deprioritized: string[] } { + // find the longest increasing subsequence of transactions + // (adapted from https://en.wikipedia.org/wiki/Longest_increasing_subsequence#Efficient_algorithms) + // should be O(n log n) + const X = transactions.slice(1).reverse(); // standard block order is by *decreasing* effective fee rate, but we want to iterate in increasing order (and skip the coinbase) + if (X.length < 2) { + return { prioritized: [], deprioritized: [] }; + } + const N = X.length; + const P: number[] = new Array(N); + const M: number[] = new Array(N + 1); + M[0] = -1; // undefined so can be set to any value + + let L = 0; + for (let i = 0; i < N; i++) { + // Binary search for the smallest positive l ≤ L + // such that X[M[l]].effectiveFeePerVsize > X[i].effectiveFeePerVsize + let lo = 1; + let hi = L + 1; + while (lo < hi) { + const mid = lo + Math.floor((hi - lo) / 2); // lo <= mid < hi + if (X[M[mid]].rate > X[i].rate) { + hi = mid; + } else { // if X[M[mid]].effectiveFeePerVsize < X[i].effectiveFeePerVsize + lo = mid + 1; + } + } + + // After searching, lo == hi is 1 greater than the + // length of the longest prefix of X[i] + const newL = lo; + + // The predecessor of X[i] is the last index of + // the subsequence of length newL-1 + P[i] = M[newL - 1]; + M[newL] = i; + + if (newL > L) { + // If we found a subsequence longer than any we've + // found yet, update L + L = newL; + } + } + + // Reconstruct the longest increasing subsequence + // It consists of the values of X at the L indices: + // ..., P[P[M[L]]], P[M[L]], M[L] + const LIS: TransactionStripped[] = new Array(L); + let k = M[L]; + for (let j = L - 1; j >= 0; j--) { + LIS[j] = X[k]; + k = P[k]; + } + + const lisMap = new Map(); + LIS.forEach((tx, index) => lisMap.set(tx.txid, index)); + + const prioritized: string[] = []; + const deprioritized: string[] = []; + + let lastRate = 0; + + for (const tx of X) { + if (lisMap.has(tx.txid)) { + lastRate = tx.rate; + } else { + if (Math.abs(tx.rate - lastRate) < 0.1) { + // skip if the rate is almost the same as the previous transaction + } else if (tx.rate <= lastRate) { + prioritized.push(tx.txid); + } else { + deprioritized.push(tx.txid); + } + } + } + + return { prioritized, deprioritized }; } \ No newline at end of file