From 832ccdac46442c4c1c7b6e8d324a4c0a0d84a391 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 19 Oct 2022 17:10:45 +0000 Subject: [PATCH] improve audit analysis and scoring --- backend/src/api/audit.ts | 114 +++++++++++++++++++++++++++ backend/src/api/websocket-handler.ts | 28 +++---- 2 files changed, 123 insertions(+), 19 deletions(-) create mode 100644 backend/src/api/audit.ts diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts new file mode 100644 index 000000000..6efb50938 --- /dev/null +++ b/backend/src/api/audit.ts @@ -0,0 +1,114 @@ +import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; + +const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners + +class Audit { + auditBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[], + projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended }, + ): { censored: string[], added: string[], score: number } { + const matches: string[] = []; // present in both mined block and template + const added: string[] = []; // present in mined block, not in template + const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN + const isCensored = {}; // missing, without excuse + const isDisplaced = {}; + let displacedWeight = 0; + + const inBlock = {}; + const inTemplate = {}; + + const now = Math.round((Date.now() / 1000)); + for (const tx of transactions) { + inBlock[tx.txid] = tx; + } + // coinbase is always expected + if (transactions[0]) { + inTemplate[transactions[0].txid] = true; + } + // look for transactions that were expected in the template, but missing from the mined block + for (const txid of projectedBlocks[0].transactionIds) { + if (!inBlock[txid]) { + // tx is recent, may have reached the miner too late for inclusion + if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) { + fresh.push(txid); + } else { + isCensored[txid] = true; + } + displacedWeight += mempool[txid].weight; + } + inTemplate[txid] = true; + } + + displacedWeight += (4000 - transactions[0].weight); + + logger.warn(`${fresh.length} fresh, ${Object.keys(isCensored).length} possibly censored, ${displacedWeight} displaced weight`); + + // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs + // these displaced transactions should occupy the first N weight units of the next projected block + let displacedWeightRemaining = displacedWeight; + let index = 0; + let lastFeeRate = Infinity; + let failures = 0; + while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) { + const txid = projectedBlocks[1].transactionIds[index]; + const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000; + const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate; + if (fits || feeMatches) { + isDisplaced[txid] = true; + if (fits) { + lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize); + } + if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) { + displacedWeightRemaining -= mempool[txid].weight; + } + failures = 0; + } else { + failures++; + } + index++; + } + + // mark unexpected transactions in the mined block as 'added' + let overflowWeight = 0; + for (const tx of transactions) { + if (inTemplate[tx.txid]) { + matches.push(tx.txid); + } else { + if (!isDisplaced[tx.txid]) { + added.push(tx.txid); + } + overflowWeight += tx.weight; + } + } + + // transactions missing from near the end of our template are probably not being censored + let overflowWeightRemaining = overflowWeight; + let lastOverflowRate = 1.00; + index = projectedBlocks[0].transactionIds.length - 1; + while (index >= 0) { + const txid = projectedBlocks[0].transactionIds[index]; + if (overflowWeightRemaining > 0) { + if (isCensored[txid]) { + delete isCensored[txid]; + } + lastOverflowRate = mempool[txid].effectiveFeePerVsize; + } else if (Math.floor(mempool[txid].effectiveFeePerVsize * 100) <= Math.ceil(lastOverflowRate * 100)) { // tolerance of 0.01 sat/vb + if (isCensored[txid]) { + delete isCensored[txid]; + } + } + overflowWeightRemaining -= (mempool[txid]?.weight || 0); + index--; + } + + const numCensored = Object.keys(isCensored).length; + const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0; + + return { + censored: Object.keys(isCensored), + added, + score + }; + } +} + +export default new Audit(); \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index f183a4799..c3d4dad9b 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -18,6 +18,7 @@ import difficultyAdjustment from './difficulty-adjustment'; import feeApi from './fee-api'; import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; +import Audit from './audit'; class WebsocketHandler { private wss: WebSocket.Server | undefined; @@ -405,7 +406,7 @@ class WebsocketHandler { }); } - handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) { + handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): void { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } @@ -414,30 +415,19 @@ class WebsocketHandler { let mBlockDeltas: undefined | MempoolBlockDelta[]; let matchRate = 0; const _memPool = memPool.getMempool(); - const projectedBlocks = mempoolBlocks.makeBlockTemplates(cloneMempool(_memPool), 1, 1); + const mempoolCopy = cloneMempool(_memPool); + + const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2, 2); if (projectedBlocks[0]) { - const matches: string[] = []; - const added: string[] = []; - const missing: string[] = []; + const { censored, added, score } = Audit.auditBlock(block, txIds, transactions, projectedBlocks, mempoolCopy); + matchRate = Math.round(score * 100 * 100) / 100; + // Update mempool to remove transactions included in the new block for (const txId of txIds) { - if (projectedBlocks[0].transactionIds.indexOf(txId) > -1) { - matches.push(txId); - } else { - added.push(txId); - } delete _memPool[txId]; } - for (const txId of projectedBlocks[0].transactionIds) { - if (matches.includes(txId) || added.includes(txId)) { - continue; - } - missing.push(txId); - } - - matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100; mempoolBlocks.updateMempoolBlocks(_memPool); mBlocks = mempoolBlocks.getMempoolBlocks(); mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); @@ -464,7 +454,7 @@ class WebsocketHandler { height: block.height, hash: block.id, addedTxs: added, - missingTxs: missing, + missingTxs: censored, matchRate: matchRate, }); }