From 638acffbad666384f94427b81d7affb51ac632d5 Mon Sep 17 00:00:00 2001 From: natsoni Date: Sat, 7 Sep 2024 18:01:51 +0200 Subject: [PATCH] Consider incoming transactions flow into ETA calculation --- backend/src/api/statistics/statistics-api.ts | 76 +++++++++---------- .../transaction/transaction.component.ts | 21 ++++- .../src/app/interfaces/node-api.interface.ts | 1 + frontend/src/app/services/eta.service.ts | 69 ++++++++++++++++- 4 files changed, 124 insertions(+), 43 deletions(-) diff --git a/backend/src/api/statistics/statistics-api.ts b/backend/src/api/statistics/statistics-api.ts index 50974158f..e23ebd730 100644 --- a/backend/src/api/statistics/statistics-api.ts +++ b/backend/src/api/statistics/statistics-api.ts @@ -736,44 +736,44 @@ class StatisticsApi { vsize_1600: s.vsizes[35], vsize_1800: s.vsizes[36], vsize_2000: s.vsizes[37], - vsize_ps_1: s.vsizes_ps[0] || 0, - vsize_ps_2: s.vsizes_ps[1] || 0, - vsize_ps_3: s.vsizes_ps[2] || 0, - vsize_ps_4: s.vsizes_ps[3] || 0, - vsize_ps_5: s.vsizes_ps[4] || 0, - vsize_ps_6: s.vsizes_ps[5] || 0, - vsize_ps_8: s.vsizes_ps[6] || 0, - vsize_ps_10: s.vsizes_ps[7] || 0, - vsize_ps_12: s.vsizes_ps[8] || 0, - vsize_ps_15: s.vsizes_ps[9] || 0, - vsize_ps_20: s.vsizes_ps[10] || 0, - vsize_ps_30: s.vsizes_ps[11] || 0, - vsize_ps_40: s.vsizes_ps[12] || 0, - vsize_ps_50: s.vsizes_ps[13] || 0, - vsize_ps_60: s.vsizes_ps[14] || 0, - vsize_ps_70: s.vsizes_ps[15] || 0, - vsize_ps_80: s.vsizes_ps[16] || 0, - vsize_ps_90: s.vsizes_ps[17] || 0, - vsize_ps_100: s.vsizes_ps[18] || 0, - vsize_ps_125: s.vsizes_ps[19] || 0, - vsize_ps_150: s.vsizes_ps[20] || 0, - vsize_ps_175: s.vsizes_ps[21] || 0, - vsize_ps_200: s.vsizes_ps[22] || 0, - vsize_ps_250: s.vsizes_ps[23] || 0, - vsize_ps_300: s.vsizes_ps[24] || 0, - vsize_ps_350: s.vsizes_ps[25] || 0, - vsize_ps_400: s.vsizes_ps[26] || 0, - vsize_ps_500: s.vsizes_ps[27] || 0, - vsize_ps_600: s.vsizes_ps[28] || 0, - vsize_ps_700: s.vsizes_ps[29] || 0, - vsize_ps_800: s.vsizes_ps[30] || 0, - vsize_ps_900: s.vsizes_ps[31] || 0, - vsize_ps_1000: s.vsizes_ps[32] || 0, - vsize_ps_1200: s.vsizes_ps[33] || 0, - vsize_ps_1400: s.vsizes_ps[34] || 0, - vsize_ps_1600: s.vsizes_ps[35] || 0, - vsize_ps_1800: s.vsizes_ps[36] || 0, - vsize_ps_2000: s.vsizes_ps[37] || 0, + vsize_ps_1: s.vsizes_ps?.[0] || 0, + vsize_ps_2: s.vsizes_ps?.[1] || 0, + vsize_ps_3: s.vsizes_ps?.[2] || 0, + vsize_ps_4: s.vsizes_ps?.[3] || 0, + vsize_ps_5: s.vsizes_ps?.[4] || 0, + vsize_ps_6: s.vsizes_ps?.[5] || 0, + vsize_ps_8: s.vsizes_ps?.[6] || 0, + vsize_ps_10: s.vsizes_ps?.[7] || 0, + vsize_ps_12: s.vsizes_ps?.[8] || 0, + vsize_ps_15: s.vsizes_ps?.[9] || 0, + vsize_ps_20: s.vsizes_ps?.[10] || 0, + vsize_ps_30: s.vsizes_ps?.[11] || 0, + vsize_ps_40: s.vsizes_ps?.[12] || 0, + vsize_ps_50: s.vsizes_ps?.[13] || 0, + vsize_ps_60: s.vsizes_ps?.[14] || 0, + vsize_ps_70: s.vsizes_ps?.[15] || 0, + vsize_ps_80: s.vsizes_ps?.[16] || 0, + vsize_ps_90: s.vsizes_ps?.[17] || 0, + vsize_ps_100: s.vsizes_ps?.[18] || 0, + vsize_ps_125: s.vsizes_ps?.[19] || 0, + vsize_ps_150: s.vsizes_ps?.[20] || 0, + vsize_ps_175: s.vsizes_ps?.[21] || 0, + vsize_ps_200: s.vsizes_ps?.[22] || 0, + vsize_ps_250: s.vsizes_ps?.[23] || 0, + vsize_ps_300: s.vsizes_ps?.[24] || 0, + vsize_ps_350: s.vsizes_ps?.[25] || 0, + vsize_ps_400: s.vsizes_ps?.[26] || 0, + vsize_ps_500: s.vsizes_ps?.[27] || 0, + vsize_ps_600: s.vsizes_ps?.[28] || 0, + vsize_ps_700: s.vsizes_ps?.[29] || 0, + vsize_ps_800: s.vsizes_ps?.[30] || 0, + vsize_ps_900: s.vsizes_ps?.[31] || 0, + vsize_ps_1000: s.vsizes_ps?.[32] || 0, + vsize_ps_1200: s.vsizes_ps?.[33] || 0, + vsize_ps_1400: s.vsizes_ps?.[34] || 0, + vsize_ps_1600: s.vsizes_ps?.[35] || 0, + vsize_ps_1800: s.vsizes_ps?.[36] || 0, + vsize_ps_2000: s.vsizes_ps?.[37] || 0, } }); } diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index c80006552..12b54f781 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -27,7 +27,7 @@ import { StorageService } from '../../services/storage.service'; import { seoDescriptionNetwork } from '../../shared/common.utils'; import { getTransactionFlags, getUnacceleratedFeeRate } from '../../shared/transaction.utils'; import { Filter, TransactionFlags, toFilters } from '../../shared/filters.utils'; -import { BlockExtended, CpfpInfo, RbfTree, MempoolPosition, DifficultyAdjustment, Acceleration, AccelerationPosition } from '../../interfaces/node-api.interface'; +import { BlockExtended, CpfpInfo, RbfTree, MempoolPosition, DifficultyAdjustment, Acceleration, AccelerationPosition, OptimizedMempoolStats } from '../../interfaces/node-api.interface'; import { LiquidUnblinding } from './liquid-ublinding'; import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; import { PriceService } from '../../services/price.service'; @@ -139,6 +139,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { firstLoad = true; waitingForAccelerationInfo: boolean = false; isLoadingFirstSeen = false; + mempoolStats: OptimizedMempoolStats[] = null; featuresEnabled: boolean; segwitEnabled: boolean; @@ -196,7 +197,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }); } - this.websocketService.want(['blocks', 'mempool-blocks']); + this.websocketService.want(['blocks', 'mempool-blocks', 'live-2h-chart']); this.stateService.networkChanged$.subscribe( (network) => { this.network = network; @@ -769,8 +770,20 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.stateService.difficultyAdjustment$.pipe(startWith(null)), this.isAccelerated$, this.txChanged$, + this.apiService.list2HStatistics$(), + this.stateService.live2Chart$.pipe(startWith(null)), ]).pipe( - map(([position, mempoolBlocks, da, isAccelerated]) => { + map(([position, mempoolBlocks, da, isAccelerated, _, mempoolStats, mempoolStat]) => { + + if (this.mempoolStats === null) { + this.mempoolStats = mempoolStats; + } + + if (this.mempoolStats.length && mempoolStat && this.mempoolStats[0].added !== mempoolStat.added) { + this.mempoolStats.pop(); + this.mempoolStats.unshift(mempoolStat); + } + return this.etaService.calculateETA( this.network, this.tx, @@ -780,6 +793,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.miningStats, isAccelerated, this.accelerationPositions, + this.mempoolStats, ); }) ); @@ -977,6 +991,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.isAcceleration = false; this.isAccelerated$.next(this.isAcceleration); this.eligibleForAcceleration = false; + this.mempoolStats = null; this.leaveTransaction(); } diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 3e38ff88b..cfd94c6a6 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -7,6 +7,7 @@ export interface OptimizedMempoolStats { total_fee: number; mempool_byte_weight: number; vsizes: number[]; + vsizes_ps: number[]; } interface Ancestor { diff --git a/frontend/src/app/services/eta.service.ts b/frontend/src/app/services/eta.service.ts index f632c9adb..e7fa183ef 100644 --- a/frontend/src/app/services/eta.service.ts +++ b/frontend/src/app/services/eta.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@angular/core'; -import { AccelerationPosition, CpfpInfo, DifficultyAdjustment, MempoolPosition, SinglePoolStats } from '../interfaces/node-api.interface'; +import { AccelerationPosition, CpfpInfo, DifficultyAdjustment, MempoolPosition, OptimizedMempoolStats, SinglePoolStats } from '../interfaces/node-api.interface'; import { StateService } from './state.service'; import { MempoolBlock } from '../interfaces/websocket.interface'; import { Transaction } from '../interfaces/electrs.interface'; @@ -7,6 +7,7 @@ import { MiningService, MiningStats } from './mining.service'; import { getUnacceleratedFeeRate } from '../shared/transaction.utils'; import { AccelerationEstimate } from '../components/accelerate-checkout/accelerate-checkout.component'; import { Observable, combineLatest, map, of, share, shareReplay, tap } from 'rxjs'; +import { feeLevels } from '../app.constants'; export interface ETA { now: number, // time at which calculation performed @@ -113,6 +114,7 @@ export class EtaService { miningStats: MiningStats, isAccelerated: boolean, accelerationPositions: AccelerationPosition[], + mempoolStats: OptimizedMempoolStats[] = [], ): ETA | null { // return this.calculateETA(tx, this.accelerationPositions, position, mempoolBlocks, da, isAccelerated) if (!tx || !mempoolBlocks) { @@ -143,7 +145,28 @@ export class EtaService { if (!isAccelerated) { const blocks = mempoolPosition.block + 1; - const wait = da.adjustedTimeAvg * (mempoolPosition.block + 1); + + // Estimate future incoming tx rate from mempool statistics + let vsizePerSecond = Math.min( + this.estimateVsizePerSecond(tx, mempoolStats), + 0.95 * this.stateService.blockVSize / da.adjustedTimeAvg * 1000 + ); + + // Count the number of blocks until we expect this tx to be mined + let blocksUntilMined = 0; + let mined = false; + let vsize = mempoolPosition.vsize + this.stateService.blockVSize * mempoolPosition.block; + // This loop will always terminate because we cap vsizePerSecond to 0.95 * maxCapacity + while (!mined) { + vsize = vsize + vsizePerSecond * da.adjustedTimeAvg / 1000 - this.stateService.blockVSize; + if (vsize + tx.weight / 8 < 0) { // Means that our tx fits in expected next block + mined = true; + } + blocksUntilMined++; + } + + const wait = blocksUntilMined * da.adjustedTimeAvg; + return { now, time: wait + now + da.timeOffset, @@ -279,4 +302,46 @@ export class EtaService { return tx.fee / (tx.weight / 4); } + + estimateVsizePerSecond(tx: Transaction, mempoolStats: OptimizedMempoolStats[], timeWindow: number = 15 * 60 * 1000): number { + const nowMinusTimeSpan = (new Date().getTime() - timeWindow) / 1000; + const vsizeAboveTransaction = mempoolStats + // Remove datapoints older than now - timeWindow + .filter(stat => stat.added > nowMinusTimeSpan) + // Remove datapoints less than 45 seconds apart from the previous one + .filter((el, i, arr) => { + if (i === 0) { + return true; + } + return arr[i - 1].added - el.added > 45; + }) + // For each datapoint, compute the total vsize of transactions with higher fee rate + .map(stat => { + let vsizeAbove = 0; + for (let i = feeLevels.length - 1; i >= 0; i--) { + if (feeLevels[i] > tx.effectiveFeePerVsize) { + vsizeAbove += stat.vsizes_ps[i]; + } else { + break; + } + } + return vsizeAbove; + }); + + // vsizeAboveTransaction is a temporal series of past vsize values above the transaction's fee rate + // From this array we need to estimate the future vsize per second + // Naive first approach: take the median of the series + if (!vsizeAboveTransaction.length) { + return 0; + } + + const sorted = Array.from(vsizeAboveTransaction).sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + + if (sorted.length % 2 === 0) { + return (sorted[middle - 1] + sorted[middle]) / 2; + } + + return sorted[middle]; + } }