diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index c31fe1a09..8f31e152d 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -399,9 +399,13 @@ class BitcoinRoutes { private async getBlockAuditSummary(req: Request, res: Response) { try { - const transactions = await blocks.$getBlockAuditSummary(req.params.hash); - res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString()); - res.json(transactions); + const auditSummary = await blocks.$getBlockAuditSummary(req.params.hash); + if (auditSummary) { + res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString()); + res.json(auditSummary); + } else { + return res.status(404).send(`audit not available`); + } } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index fae1d453b..09ed0bebf 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -158,6 +158,13 @@ class Blocks { }; } + public summarizeBlockTransactions(hash: string, transactions: TransactionExtended[]): BlockSummary { + return { + id: hash, + transactions: Common.stripTransactions(transactions), + }; + } + private convertLiquidFees(block: IBitcoinApi.VerboseBlock): IBitcoinApi.VerboseBlock { block.tx.forEach(tx => { tx.fee = Object.values(tx.fee || {}).reduce((total, output) => total + output, 0); @@ -646,7 +653,7 @@ class Blocks { } const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions); const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); - const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock); + const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions); this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`); // start async callbacks @@ -668,12 +675,13 @@ class Blocks { for (let i = 10; i >= 0; --i) { const newBlock = await this.$indexBlock(lastBlock.height - i); this.updateTimerProgress(timer, `reindexed block`); - await this.$getStrippedBlockTransactions(newBlock.id, true, true); - this.updateTimerProgress(timer, `reindexed block summary`); + let cpfpSummary; if (config.MEMPOOL.CPFP_INDEXING) { - await this.$indexCPFP(newBlock.id, lastBlock.height - i); + cpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i); this.updateTimerProgress(timer, `reindexed block cpfp`); } + await this.$getStrippedBlockTransactions(newBlock.id, true, true, cpfpSummary, newBlock.height); + this.updateTimerProgress(timer, `reindexed block summary`); } await mining.$indexDifficultyAdjustments(); await DifficultyAdjustmentsRepository.$deleteLastAdjustment(); @@ -704,7 +712,7 @@ class Blocks { // Save blocks summary for visualization if it's enabled if (Common.blocksSummariesIndexingEnabled() === true) { - await this.$getStrippedBlockTransactions(blockExtended.id, true); + await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height); this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`); } if (config.MEMPOOL.CPFP_INDEXING) { @@ -730,6 +738,11 @@ class Blocks { this.currentDifficulty = block.difficulty; } + // wait for pending async callbacks to finish + this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`); + await Promise.all(callbackPromises); + this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`); + this.blocks.push(blockExtended); if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) { this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4); @@ -746,11 +759,6 @@ class Blocks { diskCache.$saveCacheToDisk(); } - // wait for pending async callbacks to finish - this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`); - await Promise.all(callbackPromises); - this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`); - handledBlocks++; } @@ -827,7 +835,7 @@ class Blocks { } public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false, - skipDBLookup = false): Promise + skipDBLookup = false, cpfpSummary?: CpfpSummary, blockHeight?: number): Promise { if (skipMemoryCache === false) { // Check the memory cache @@ -845,13 +853,35 @@ class Blocks { } } - // Call Core RPC - const block = await bitcoinClient.getBlock(hash, 2); - const summary = this.summarizeBlock(block); + let height = blockHeight; + let summary: BlockSummary; + if (cpfpSummary) { + summary = { + id: hash, + transactions: cpfpSummary.transactions.map(tx => { + return { + txid: tx.txid, + fee: tx.fee, + vsize: tx.vsize, + value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)), + rate: tx.effectiveFeePerVsize + }; + }), + }; + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(hash, 2); + summary = this.summarizeBlock(block); + height = block.height; + } + if (height == null) { + const block = await bitcoinApi.$getBlock(hash); + height = block.height; + } // Index the response if needed if (Common.blocksSummariesIndexingEnabled() === true) { - await BlocksSummariesRepository.$saveTransactions(block.height, block.hash, summary.transactions); + await BlocksSummariesRepository.$saveTransactions(height, hash, summary.transactions); } return summary.transactions; @@ -1007,19 +1037,11 @@ class Blocks { } public async $getBlockAuditSummary(hash: string): Promise { - let summary; if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { - summary = await BlocksAuditsRepository.$getBlockAudit(hash); + return BlocksAuditsRepository.$getBlockAudit(hash); + } else { + return null; } - - // fallback to non-audited transaction summary - if (!summary?.transactions?.length) { - const strippedTransactions = await this.$getStrippedBlockTransactions(hash); - summary = { - transactions: strippedTransactions - }; - } - return summary; } public getLastDifficultyAdjustmentTime(): number { @@ -1050,9 +1072,13 @@ class Blocks { } public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { - const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters); - if (!result) { - await cpfpRepository.$insertProgressMarker(height); + try { + const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters); + if (!result) { + await cpfpRepository.$insertProgressMarker(height); + } + } catch (e) { + // not a fatal error, we'll try again next time the indexer runs } } } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 234535cc4..9836559ae 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -113,6 +113,10 @@ export class Common { }; } + static stripTransactions(txs: TransactionExtended[]): TransactionStripped[] { + return txs.map(this.stripTransaction); + } + static sleep$(ms: number): Promise { return new Promise((resolve) => { setTimeout(() => { diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 224e31744..57d1a393f 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -143,7 +143,7 @@ class MempoolBlocks { const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0); if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) { onlineStats = true; - feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5); + feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]); feeStatsCalculator.processNext(tx); } } @@ -334,7 +334,7 @@ class MempoolBlocks { if (hasBlockStack) { stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0); hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS; - feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5); + feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]); } const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = []; diff --git a/backend/src/api/statistics/statistics-api.ts b/backend/src/api/statistics/statistics-api.ts index 1e8b0b7bb..9df12d704 100644 --- a/backend/src/api/statistics/statistics-api.ts +++ b/backend/src/api/statistics/statistics-api.ts @@ -211,7 +211,7 @@ class StatisticsApi { CAST(avg(vsize_1800) as DOUBLE) as vsize_1800, CAST(avg(vsize_2000) as DOUBLE) as vsize_2000 \ FROM statistics \ - WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \ + ${interval === 'all' ? '' : `WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`} \ GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \ ORDER BY statistics.added DESC;`; } @@ -259,7 +259,7 @@ class StatisticsApi { vsize_1800, vsize_2000 \ FROM statistics \ - WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \ + ${interval === 'all' ? '' : `WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`} \ GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \ ORDER BY statistics.added DESC;`; } @@ -386,6 +386,17 @@ class StatisticsApi { } } + public async $listAll(): Promise { + try { + const query = this.getQueryForDays(43200, 'all'); // 12h interval + const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout }); + return this.mapStatisticToOptimizedStatistic(rows as Statistic[]); + } catch (e) { + logger.err('$listAll() error' + (e instanceof Error ? e.message : e)); + return []; + } + } + private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] { return statistic.map((s) => { return { diff --git a/backend/src/api/statistics/statistics.routes.ts b/backend/src/api/statistics/statistics.routes.ts index 2a5871dd6..31db5198c 100644 --- a/backend/src/api/statistics/statistics.routes.ts +++ b/backend/src/api/statistics/statistics.routes.ts @@ -15,10 +15,11 @@ class StatisticsRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', this.$getStatisticsByTime.bind(this, '2y')) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', this.$getStatisticsByTime.bind(this, '3y')) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/4y', this.$getStatisticsByTime.bind(this, '4y')) + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/all', this.$getStatisticsByTime.bind(this, 'all')) ; } - private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y', req: Request, res: Response) { + private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y' | 'all', req: Request, res: Response) { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); @@ -26,10 +27,6 @@ class StatisticsRoutes { try { let result; switch (time as string) { - case '2h': - result = await statisticsApi.$list2H(); - res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); - break; case '24h': result = await statisticsApi.$list24H(); res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); @@ -58,8 +55,13 @@ class StatisticsRoutes { case '4y': result = await statisticsApi.$list4Y(); break; + case 'all': + result = await statisticsApi.$listAll(); + break; default: result = await statisticsApi.$list2H(); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + break; } res.json(result); } catch (e) { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 8aaab5ab5..041c7e767 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -22,6 +22,14 @@ import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; +// valid 'want' subscriptions +const wantable = [ + 'blocks', + 'mempool-blocks', + 'live-2h-chart', + 'stats', +]; + class WebsocketHandler { private wss: WebSocket.Server | undefined; private extraInitProperties = {}; @@ -30,7 +38,7 @@ class WebsocketHandler { private numConnected = 0; private numDisconnected = 0; - private initData: { [key: string]: string } = {}; + private socketData: { [key: string]: string } = {}; private serializedInitData: string = '{}'; constructor() { } @@ -39,28 +47,28 @@ class WebsocketHandler { this.wss = wss; } - setExtraInitProperties(property: string, value: any) { + setExtraInitData(property: string, value: any) { this.extraInitProperties[property] = value; - this.setInitDataFields(this.extraInitProperties); + this.updateSocketDataFields(this.extraInitProperties); } - private setInitDataFields(data: { [property: string]: any }): void { + private updateSocketDataFields(data: { [property: string]: any }): void { for (const property of Object.keys(data)) { if (data[property] != null) { - this.initData[property] = JSON.stringify(data[property]); + this.socketData[property] = JSON.stringify(data[property]); } else { - delete this.initData[property]; + delete this.socketData[property]; } } this.serializedInitData = '{' - + Object.keys(this.initData).map(key => `"${key}": ${this.initData[key]}`).join(', ') - + '}'; + + Object.keys(this.socketData).map(key => `"${key}": ${this.socketData[key]}`).join(', ') + + '}'; } - private updateInitData(): void { + private updateSocketData(): void { const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT); const da = difficultyAdjustment.getDifficultyAdjustment(); - this.setInitDataFields({ + this.updateSocketDataFields({ 'mempoolInfo': memPool.getMempoolInfo(), 'vBytesPerSecond': memPool.getVBytesPerSecond(), 'blocks': _blocks, @@ -94,11 +102,33 @@ class WebsocketHandler { const parsedMessage: WebsocketResponse = JSON.parse(message); const response = {}; - if (parsedMessage.action === 'want') { - client['want-blocks'] = parsedMessage.data.indexOf('blocks') > -1; - client['want-mempool-blocks'] = parsedMessage.data.indexOf('mempool-blocks') > -1; - client['want-live-2h-chart'] = parsedMessage.data.indexOf('live-2h-chart') > -1; - client['want-stats'] = parsedMessage.data.indexOf('stats') > -1; + const wantNow = {}; + if (parsedMessage && parsedMessage.action === 'want' && Array.isArray(parsedMessage.data)) { + for (const sub of wantable) { + const key = `want-${sub}`; + const wants = parsedMessage.data.includes(sub); + if (wants && client['wants'] && !client[key]) { + wantNow[key] = true; + } + client[key] = wants; + } + client['wants'] = true; + } + + // send initial data when a client first starts a subscription + if (wantNow['want-blocks'] || (parsedMessage && parsedMessage['refresh-blocks'])) { + response['blocks'] = this.socketData['blocks']; + } + + if (wantNow['want-mempool-blocks']) { + response['mempool-blocks'] = this.socketData['mempool-blocks']; + } + + if (wantNow['want-stats']) { + response['mempoolInfo'] = this.socketData['mempoolInfo']; + response['vBytesPerSecond'] = this.socketData['vBytesPerSecond']; + response['fees'] = this.socketData['fees']; + response['da'] = this.socketData['da']; } if (parsedMessage && parsedMessage['track-tx']) { @@ -109,21 +139,21 @@ class WebsocketHandler { if (parsedMessage['watch-mempool']) { const rbfCacheTxid = rbfCache.getReplacedBy(trackTxid); if (rbfCacheTxid) { - response['txReplaced'] = { + response['txReplaced'] = JSON.stringify({ txid: rbfCacheTxid, - }; + }); client['track-tx'] = null; } else { // It might have appeared before we had the time to start watching for it const tx = memPool.getMempool()[trackTxid]; if (tx) { if (config.MEMPOOL.BACKEND === 'esplora') { - response['tx'] = tx; + response['tx'] = JSON.stringify(tx); } else { // tx.prevout is missing from transactions when in bitcoind mode try { const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); - response['tx'] = fullTx; + response['tx'] = JSON.stringify(fullTx); } catch (e) { logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e)); } @@ -131,7 +161,7 @@ class WebsocketHandler { } else { try { const fullTx = await transactionUtils.$getMempoolTransactionExtended(client['track-tx'], true); - response['tx'] = fullTx; + response['tx'] = JSON.stringify(fullTx); } catch (e) { logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e)); client['track-mempool-tx'] = parsedMessage['track-tx']; @@ -141,10 +171,10 @@ class WebsocketHandler { } const tx = memPool.getMempool()[trackTxid]; if (tx && tx.position) { - response['txPosition'] = { + response['txPosition'] = JSON.stringify({ txid: trackTxid, position: tx.position, - }; + }); } } else { client['track-tx'] = null; @@ -177,10 +207,10 @@ class WebsocketHandler { const index = parsedMessage['track-mempool-block']; client['track-mempool-block'] = index; const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions(); - response['projected-block-transactions'] = { + response['projected-block-transactions'] = JSON.stringify({ index: index, blockTransactions: mBlocksWithTransactions[index]?.transactions || [], - }; + }); } else { client['track-mempool-block'] = null; } @@ -189,23 +219,24 @@ class WebsocketHandler { if (parsedMessage && parsedMessage['track-rbf'] !== undefined) { if (['all', 'fullRbf'].includes(parsedMessage['track-rbf'])) { client['track-rbf'] = parsedMessage['track-rbf']; + response['rbfLatest'] = JSON.stringify(rbfCache.getRbfTrees(parsedMessage['track-rbf'] === 'fullRbf')); } else { client['track-rbf'] = false; } } if (parsedMessage.action === 'init') { - if (!this.initData['blocks']?.length || !this.initData['da']) { - this.updateInitData(); + if (!this.socketData['blocks']?.length || !this.socketData['da']) { + this.updateSocketData(); } - if (!this.initData['blocks']?.length) { + if (!this.socketData['blocks']?.length) { return; } client.send(this.serializedInitData); } if (parsedMessage.action === 'ping') { - response['pong'] = true; + response['pong'] = JSON.stringify(true); } if (parsedMessage['track-donation'] && parsedMessage['track-donation'].length === 22) { @@ -221,7 +252,8 @@ class WebsocketHandler { } if (Object.keys(response).length) { - client.send(JSON.stringify(response)); + const serializedResponse = this.serializeResponse(response); + client.send(serializedResponse); } } catch (e) { logger.debug('Error parsing websocket message: ' + (e instanceof Error ? e.message : e)); @@ -250,7 +282,7 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } - this.setInitDataFields({ 'loadingIndicators': indicators }); + this.updateSocketDataFields({ 'loadingIndicators': indicators }); const response = JSON.stringify({ loadingIndicators: indicators }); this.wss.clients.forEach((client) => { @@ -266,7 +298,7 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } - this.setInitDataFields({ 'conversions': conversionRates }); + this.updateSocketDataFields({ 'conversions': conversionRates }); const response = JSON.stringify({ conversions: conversionRates }); this.wss.clients.forEach((client) => { @@ -336,11 +368,21 @@ class WebsocketHandler { memPool.addToSpendMap(newTransactions); const recommendedFees = feeApi.getRecommendedFee(); + const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx)); + // update init data - this.updateInitData(); + this.updateSocketDataFields({ + 'mempoolInfo': mempoolInfo, + 'vBytesPerSecond': vBytesPerSecond, + 'mempool-blocks': mBlocks, + 'transactions': latestTransactions, + 'loadingIndicators': loadingIndicators.getLoadingIndicators(), + 'da': da?.previousTime ? da : undefined, + 'fees': recommendedFees, + }); // cache serialized objects to avoid stringify-ing the same thing for every client - const responseCache = { ...this.initData }; + const responseCache = { ...this.socketData }; function getCachedResponse(key: string, data): string { if (!responseCache[key]) { responseCache[key] = JSON.stringify(data); @@ -371,8 +413,6 @@ class WebsocketHandler { } } - const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx)); - this.wss.clients.forEach(async (client) => { if (client.readyState !== WebSocket.OPEN) { return; @@ -490,7 +530,7 @@ class WebsocketHandler { if (rbfReplacedBy) { response['rbfTransaction'] = JSON.stringify({ txid: rbfReplacedBy, - }) + }); } const rbfChange = rbfChanges.map[client['track-tx']]; @@ -524,9 +564,7 @@ class WebsocketHandler { } if (Object.keys(response).length) { - const serializedResponse = '{' - + Object.keys(response).map(key => `"${key}": ${response[key]}`).join(', ') - + '}'; + const serializedResponse = this.serializeResponse(response); client.send(serializedResponse); } }); @@ -562,14 +600,7 @@ class WebsocketHandler { const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; - const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => { - return { - txid: tx.txid, - vsize: tx.vsize, - fee: tx.fee ? Math.round(tx.fee) : 0, - value: tx.value, - }; - }) : []; + const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; let totalFees = 0; let totalWeight = 0; @@ -633,11 +664,19 @@ class WebsocketHandler { const da = difficultyAdjustment.getDifficultyAdjustment(); const fees = feeApi.getRecommendedFee(); + const mempoolInfo = memPool.getMempoolInfo(); // update init data - this.updateInitData(); + this.updateSocketDataFields({ + 'mempoolInfo': mempoolInfo, + 'blocks': [...blocks.getBlocks(), block].slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT), + 'mempool-blocks': mBlocks, + 'loadingIndicators': loadingIndicators.getLoadingIndicators(), + 'da': da?.previousTime ? da : undefined, + 'fees': fees, + }); - const responseCache = { ...this.initData }; + const responseCache = { ...this.socketData }; function getCachedResponse(key, data): string { if (!responseCache[key]) { responseCache[key] = JSON.stringify(data); @@ -645,22 +684,26 @@ class WebsocketHandler { return responseCache[key]; } - const mempoolInfo = memPool.getMempoolInfo(); - this.wss.clients.forEach((client) => { if (client.readyState !== WebSocket.OPEN) { return; } - if (!client['want-blocks']) { - return; + const response = {}; + + if (client['want-blocks']) { + response['block'] = getCachedResponse('block', block); } - const response = {}; - response['block'] = getCachedResponse('block', block); - response['mempoolInfo'] = getCachedResponse('mempoolInfo', mempoolInfo); - response['da'] = getCachedResponse('da', da?.previousTime ? da : undefined); - response['fees'] = getCachedResponse('fees', fees); + if (client['want-stats']) { + response['mempoolInfo'] = getCachedResponse('mempoolInfo', mempoolInfo); + response['vBytesPerSecond'] = getCachedResponse('vBytesPerSecond', memPool.getVBytesPerSecond()); + response['fees'] = getCachedResponse('fees', fees); + + if (da?.previousTime) { + response['da'] = getCachedResponse('da', da); + } + } if (mBlocks && client['want-mempool-blocks']) { response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks); @@ -755,11 +798,19 @@ class WebsocketHandler { } } - const serializedResponse = '{' + if (Object.keys(response).length) { + const serializedResponse = this.serializeResponse(response); + client.send(serializedResponse); + } + }); + } + + // takes a dictionary of JSON serialized values + // and zips it together into a valid JSON object + private serializeResponse(response): string { + return '{' + Object.keys(response).map(key => `"${key}": ${response[key]}`).join(', ') + '}'; - client.send(serializedResponse); - }); } private printLogs(): void { diff --git a/backend/src/database.ts b/backend/src/database.ts index 070774c92..6ad545fda 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -30,7 +30,7 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr } public async query(query, params?): Promise<[T, FieldPacket[]]> + OkPacket[] | ResultSetHeader>(query, params?, connection?: PoolConnection): Promise<[T, FieldPacket[]]> { this.checkDBFlag(); let hardTimeout; @@ -45,7 +45,9 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr reject(new Error(`DB query failed to return, reject or time out within ${hardTimeout / 1000}s - ${query?.sql?.slice(0, 160) || (typeof(query) === 'string' || query instanceof String ? query?.slice(0, 160) : 'unknown query')}`)); }, hardTimeout); - this.getPool().then(pool => { + // Use a specific connection if provided, otherwise delegate to the pool + const connectionPromise = connection ? Promise.resolve(connection) : this.getPool(); + connectionPromise.then((pool: PoolConnection | Pool) => { return pool.query(query, params) as Promise<[T, FieldPacket[]]>; }).then(result => { resolve(result); @@ -61,6 +63,33 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr } } + public async $atomicQuery(queries: { query, params }[]): Promise<[T, FieldPacket[]][]> + { + const pool = await this.getPool(); + const connection = await pool.getConnection(); + try { + await connection.beginTransaction(); + + const results: [T, FieldPacket[]][] = []; + for (const query of queries) { + const result = await this.query(query.query, query.params, connection) as [T, FieldPacket[]]; + results.push(result); + } + + await connection.commit(); + + return results; + } catch (e) { + logger.err('Could not complete db transaction, rolling back: ' + (e instanceof Error ? e.message : e)); + connection.rollback(); + connection.release(); + throw e; + } finally { + connection.release(); + } + } + public async checkDbConnection() { this.checkDBFlag(); try { diff --git a/backend/src/index.ts b/backend/src/index.ts index 9f543d644..81863a208 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -150,7 +150,7 @@ class Server { if (config.BISQ.ENABLED) { bisq.startBisqService(); - bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitProperties('bsq-price', price)); + bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitData('bsq-price', price)); blocks.setNewBlockCallback(bisq.handleNewBitcoinBlock.bind(bisq)); bisqMarkets.startBisqService(); } diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 1fa2b0209..8ad035f32 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -64,7 +64,6 @@ class BlocksAuditRepositories { const [rows]: any[] = await DB.query( `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size, blocks.weight, blocks.tx_count, - transactions, template, missing_txs as missingTxs, added_txs as addedTxs, @@ -76,7 +75,6 @@ class BlocksAuditRepositories { FROM blocks_audits JOIN blocks ON blocks.hash = blocks_audits.hash JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash - JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash WHERE blocks_audits.hash = "${hash}" `); @@ -85,12 +83,9 @@ class BlocksAuditRepositories { rows[0].addedTxs = JSON.parse(rows[0].addedTxs); rows[0].freshTxs = JSON.parse(rows[0].freshTxs); rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); - rows[0].transactions = JSON.parse(rows[0].transactions); rows[0].template = JSON.parse(rows[0].template); - if (rows[0].transactions.length) { - return rows[0]; - } + return rows[0]; } return null; } catch (e: any) { diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index 90ba2ac80..b33ff1e4a 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -5,52 +5,10 @@ import { Ancestor, CpfpCluster } from '../mempool.interfaces'; import transactionRepository from '../repositories/TransactionRepository'; class CpfpRepository { - public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise { - if (!txs[0]) { - return false; - } - // skip clusters of transactions with the same fees - const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100; - const equalFee = txs.length > 1 && txs.reduce((acc, tx) => { - return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee); - }, true); - if (equalFee) { - return false; - } - - try { - const packedTxs = Buffer.from(this.pack(txs)); - await DB.query( - ` - INSERT INTO compact_cpfp_clusters(root, height, txs, fee_rate) - VALUE (UNHEX(?), ?, ?, ?) - ON DUPLICATE KEY UPDATE - height = ?, - txs = ?, - fee_rate = ? - `, - [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize] - ); - const maxChunk = 10; - let chunkIndex = 0; - while (chunkIndex < txs.length) { - const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk).map(tx => { - return { txid: tx.txid, cluster: clusterRoot }; - }); - await transactionRepository.$batchSetCluster(chunk); - chunkIndex += maxChunk; - } - return true; - } catch (e: any) { - logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e)); - throw e; - } - } - public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise { try { - const clusterValues: any[] = []; - const txs: any[] = []; + const clusterValues: [string, number, Buffer, number][] = []; + const txs: { txid: string, cluster: string }[] = []; for (const cluster of clusters) { if (cluster.txs?.length) { @@ -76,6 +34,8 @@ class CpfpRepository { return false; } + const queries: { query, params }[] = []; + const maxChunk = 100; let chunkIndex = 0; // insert clusters in batches of up to 100 rows @@ -89,10 +49,10 @@ class CpfpRepository { return (' (UNHEX(?), ?, ?, ?)'); }) + ';'; const values = chunk.flat(); - await DB.query( + queries.push({ query, - values - ); + params: values, + }); chunkIndex += maxChunk; } @@ -100,10 +60,12 @@ class CpfpRepository { // insert transactions in batches of up to 100 rows while (chunkIndex < txs.length) { const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk); - await transactionRepository.$batchSetCluster(chunk); + queries.push(transactionRepository.buildBatchSetQuery(chunk)); chunkIndex += maxChunk; } + await DB.$atomicQuery(queries); + return true; } catch (e: any) { logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts index bde95df9b..b5067f790 100644 --- a/backend/src/repositories/TransactionRepository.ts +++ b/backend/src/repositories/TransactionRepository.ts @@ -25,9 +25,8 @@ class TransactionRepository { } } - public async $batchSetCluster(txs): Promise { - try { - let query = ` + public buildBatchSetQuery(txs: { txid: string, cluster: string }[]): { query, params } { + let query = ` INSERT IGNORE INTO compact_transactions ( txid, @@ -35,13 +34,22 @@ class TransactionRepository { ) VALUES `; - query += txs.map(tx => { - return (' (UNHEX(?), UNHEX(?))'); - }) + ';'; - const values = txs.map(tx => [tx.txid, tx.cluster]).flat(); + query += txs.map(tx => { + return (' (UNHEX(?), UNHEX(?))'); + }) + ';'; + const values = txs.map(tx => [tx.txid, tx.cluster]).flat(); + return { + query, + params: values, + }; + } + + public async $batchSetCluster(txs): Promise { + try { + const query = this.buildBatchSetQuery(txs); await DB.query( - query, - values + query.query, + query.params, ); } catch (e: any) { logger.err(`Cannot save cpfp transactions into db. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 9b2bdf6c0..381353948 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -220,7 +220,7 @@ myNode - + RoninDojo diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 1a0a81026..00cb2fcb1 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -63,7 +63,7 @@ *ngIf="blockAudit?.matchRate != null; else nullHealth" >{{ blockAudit?.matchRate }}% - + Unknown diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 17e6e9b7f..927222dbc 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -2,9 +2,9 @@ import { Component, OnInit, OnDestroy, ViewChildren, QueryList } from '@angular/ import { Location } from '@angular/common'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; import { ElectrsApiService } from '../../services/electrs-api.service'; -import { switchMap, tap, throttleTime, catchError, map, shareReplay, startWith, pairwise, filter } from 'rxjs/operators'; +import { switchMap, tap, throttleTime, catchError, map, shareReplay, startWith } from 'rxjs/operators'; import { Transaction, Vout } from '../../interfaces/electrs.interface'; -import { Observable, of, Subscription, asyncScheduler, EMPTY, combineLatest } from 'rxjs'; +import { Observable, of, Subscription, asyncScheduler, EMPTY, combineLatest, forkJoin } from 'rxjs'; import { StateService } from '../../services/state.service'; import { SeoService } from '../../services/seo.service'; import { WebsocketService } from '../../services/websocket.service'; @@ -44,7 +44,6 @@ export class BlockComponent implements OnInit, OnDestroy { strippedTransactions: TransactionStripped[]; overviewTransitionDirection: string; isLoadingOverview = true; - isLoadingAudit = true; error: any; blockSubsidy: number; fees: number; @@ -281,143 +280,111 @@ export class BlockComponent implements OnInit, OnDestroy { this.isLoadingOverview = false; }); - if (!this.auditSupported) { - this.overviewSubscription = block$.pipe( - startWith(null), - pairwise(), - switchMap(([prevBlock, block]) => this.apiService.getStrippedBlockTransactions$(block.id) - .pipe( - catchError((err) => { - this.overviewError = err; - return of([]); - }), - switchMap((transactions) => { - if (prevBlock) { - return of({ transactions, direction: (prevBlock.height < block.height) ? 'right' : 'left' }); - } else { - return of({ transactions, direction: 'down' }); - } - }) - ) - ), - ) - .subscribe(({transactions, direction}: {transactions: TransactionStripped[], direction: string}) => { - this.strippedTransactions = transactions; - this.isLoadingOverview = false; - this.setupBlockGraphs(); - }, - (error) => { - this.error = error; - this.isLoadingOverview = false; - }); - } - - if (this.auditSupported) { - this.auditSubscription = block$.pipe( - startWith(null), - pairwise(), - switchMap(([prevBlock, block]) => { - this.isLoadingAudit = true; - this.blockAudit = null; - return this.apiService.getBlockAudit$(block.id) + this.overviewSubscription = block$.pipe( + switchMap((block) => { + return forkJoin([ + this.apiService.getStrippedBlockTransactions$(block.id) .pipe( catchError((err) => { this.overviewError = err; - this.isLoadingAudit = false; - return of([]); + return of(null); }) - ); - } - ), - filter((response) => response != null), - map((response) => { - const blockAudit = response.body; - const inTemplate = {}; - const inBlock = {}; - const isAdded = {}; - const isCensored = {}; - const isMissing = {}; - const isSelected = {}; - const isFresh = {}; - const isSigop = {}; - this.numMissing = 0; - this.numUnexpected = 0; + ), + !this.isAuditAvailableFromBlockHeight(block.height) ? of(null) : this.apiService.getBlockAudit$(block.id) + .pipe( + catchError((err) => { + this.overviewError = err; + return of(null); + }) + ) + ]); + }) + ) + .subscribe(([transactions, blockAudit]) => { + if (transactions) { + this.strippedTransactions = transactions; + } else { + this.strippedTransactions = []; + } - if (blockAudit?.template) { - for (const tx of blockAudit.template) { - inTemplate[tx.txid] = true; - } - for (const tx of blockAudit.transactions) { - inBlock[tx.txid] = true; - } - for (const txid of blockAudit.addedTxs) { - isAdded[txid] = true; - } - for (const txid of blockAudit.missingTxs) { - isCensored[txid] = true; - } - for (const txid of blockAudit.freshTxs || []) { - isFresh[txid] = true; - } - for (const txid of blockAudit.sigopTxs || []) { - isSigop[txid] = true; - } - // set transaction statuses - for (const tx of blockAudit.template) { - tx.context = 'projected'; - if (isCensored[tx.txid]) { - tx.status = 'censored'; - } else if (inBlock[tx.txid]) { - tx.status = 'found'; - } else { - tx.status = isFresh[tx.txid] ? 'fresh' : (isSigop[tx.txid] ? 'sigop' : 'missing'); - isMissing[tx.txid] = true; - this.numMissing++; - } - } - for (const [index, tx] of blockAudit.transactions.entries()) { - tx.context = 'actual'; - if (index === 0) { - tx.status = null; - } else if (isAdded[tx.txid]) { - tx.status = 'added'; - } else if (inTemplate[tx.txid]) { - tx.status = 'found'; - } else { - tx.status = 'selected'; - isSelected[tx.txid] = true; - this.numUnexpected++; - } - } - for (const tx of blockAudit.transactions) { - inBlock[tx.txid] = true; - } + this.blockAudit = null; + if (transactions && blockAudit) { + const inTemplate = {}; + const inBlock = {}; + const isAdded = {}; + const isCensored = {}; + const isMissing = {}; + const isSelected = {}; + const isFresh = {}; + const isSigop = {}; + this.numMissing = 0; + this.numUnexpected = 0; - blockAudit.feeDelta = blockAudit.expectedFees > 0 ? (blockAudit.expectedFees - this.block.extras.totalFees) / blockAudit.expectedFees : 0; - blockAudit.weightDelta = blockAudit.expectedWeight > 0 ? (blockAudit.expectedWeight - this.block.weight) / blockAudit.expectedWeight : 0; - blockAudit.txDelta = blockAudit.template.length > 0 ? (blockAudit.template.length - this.block.tx_count) / blockAudit.template.length : 0; - - this.setAuditAvailable(true); - } else { - this.setAuditAvailable(false); + if (blockAudit?.template) { + for (const tx of blockAudit.template) { + inTemplate[tx.txid] = true; } - return blockAudit; - }), - catchError((err) => { - console.log(err); - this.error = err; - this.isLoadingOverview = false; - this.isLoadingAudit = false; + for (const tx of transactions) { + inBlock[tx.txid] = true; + } + for (const txid of blockAudit.addedTxs) { + isAdded[txid] = true; + } + for (const txid of blockAudit.missingTxs) { + isCensored[txid] = true; + } + for (const txid of blockAudit.freshTxs || []) { + isFresh[txid] = true; + } + for (const txid of blockAudit.sigopTxs || []) { + isSigop[txid] = true; + } + // set transaction statuses + for (const tx of blockAudit.template) { + tx.context = 'projected'; + if (isCensored[tx.txid]) { + tx.status = 'censored'; + } else if (inBlock[tx.txid]) { + tx.status = 'found'; + } else { + tx.status = isFresh[tx.txid] ? 'fresh' : (isSigop[tx.txid] ? 'sigop' : 'missing'); + isMissing[tx.txid] = true; + this.numMissing++; + } + } + for (const [index, tx] of transactions.entries()) { + tx.context = 'actual'; + if (index === 0) { + tx.status = null; + } else if (isAdded[tx.txid]) { + tx.status = 'added'; + } else if (inTemplate[tx.txid]) { + tx.status = 'found'; + } else { + tx.status = 'selected'; + isSelected[tx.txid] = true; + this.numUnexpected++; + } + } + for (const tx of transactions) { + inBlock[tx.txid] = true; + } + + blockAudit.feeDelta = blockAudit.expectedFees > 0 ? (blockAudit.expectedFees - this.block.extras.totalFees) / blockAudit.expectedFees : 0; + blockAudit.weightDelta = blockAudit.expectedWeight > 0 ? (blockAudit.expectedWeight - this.block.weight) / blockAudit.expectedWeight : 0; + blockAudit.txDelta = blockAudit.template.length > 0 ? (blockAudit.template.length - this.block.tx_count) / blockAudit.template.length : 0; + this.blockAudit = blockAudit; + this.setAuditAvailable(true); + } else { this.setAuditAvailable(false); - return of(null); - }), - ).subscribe((blockAudit) => { - this.blockAudit = blockAudit; - this.setupBlockGraphs(); - this.isLoadingOverview = false; - this.isLoadingAudit = false; - }); - } + } + } else { + this.setAuditAvailable(false); + } + + this.isLoadingOverview = false; + this.setupBlockGraphs(); + }); this.networkChangedSubscription = this.stateService.networkChanged$ .subscribe((network) => this.network = network); @@ -652,25 +619,32 @@ export class BlockComponent implements OnInit, OnDestroy { } updateAuditAvailableFromBlockHeight(blockHeight: number): void { - if (!this.auditSupported) { + if (!this.isAuditAvailableFromBlockHeight(blockHeight)) { this.setAuditAvailable(false); } + } + + isAuditAvailableFromBlockHeight(blockHeight: number): boolean { + if (!this.auditSupported) { + return false; + } switch (this.stateService.network) { case 'testnet': if (blockHeight < this.stateService.env.TESTNET_BLOCK_AUDIT_START_HEIGHT) { - this.setAuditAvailable(false); + return false; } break; case 'signet': if (blockHeight < this.stateService.env.SIGNET_BLOCK_AUDIT_START_HEIGHT) { - this.setAuditAvailable(false); + return false; } break; default: if (blockHeight < this.stateService.env.MAINNET_BLOCK_AUDIT_START_HEIGHT) { - this.setAuditAvailable(false); + return false; } } + return true; } getMinBlockFee(block: BlockExtended): number { diff --git a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts index 8c90036fd..823d271a1 100644 --- a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts +++ b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts @@ -1,5 +1,9 @@ import { OnChanges } from '@angular/core'; import { Component, Input, OnInit, ChangeDetectionStrategy } from '@angular/core'; +import { TransactionStripped } from '../../interfaces/websocket.interface'; +import { StateService } from '../../services/state.service'; +import { VbytesPipe } from '../../shared/pipes/bytes-pipe/vbytes.pipe'; +import { selectPowerOfTen } from '../../bitcoin.utils'; @Component({ selector: 'app-fee-distribution-graph', @@ -7,47 +11,121 @@ import { Component, Input, OnInit, ChangeDetectionStrategy } from '@angular/core changeDetection: ChangeDetectionStrategy.OnPush, }) export class FeeDistributionGraphComponent implements OnInit, OnChanges { - @Input() data: any; + @Input() feeRange: number[]; + @Input() vsize: number; + @Input() transactions: TransactionStripped[]; @Input() height: number | string = 210; @Input() top: number | string = 20; @Input() right: number | string = 22; @Input() left: number | string = 30; + @Input() numSamples: number = 200; + @Input() numLabels: number = 10; + + simple: boolean = false; + data: number[][]; + labelInterval: number = 50; mempoolVsizeFeesOptions: any; mempoolVsizeFeesInitOptions = { renderer: 'svg' }; - constructor() { } + constructor( + private stateService: StateService, + private vbytesPipe: VbytesPipe, + ) { } - ngOnInit() { + ngOnInit(): void { this.mountChart(); } - ngOnChanges() { + ngOnChanges(): void { + this.simple = !!this.feeRange?.length; + this.prepareChart(); this.mountChart(); } - mountChart() { + prepareChart(): void { + if (this.simple) { + this.data = this.feeRange.map((rate, index) => [index * 10, rate]); + this.labelInterval = 1; + return; + } + this.data = []; + if (!this.transactions?.length) { + return; + } + const samples = []; + const txs = this.transactions.map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; }); + const maxBlockVSize = this.stateService.env.BLOCK_WEIGHT_UNITS / 4; + const sampleInterval = maxBlockVSize / this.numSamples; + let cumVSize = 0; + let sampleIndex = 0; + let nextSample = 0; + let txIndex = 0; + this.labelInterval = this.numSamples / this.numLabels; + while (nextSample <= maxBlockVSize) { + if (txIndex >= txs.length) { + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0]); + nextSample += sampleInterval; + sampleIndex++; + continue; + } + + while (txs[txIndex] && nextSample < cumVSize + txs[txIndex].vsize) { + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate]); + nextSample += sampleInterval; + sampleIndex++; + } + cumVSize += txs[txIndex].vsize; + txIndex++; + } + this.data = samples.reverse(); + } + + mountChart(): void { this.mempoolVsizeFeesOptions = { grid: { height: '210', right: '20', top: '22', - left: '30', + left: '40', }, xAxis: { type: 'category', boundaryGap: false, + name: '% Weight', + nameLocation: 'middle', + nameGap: 0, + nameTextStyle: { + verticalAlign: 'top', + padding: [30, 0, 0, 0], + }, + axisLabel: { + interval: (index: number): boolean => { return index && (index % this.labelInterval === 0); }, + formatter: (value: number): string => { return Number(value).toFixed(0); }, + }, + axisTick: { + interval: (index:number): boolean => { return (index % this.labelInterval === 0); }, + }, }, yAxis: { type: 'value', + // name: 'Effective Fee Rate s/vb', + // nameLocation: 'middle', splitLine: { lineStyle: { type: 'dotted', color: '#ffffff66', opacity: 0.25, } + }, + axisLabel: { + formatter: (value: number): string => { + const selectedPowerOfTen = selectPowerOfTen(value); + const newVal = Math.round(value / selectedPowerOfTen.divider); + return `${newVal}${selectedPowerOfTen.unit}`; + }, } }, series: [{ @@ -58,14 +136,18 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges { position: 'top', color: '#ffffff', textShadowBlur: 0, - formatter: (label: any) => { - return Math.floor(label.data); + formatter: (label: { data: number[] }): string => { + const value = label.data[1]; + const selectedPowerOfTen = selectPowerOfTen(value); + const newVal = Math.round(value / selectedPowerOfTen.divider); + return `${newVal}${selectedPowerOfTen.unit}`; }, }, + showAllSymbol: false, smooth: true, lineStyle: { color: '#D81B60', - width: 4, + width: 1, }, itemStyle: { color: '#b71c1c', diff --git a/frontend/src/app/components/mempool-block/mempool-block.component.html b/frontend/src/app/components/mempool-block/mempool-block.component.html index 3626e6ff5..7d5b18ccb 100644 --- a/frontend/src/app/components/mempool-block/mempool-block.component.html +++ b/frontend/src/app/components/mempool-block/mempool-block.component.html @@ -39,11 +39,11 @@ - +
- +
diff --git a/frontend/src/app/components/mempool-block/mempool-block.component.ts b/frontend/src/app/components/mempool-block/mempool-block.component.ts index b9bdc55bb..6e0b21196 100644 --- a/frontend/src/app/components/mempool-block/mempool-block.component.ts +++ b/frontend/src/app/components/mempool-block/mempool-block.component.ts @@ -17,6 +17,7 @@ export class MempoolBlockComponent implements OnInit, OnDestroy { network$: Observable; mempoolBlockIndex: number; mempoolBlock$: Observable; + mempoolBlockTransactions$: Observable; ordinal$: BehaviorSubject = new BehaviorSubject(''); previewTx: TransactionStripped | void; webGlEnabled: boolean; @@ -53,6 +54,7 @@ export class MempoolBlockComponent implements OnInit, OnDestroy { const ordinal = this.getOrdinal(mempoolBlocks[this.mempoolBlockIndex]); this.ordinal$.next(ordinal); this.seoService.setTitle(ordinal); + mempoolBlocks[this.mempoolBlockIndex].isStack = mempoolBlocks[this.mempoolBlockIndex].blockVSize > this.stateService.blockVSize; return mempoolBlocks[this.mempoolBlockIndex]; }) ); @@ -62,6 +64,8 @@ export class MempoolBlockComponent implements OnInit, OnDestroy { }) ); + this.mempoolBlockTransactions$ = this.stateService.liveMempoolBlockTransactions$.pipe(map(txMap => Object.values(txMap))); + this.network$ = this.stateService.networkChanged$; } diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts index 561f01585..a72d24899 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts @@ -143,6 +143,8 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { this.mempoolBlocksFull = JSON.parse(stringifiedBlocks); this.mempoolBlocks = this.reduceMempoolBlocksToFitScreen(JSON.parse(stringifiedBlocks)); + this.now = Date.now(); + this.updateMempoolBlockStyles(); this.calculateTransactionPosition(); return this.mempoolBlocks; @@ -152,7 +154,8 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { this.difficultyAdjustments$ = this.stateService.difficultyAdjustment$ .pipe( map((da) => { - this.now = new Date().getTime(); + this.now = Date.now(); + this.cd.markForCheck(); return da; }) ); diff --git a/frontend/src/app/components/statistics/statistics.component.html b/frontend/src/app/components/statistics/statistics.component.html index 1b291e978..29089e43d 100644 --- a/frontend/src/app/components/statistics/statistics.component.html +++ b/frontend/src/app/components/statistics/statistics.component.html @@ -20,39 +20,46 @@ -
- - - - - - - - - - +
+
+ + + + + +
+
+ + + + + + +
diff --git a/frontend/src/app/components/statistics/statistics.component.scss b/frontend/src/app/components/statistics/statistics.component.scss index f7230ce4b..d883e48c3 100644 --- a/frontend/src/app/components/statistics/statistics.component.scss +++ b/frontend/src/app/components/statistics/statistics.component.scss @@ -53,17 +53,17 @@ } } .formRadioGroup.mining { - @media (min-width: 991px) { + @media (min-width: 1035px) { position: relative; top: -100px; } - @media (min-width: 830px) and (max-width: 991px) { + @media (min-width: 830px) and (max-width: 1035px) { position: relative; top: 0px; } } .formRadioGroup.no-menu { - @media (min-width: 991px) { + @media (min-width: 1035px) { position: relative; top: -33px; } @@ -183,3 +183,43 @@ } } } + +.btn-toggle-rows { + display: flex; + flex-direction: row; + align-items: stretch; + justify-content: stretch; + + .btn-group { + flex-grow: 1; + flex-shrink: 1; + } + + @media (min-width: 500px) { + .btn-group:first-child > .btn:last-child { + border-top-right-radius: 0; + border-bottom-right-radius: 0; + } + .btn-group:last-child > .btn:first-child { + border-top-left-radius: 0; + border-bottom-left-radius: 0; + } + } + + @media (max-width: 499px) { + flex-direction: column; + + .btn-group:first-child > .btn:first-child { + border-bottom-left-radius: 0; + } + .btn-group:first-child > .btn:last-child { + border-bottom-right-radius: 0; + } + .btn-group:last-child > .btn:first-child { + border-top-left-radius: 0; + } + .btn-group:last-child > .btn:last-child { + border-top-right-radius: 0; + } + } +} \ No newline at end of file diff --git a/frontend/src/app/components/statistics/statistics.component.ts b/frontend/src/app/components/statistics/statistics.component.ts index 35137bff1..eebee5bce 100644 --- a/frontend/src/app/components/statistics/statistics.component.ts +++ b/frontend/src/app/components/statistics/statistics.component.ts @@ -72,8 +72,10 @@ export class StatisticsComponent implements OnInit { this.route .fragment .subscribe((fragment) => { - if (['2h', '24h', '1w', '1m', '3m', '6m', '1y', '2y', '3y', '4y'].indexOf(fragment) > -1) { + if (['2h', '24h', '1w', '1m', '3m', '6m', '1y', '2y', '3y', '4y', 'all'].indexOf(fragment) > -1) { this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false }); + } else { + this.radioGroupForm.controls.dateSpan.setValue('2h', { emitEvent: false }); } }); @@ -114,7 +116,12 @@ export class StatisticsComponent implements OnInit { if (this.radioGroupForm.controls.dateSpan.value === '3y') { return this.apiService.list3YStatistics$(); } - return this.apiService.list4YStatistics$(); + if (this.radioGroupForm.controls.dateSpan.value === '4y') { + return this.apiService.list4YStatistics$(); + } + if (this.radioGroupForm.controls.dateSpan.value === 'all') { + return this.apiService.listAllTimeStatistics$(); + } }) ) .subscribe((mempoolStats: any) => { diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index a5a44ee90..216114cd0 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -105,7 +105,7 @@ - + diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index dd32b05e8..e7fbaa913 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -19,7 +19,7 @@ import { WebsocketService } from '../../services/websocket.service'; import { AudioService } from '../../services/audio.service'; import { ApiService } from '../../services/api.service'; import { SeoService } from '../../services/seo.service'; -import { BlockExtended, CpfpInfo, RbfTree, MempoolPosition } from '../../interfaces/node-api.interface'; +import { BlockExtended, CpfpInfo, RbfTree, MempoolPosition, DifficultyAdjustment } from '../../interfaces/node-api.interface'; import { LiquidUnblinding } from './liquid-ublinding'; import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; import { Price, PriceService } from '../../services/price.service'; @@ -65,7 +65,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { fetchCachedTx$ = new Subject(); isCached: boolean = false; now = Date.now(); - timeAvg$: Observable; + da$: Observable; liquidUnblinding = new LiquidUnblinding(); inputIndex: number; outputIndex: number; @@ -117,11 +117,11 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.setFlowEnabled(); }); - this.timeAvg$ = timer(0, 1000) - .pipe( - switchMap(() => this.stateService.difficultyAdjustment$), - map((da) => da.timeAvg) - ); + this.da$ = this.stateService.difficultyAdjustment$.pipe( + tap(() => { + this.now = Date.now(); + }) + ); this.urlFragmentSubscription = this.route.fragment.subscribe((fragment) => { this.fragmentParams = new URLSearchParams(fragment || ''); @@ -236,6 +236,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }); this.mempoolPositionSubscription = this.stateService.mempoolTxPosition$.subscribe(txPosition => { + this.now = Date.now(); if (txPosition && txPosition.txid === this.txId && txPosition.position) { this.mempoolPosition = txPosition.position; if (this.tx && !this.tx.status.confirmed) { @@ -434,12 +435,12 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }); this.mempoolBlocksSubscription = this.stateService.mempoolBlocks$.subscribe((mempoolBlocks) => { + this.now = Date.now(); + if (!this.tx || this.mempoolPosition) { return; } - this.now = Date.now(); - const txFeePerVSize = this.tx.effectiveFeePerVsize || this.tx.fee / (this.tx.weight / 4); diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 2f9b95ab1..648eb38ea 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -153,6 +153,8 @@ export interface BlockExtended extends Block { export interface BlockAudit extends BlockExtended { missingTxs: string[], addedTxs: string[], + freshTxs: string[], + sigopTxs: string[], matchRate: number, expectedFees: number, expectedWeight: number, @@ -169,6 +171,7 @@ export interface TransactionStripped { vsize: number; value: number; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected'; + context?: 'projected' | 'actual'; } interface RbfTransaction extends TransactionStripped { diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index 83a0c636e..41643fb73 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -31,6 +31,7 @@ export interface WebsocketResponse { 'track-rbf'?: string; 'watch-mempool'?: boolean; 'track-bisq-market'?: string; + 'refresh-blocks'?: boolean; } export interface ReplacedTransaction extends Transaction { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 8521ddc83..e2d3be9be 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams, HttpResponse } from '@angular/common/http'; import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators, - PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore, BlockSizesAndWeights, RbfTree } from '../interfaces/node-api.interface'; + PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore, BlockSizesAndWeights, RbfTree, BlockAudit } from '../interfaces/node-api.interface'; import { Observable } from 'rxjs'; import { StateService } from './state.service'; import { WebsocketResponse } from '../interfaces/websocket.interface'; @@ -72,6 +72,10 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/statistics/4y'); } + listAllTimeStatistics$(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/statistics/all'); + } + getTransactionTimes$(txIds: string[]): Observable { let params = new HttpParams(); txIds.forEach((txId: string) => { @@ -245,9 +249,9 @@ export class ApiService { ); } - getBlockAudit$(hash: string) : Observable { - return this.httpClient.get( - this.apiBaseUrl + this.apiBasePath + `/api/v1/block/${hash}/audit-summary`, { observe: 'response' } + getBlockAudit$(hash: string) : Observable { + return this.httpClient.get( + this.apiBaseUrl + this.apiBasePath + `/api/v1/block/${hash}/audit-summary` ); } diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 31f5f3aab..fb3b37e05 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -1,11 +1,11 @@ import { Inject, Injectable, PLATFORM_ID, LOCALE_ID } from '@angular/core'; -import { ReplaySubject, BehaviorSubject, Subject, fromEvent, Observable } from 'rxjs'; +import { ReplaySubject, BehaviorSubject, Subject, fromEvent, Observable, merge } from 'rxjs'; import { Transaction } from '../interfaces/electrs.interface'; import { IBackendInfo, MempoolBlock, MempoolBlockWithTransactions, MempoolBlockDelta, MempoolInfo, Recommendedfees, ReplacedTransaction, TransactionStripped } from '../interfaces/websocket.interface'; import { BlockExtended, DifficultyAdjustment, MempoolPosition, OptimizedMempoolStats, RbfTree } from '../interfaces/node-api.interface'; import { Router, NavigationStart } from '@angular/router'; import { isPlatformBrowser } from '@angular/common'; -import { map, shareReplay } from 'rxjs/operators'; +import { map, scan, shareReplay, tap } from 'rxjs/operators'; import { StorageService } from './storage.service'; interface MarkBlockState { @@ -100,6 +100,7 @@ export class StateService { mempoolBlocks$ = new ReplaySubject(1); mempoolBlockTransactions$ = new Subject(); mempoolBlockDelta$ = new Subject(); + liveMempoolBlockTransactions$: Observable<{ [txid: string]: TransactionStripped}>; txReplaced$ = new Subject(); txRbfInfo$ = new Subject(); rbfLatest$ = new Subject(); @@ -166,6 +167,30 @@ export class StateService { this.blocks$ = new ReplaySubject<[BlockExtended, string]>(this.env.KEEP_BLOCKS_AMOUNT); + this.liveMempoolBlockTransactions$ = merge( + this.mempoolBlockTransactions$.pipe(map(transactions => { return { transactions }; })), + this.mempoolBlockDelta$.pipe(map(delta => { return { delta }; })), + ).pipe(scan((transactions: { [txid: string]: TransactionStripped }, change: any): { [txid: string]: TransactionStripped } => { + if (change.transactions) { + const txMap = {} + change.transactions.forEach(tx => { + txMap[tx.txid] = tx; + }) + return txMap; + } else { + change.delta.changed.forEach(tx => { + transactions[tx.txid].rate = tx.rate; + }) + change.delta.removed.forEach(txid => { + delete transactions[txid]; + }); + change.delta.added.forEach(tx => { + transactions[tx.txid] = tx; + }); + return transactions; + } + }, {})); + if (this.env.BASE_MODULE === 'bisq') { this.network = this.env.BASE_MODULE; this.networkChanged$.next(this.env.BASE_MODULE); diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index af7a465f8..d22717b2a 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -235,6 +235,8 @@ export class WebsocketService { } handleResponse(response: WebsocketResponse) { + let reinitBlocks = false; + if (response.blocks && response.blocks.length) { const blocks = response.blocks; let maxHeight = 0; @@ -256,9 +258,11 @@ export class WebsocketService { } if (response.block) { - if (response.block.height > this.stateService.latestBlockHeight) { + if (response.block.height === this.stateService.latestBlockHeight + 1) { this.stateService.updateChainTip(response.block.height); this.stateService.blocks$.next([response.block, response.txConfirmed || '']); + } else if (response.block.height > this.stateService.latestBlockHeight + 1) { + reinitBlocks = true; } if (response.txConfirmed) { @@ -369,5 +373,9 @@ export class WebsocketService { if (response['git-commit']) { this.stateService.backendInfo$.next(response['git-commit']); } + + if (reinitBlocks) { + this.websocketSubject.next({'refresh-blocks': true}); + } } } diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index b38a60898..aea8e8d6e 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -500,7 +500,7 @@ html:lang(ru) .card-title { } .fee-distribution-chart { - height: 250px; + height: 265px; } .fees-wrapper-tooltip-chart { diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index 6b58e5b78..6a82871a5 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -21,6 +21,7 @@ do for url in / \ '/api/v1/statistics/2y' \ '/api/v1/statistics/3y' \ '/api/v1/statistics/4y' \ + '/api/v1/statistics/all' \ '/api/v1/mining/pools/24h' \ '/api/v1/mining/pools/3d' \ '/api/v1/mining/pools/1w' \