From 3c0bb11208278f68b3c627e067047a4489832946 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Mon, 5 Jun 2023 13:20:46 +0200 Subject: [PATCH 01/11] Add expected total fees audit --- backend/src/api/database-migration.ts | 7 ++++++- backend/src/api/websocket-handler.ts | 8 +++++++- backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/BlocksAuditsRepository.ts | 8 ++++---- frontend/src/app/components/block/block.component.html | 9 +++++++++ frontend/src/app/interfaces/node-api.interface.ts | 1 + 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 21c87f9e2..7d4d461e4 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 60; + private static currentVersion = 61; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -521,6 +521,11 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `blocks_audits` ADD sigop_txs JSON DEFAULT "[]"'); await this.updateToSchemaVersion(60); } + + if (databaseSchemaVersion < 61 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED NOT NULL DEFAULT "0"'); + await this.updateToSchemaVersion(61); + } } /** diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 557d751e4..387304165 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -559,6 +559,8 @@ class WebsocketHandler { } if (Common.indexingEnabled() && memPool.isInSync()) { + logger.debug(`Auditing block ${block.height} (${block.id})`); + const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; @@ -571,11 +573,14 @@ class WebsocketHandler { }; }) : []; + const totalFees = stripped.reduce((total, transaction) => total + transaction.fee, 0); + logger.debug(`Projected block fees: ${totalFees} sats`); + BlocksSummariesRepository.$saveTemplate({ height: block.height, template: { id: block.id, - transactions: stripped + transactions: stripped, } }); @@ -588,6 +593,7 @@ class WebsocketHandler { freshTxs: fresh, sigopTxs: sigop, matchRate: matchRate, + expectedFees: totalFees }); if (block.extras) { diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index c3e0d02ba..8887dd45b 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -35,6 +35,7 @@ export interface BlockAudit { sigopTxs: string[], addedTxs: string[], matchRate: number, + expectedFees: number; } export interface AuditScore { diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 33075f43c..aaac7018c 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -52,7 +52,7 @@ class BlocksAuditRepositories { const [rows]: any[] = await DB.query( `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size, blocks.weight, blocks.tx_count, - transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, sigop_txs as sigopTxs, match_rate as matchRate + transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, sigop_txs as sigopTxs, match_rate as matchRate, expected_fees as expectedFees FROM blocks_audits JOIN blocks ON blocks.hash = blocks_audits.hash JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index c34a3e523..d14b70389 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -72,6 +72,15 @@ + + Expected total fees + + + + + + + diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index a2e7b6537..53763bbf9 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -149,6 +149,7 @@ export interface BlockAudit extends BlockExtended { missingTxs: string[], addedTxs: string[], matchRate: number, + expectedFees: number, template: TransactionStripped[], transactions: TransactionStripped[], } From 74b2014dffc2961bcb5b5ced3ea2a5658412c33b Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 6 Jun 2023 08:52:29 +0200 Subject: [PATCH 02/11] Show expected fees in blocks list --- backend/src/api/blocks.ts | 2 ++ backend/src/api/websocket-handler.ts | 1 + backend/src/mempool.interfaces.ts | 2 ++ backend/src/repositories/BlocksAuditsRepository.ts | 4 ++-- backend/src/repositories/BlocksRepository.ts | 2 ++ .../app/components/blocks-list/blocks-list.component.html | 5 +++++ frontend/src/app/interfaces/node-api.interface.ts | 1 + 7 files changed, 15 insertions(+), 2 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index fc12b5998..bae8e80dc 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -282,10 +282,12 @@ class Blocks { } extras.matchRate = null; + extras.expectedFees = null; if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); if (auditScore != null) { extras.matchRate = auditScore.matchRate; + extras.expectedFees = auditScore.expectedFees; } } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 387304165..8d9a74d12 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -598,6 +598,7 @@ class WebsocketHandler { if (block.extras) { block.extras.matchRate = matchRate; + block.extras.expectedFees = totalFees; block.extras.similarity = similarity; } } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 8887dd45b..bc73c0a8a 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -41,6 +41,7 @@ export interface BlockAudit { export interface AuditScore { hash: string, matchRate?: number, + expectedFees?: number } export interface MempoolBlock { @@ -183,6 +184,7 @@ export interface BlockExtension { feeRange: number[]; // fee rate percentiles reward: number; matchRate: number | null; + expectedFees: number | null; similarity?: number; pool: { id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id` diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index aaac7018c..0e77bdeb7 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -81,7 +81,7 @@ class BlocksAuditRepositories { public async $getBlockAuditScore(hash: string): Promise { try { const [rows]: any[] = await DB.query( - `SELECT hash, match_rate as matchRate + `SELECT hash, match_rate as matchRate, expected_fees as expectedFees FROM blocks_audits WHERE blocks_audits.hash = "${hash}" `); @@ -95,7 +95,7 @@ class BlocksAuditRepositories { public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise { try { const [rows]: any[] = await DB.query( - `SELECT hash, match_rate as matchRate + `SELECT hash, match_rate as matchRate, expected_fees as expectedFees FROM blocks_audits WHERE blocks_audits.height BETWEEN ? AND ? `, [minHeight, maxHeight]); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index a014e317e..fff9c1b1b 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1032,10 +1032,12 @@ class BlocksRepository { // Match rate is not part of the blocks table, but it is part of APIs so we must include it extras.matchRate = null; + extras.expectedFees = null; if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id); if (auditScore != null) { extras.matchRate = auditScore.matchRate; + extras.expectedFees = auditScore.expectedFees; } } diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index d6c229846..6748da448 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -16,6 +16,8 @@ Mined Health + Expected fees Reward Fees @@ -64,6 +66,9 @@ + + + diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 53763bbf9..3de82b910 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -133,6 +133,7 @@ export interface BlockExtension { reward?: number; coinbaseRaw?: string; matchRate?: number; + expectedFees?: number; similarity?: number; pool?: { id: number; From 7157efcf7917e0ba4122f04807668db355688c33 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Sat, 10 Jun 2023 11:08:32 +0200 Subject: [PATCH 03/11] Add CLA for joostjager Signed-off-by: Joost Jager --- contributors/joostjager.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/joostjager.txt diff --git a/contributors/joostjager.txt b/contributors/joostjager.txt new file mode 100644 index 000000000..2a31f307d --- /dev/null +++ b/contributors/joostjager.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022. + +Signed: joostjager From 3013386ca545165dd10e9f092930de4c7c8a5aa0 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 9 Jun 2023 13:46:14 -0400 Subject: [PATCH 04/11] Add expected weight to audit table --- backend/src/api/database-migration.ts | 3 +- backend/src/api/websocket-handler.ts | 13 ++-- backend/src/mempool.interfaces.ts | 5 +- .../repositories/BlocksAuditsRepository.ts | 20 ++++-- backend/src/repositories/BlocksRepository.ts | 1 + .../app/components/block/block.component.html | 64 ++++++++++++++++--- .../app/components/block/block.component.scss | 11 ++++ .../app/components/block/block.component.ts | 1 + .../src/app/interfaces/node-api.interface.ts | 2 + 9 files changed, 98 insertions(+), 22 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index ff06d3811..a87fdc0fd 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -534,7 +534,8 @@ class DatabaseMigration { } if (databaseSchemaVersion < 62 && isBitcoin === true) { - await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_weight BIGINT UNSIGNED DEFAULT NULL'); await this.updateToSchemaVersion(62); } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 8d9a74d12..b2c529220 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -559,8 +559,6 @@ class WebsocketHandler { } if (Common.indexingEnabled() && memPool.isInSync()) { - logger.debug(`Auditing block ${block.height} (${block.id})`); - const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; @@ -573,8 +571,12 @@ class WebsocketHandler { }; }) : []; - const totalFees = stripped.reduce((total, transaction) => total + transaction.fee, 0); - logger.debug(`Projected block fees: ${totalFees} sats`); + let totalFees = 0; + let totalWeight = 0; + for (const tx of stripped) { + totalFees += tx.fee; + totalWeight += (tx.vsize * 4); + } BlocksSummariesRepository.$saveTemplate({ height: block.height, @@ -593,7 +595,8 @@ class WebsocketHandler { freshTxs: fresh, sigopTxs: sigop, matchRate: matchRate, - expectedFees: totalFees + expectedFees: totalFees, + expectedWeight: totalWeight, }); if (block.extras) { diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index bc73c0a8a..478e88c33 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -35,13 +35,15 @@ export interface BlockAudit { sigopTxs: string[], addedTxs: string[], matchRate: number, - expectedFees: number; + expectedFees?: number, + expectedWeight?: number, } export interface AuditScore { hash: string, matchRate?: number, expectedFees?: number + expectedWeight?: number } export interface MempoolBlock { @@ -185,6 +187,7 @@ export interface BlockExtension { reward: number; matchRate: number | null; expectedFees: number | null; + expectedWeight: number | null; similarity?: number; pool: { id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id` diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 449c1d189..d755654ea 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees, expected_weight) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -51,7 +51,15 @@ class BlocksAuditRepositories { const [rows]: any[] = await DB.query( `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size, blocks.weight, blocks.tx_count, - transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, sigop_txs as sigopTxs, match_rate as matchRate, expected_fees as expectedFees + transactions, + template, + missing_txs as missingTxs, + added_txs as addedTxs, + fresh_txs as freshTxs, + sigop_txs as sigopTxs, + match_rate as matchRate, + expected_fees as expectedFees, + expected_weight as expectedWeight FROM blocks_audits JOIN blocks ON blocks.hash = blocks_audits.hash JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash @@ -81,7 +89,7 @@ class BlocksAuditRepositories { public async $getBlockAuditScore(hash: string): Promise { try { const [rows]: any[] = await DB.query( - `SELECT hash, match_rate as matchRate, expected_fees as expectedFees + `SELECT hash, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight FROM blocks_audits WHERE blocks_audits.hash = "${hash}" `); @@ -95,7 +103,7 @@ class BlocksAuditRepositories { public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise { try { const [rows]: any[] = await DB.query( - `SELECT hash, match_rate as matchRate, expected_fees as expectedFees + `SELECT hash, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight FROM blocks_audits WHERE blocks_audits.height BETWEEN ? AND ? `, [minHeight, maxHeight]); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index fff9c1b1b..9b07e6f03 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1038,6 +1038,7 @@ class BlocksRepository { if (auditScore != null) { extras.matchRate = auditScore.matchRate; extras.expectedFees = auditScore.expectedFees; + extras.expectedWeight = auditScore.expectedWeight; } } diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index d14b70389..4e58d32f2 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -72,15 +72,6 @@ - - Expected total fees - - - - - - - @@ -235,6 +226,9 @@ (txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)" [unavailable]="!isMobile && !showAudit"> + + +

Actual Block

@@ -244,6 +238,9 @@ (txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)" [unavailable]="isMobile && !showAudit">
+ + + @@ -394,5 +391,54 @@ + + + + + + + + + + + + + + + + +
Expected fees + +
Expected weight
Expected transactions{{ blockAudit.template?.length || 0 }}
+ + + + + + + + + + + + + + + + + + +
Actual fees + + + {{ blockAudit.feeDelta < 0 ? '+' : '' }}{{ (-blockAudit.feeDelta * 100) | amountShortener: 2 }}% + +
Actual weight + +
Actual transactions + {{ block.tx_count }} +
+ +

diff --git a/frontend/src/app/components/block/block.component.scss b/frontend/src/app/components/block/block.component.scss index 319f53804..48f094926 100644 --- a/frontend/src/app/components/block/block.component.scss +++ b/frontend/src/app/components/block/block.component.scss @@ -38,6 +38,17 @@ color: rgba(255, 255, 255, 0.4); margin-left: 5px; } + + .difference { + margin-left: 0.5em; + + &.positive { + color: rgb(66, 183, 71); + } + &.negative { + color: rgb(183, 66, 66); + } + } } } diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index f5fe1a469..671fd9d5b 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -388,6 +388,7 @@ export class BlockComponent implements OnInit, OnDestroy { for (const tx of blockAudit.transactions) { inBlock[tx.txid] = true; } + blockAudit.feeDelta = (blockAudit.expectedFees - this.block.extras.totalFees) / blockAudit.expectedFees; this.setAuditAvailable(true); } else { this.setAuditAvailable(false); diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 3de82b910..5aaeda545 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -151,6 +151,8 @@ export interface BlockAudit extends BlockExtended { addedTxs: string[], matchRate: number, expectedFees: number, + expectedWeight: number, + feeDelta?: number, template: TransactionStripped[], transactions: TransactionStripped[], } From 5b62966863a8bbcdb4f1aad0ad0042ac5a746599 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 9 Jun 2023 13:46:43 -0400 Subject: [PATCH 05/11] Add indexer task to backfill audit fee/weight stats --- backend/src/api/blocks.ts | 35 +++++++++++++++++ backend/src/indexer.ts | 1 + .../repositories/BlocksAuditsRepository.ts | 39 +++++++++++++++++++ .../repositories/BlocksSummariesRepository.ts | 15 +++++++ 4 files changed, 90 insertions(+) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index ef4db683e..116a99340 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -457,6 +457,41 @@ class Blocks { } } + /** + * [INDEXING] Index expected fees & weight for all audited blocks + */ + public async $generateAuditStats(): Promise { + const blockIds = await BlocksAuditsRepository.$getBlocksWithoutSummaries(); + if (!blockIds?.length) { + return; + } + let timer = Date.now(); + let indexedThisRun = 0; + let indexedTotal = 0; + logger.debug(`Indexing ${blockIds.length} block audit details`); + for (const hash of blockIds) { + const summary = await BlocksSummariesRepository.$getTemplate(hash); + let totalFees = 0; + let totalWeight = 0; + for (const tx of summary?.transactions || []) { + totalFees += tx.fee; + totalWeight += (tx.vsize * 4); + } + await BlocksAuditsRepository.$setSummary(hash, totalFees, totalWeight); + + indexedThisRun++; + indexedTotal++; + const elapsedSeconds = (Date.now() - timer) / 1000; + if (elapsedSeconds > 5) { + const blockPerSeconds = indexedThisRun / elapsedSeconds; + logger.debug(`Indexed ${indexedTotal} / ${blockIds.length} block audit details (${blockPerSeconds.toFixed(1)}/s)`); + timer = Date.now(); + indexedThisRun = 0; + } + } + logger.debug(`Indexing block audit details completed`); + } + /** * [INDEXING] Index all blocks metadata for the mining dashboard */ diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 3b16ad155..4b120867f 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -134,6 +134,7 @@ class Indexer { await mining.$generatePoolHashrateHistory(); await blocks.$generateBlocksSummariesDatabase(); await blocks.$generateCPFPDatabase(); + await blocks.$generateAuditStats(); } catch (e) { this.indexerRunning = false; logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index d755654ea..1fa2b0209 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -18,6 +18,19 @@ class BlocksAuditRepositories { } } + public async $setSummary(hash: string, expectedFees: number, expectedWeight: number) { + try { + await DB.query(` + UPDATE blocks_audits SET + expected_fees = ?, + expected_weight = ? + WHERE hash = ? + `, [expectedFees, expectedWeight, hash]); + } catch (e: any) { + logger.err(`Cannot update block audit in db. Reason: ` + (e instanceof Error ? e.message : e)); + } + } + public async $getBlocksHealthHistory(div: number, interval: string | null): Promise { try { let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`; @@ -113,6 +126,32 @@ class BlocksAuditRepositories { throw e; } } + + public async $getBlocksWithoutSummaries(): Promise { + try { + const [fromRows]: any[] = await DB.query(` + SELECT height + FROM blocks_audits + WHERE expected_fees IS NULL + ORDER BY height DESC + LIMIT 1 + `); + if (!fromRows?.length) { + return []; + } + const fromHeight = fromRows[0].height; + const [idRows]: any[] = await DB.query(` + SELECT hash + FROM blocks_audits + WHERE height <= ? + ORDER BY height DESC + `, [fromHeight]); + return idRows.map(row => row.hash); + } catch (e: any) { + logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksAuditRepositories(); diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 2d2c23d07..09598db03 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -50,6 +50,21 @@ class BlocksSummariesRepository { } } + public async $getTemplate(id: string): Promise { + try { + const [templates]: any[] = await DB.query(`SELECT * from blocks_templates WHERE id = ?`, [id]); + if (templates.length > 0) { + return { + id: templates[0].id, + transactions: JSON.parse(templates[0].template), + }; + } + } catch (e) { + logger.err(`Cannot get block template for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e)); + } + return undefined; + } + public async $getIndexedSummariesId(): Promise { try { const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`); From bfb842d7eaaf25a5c273b9793ec04e575c969f94 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 9 Jun 2023 14:19:26 -0400 Subject: [PATCH 06/11] Add % difference to weight and tx count in audit details --- frontend/src/app/components/block/block.component.html | 6 ++++++ frontend/src/app/components/block/block.component.ts | 6 +++++- frontend/src/app/interfaces/node-api.interface.ts | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 4e58d32f2..e3f4d195c 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -428,12 +428,18 @@ Actual weight + + {{ blockAudit.weightDelta < 0 ? '+' : '' }}{{ (-blockAudit.weightDelta * 100) | amountShortener: 2 }}% + Actual transactions {{ block.tx_count }} + + {{ blockAudit.txDelta < 0 ? '+' : '' }}{{ (-blockAudit.txDelta * 100) | amountShortener: 2 }}% + diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 671fd9d5b..be0e1318c 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -388,7 +388,11 @@ export class BlockComponent implements OnInit, OnDestroy { for (const tx of blockAudit.transactions) { inBlock[tx.txid] = true; } - blockAudit.feeDelta = (blockAudit.expectedFees - this.block.extras.totalFees) / blockAudit.expectedFees; + + blockAudit.feeDelta = blockAudit.expectedFees > 0 ? (blockAudit.expectedFees - this.block.extras.totalFees) / blockAudit.expectedFees : 0; + blockAudit.weightDelta = blockAudit.expectedWeight > 0 ? (blockAudit.expectedWeight - this.block.weight) / blockAudit.expectedWeight : 0; + blockAudit.txDelta = blockAudit.template.length > 0 ? (blockAudit.template.length - this.block.tx_count) / blockAudit.template.length : 0; + this.setAuditAvailable(true); } else { this.setAuditAvailable(false); diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 5aaeda545..cf1ae1fc0 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -153,6 +153,8 @@ export interface BlockAudit extends BlockExtended { expectedFees: number, expectedWeight: number, feeDelta?: number, + weightDelta?: number, + txDelta?: number, template: TransactionStripped[], transactions: TransactionStripped[], } From 93d24d1cf7fa1f7e37b766c7983b196e8656a5f0 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 10 Jun 2023 12:09:06 -0400 Subject: [PATCH 07/11] Add expected fee % diff to blocks list page --- backend/src/api/blocks.ts | 2 ++ backend/src/api/websocket-handler.ts | 1 + backend/src/repositories/BlocksRepository.ts | 1 + .../blocks-list/blocks-list.component.html | 15 ++++------ .../blocks-list/blocks-list.component.scss | 28 ++++++++++++++++--- .../blocks-list/blocks-list.component.ts | 9 +++++- .../src/app/interfaces/node-api.interface.ts | 2 ++ 7 files changed, 44 insertions(+), 14 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 116a99340..e080285ed 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -283,11 +283,13 @@ class Blocks { extras.matchRate = null; extras.expectedFees = null; + extras.expectedWeight = null; if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); if (auditScore != null) { extras.matchRate = auditScore.matchRate; extras.expectedFees = auditScore.expectedFees; + extras.expectedWeight = auditScore.expectedWeight; } } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index b2c529220..8aaab5ab5 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -602,6 +602,7 @@ class WebsocketHandler { if (block.extras) { block.extras.matchRate = matchRate; block.extras.expectedFees = totalFees; + block.extras.expectedWeight = totalWeight; block.extras.similarity = similarity; } } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 9b07e6f03..2689e19b3 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1033,6 +1033,7 @@ class BlocksRepository { // Match rate is not part of the blocks table, but it is part of APIs so we must include it extras.matchRate = null; extras.expectedFees = null; + extras.expectedWeight = null; if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id); if (auditScore != null) { diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index 6748da448..727ca2782 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -13,14 +13,12 @@ Pool Timestamp - Mined Health - Expected fees Reward Fees + vs Expected TXs Transactions @@ -44,9 +42,6 @@ ‎{{ block.timestamp * 1000 | date:'yyyy-MM-dd HH:mm' }} - - - - - - + + + {{ block.extras.feeDelta > 0 ? '+' : '' }}{{ (block.extras.feeDelta * 100) | amountShortener: 2 }}% + + {{ block.tx_count | number }} diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.scss b/frontend/src/app/components/blocks-list/blocks-list.component.scss index ea6e93347..39d6b55b0 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.scss +++ b/frontend/src/app/components/blocks-list/blocks-list.component.scss @@ -23,6 +23,17 @@ tr, td, th { border: 0px; padding-top: 0.65rem !important; padding-bottom: 0.7rem !important; + + .difference { + margin-left: 0.5em; + + &.positive { + color: rgb(66, 183, 71); + } + &.negative { + color: rgb(183, 66, 66); + } + } } .clear-link { @@ -90,7 +101,7 @@ tr, td, th { } .timestamp { - width: 18%; + width: 10%; @media (max-width: 1100px) { display: none; } @@ -123,8 +134,8 @@ tr, td, th { } .txs { - padding-right: 40px; - width: 8%; + padding-right: 20px; + width: 6%; @media (max-width: 1100px) { padding-right: 10px; } @@ -160,6 +171,15 @@ tr, td, th { .fees.widget { width: 20%; } +.fee-delta { + width: 6%; + @media (max-width: 991px) { + display: none; + } +} +.fee-delta.widget { + display: none; +} .reward { width: 8%; @@ -214,7 +234,7 @@ tr, td, th { .health { width: 10%; - @media (max-width: 1105px) { + @media (max-width: 1100px) { width: 13%; } @media (max-width: 560px) { diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.ts b/frontend/src/app/components/blocks-list/blocks-list.component.ts index 0086ff902..9ec09bfa9 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.ts +++ b/frontend/src/app/components/blocks-list/blocks-list.component.ts @@ -112,7 +112,14 @@ export class BlocksList implements OnInit, OnDestroy { acc = acc.slice(0, this.widget ? 6 : 15); } return acc; - }, []) + }, []), + switchMap((blocks) => { + console.log(blocks); + blocks.forEach(block => { + block.extras.feeDelta = block.extras.expectedFees ? (block.extras.totalFees - block.extras.expectedFees) / block.extras.expectedFees : 0; + }); + return of(blocks); + }) ); if (this.indexingAvailable && this.auditAvailable) { diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index cf1ae1fc0..2e58c79e4 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -134,6 +134,8 @@ export interface BlockExtension { coinbaseRaw?: string; matchRate?: number; expectedFees?: number; + expectedWeight?: number; + feeDelta?: number; similarity?: number; pool?: { id: number; From ae5a0312be37168e00f09b1e500640ec3c0c308c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 12 Jun 2023 12:40:19 -0400 Subject: [PATCH 08/11] change audit detail labels --- .../src/app/components/block/block.component.html | 12 ++++++------ .../src/app/components/block/block.component.scss | 7 +++++++ .../blocks-list/blocks-list.component.html | 9 ++++++--- .../blocks-list/blocks-list.component.scss | 1 + 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index e3f4d195c..7d2652829 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -395,17 +395,17 @@ - + - + - + @@ -416,7 +416,7 @@
Expected feesTotal fees
Expected weightWeight
Expected transactionsTransactions {{ blockAudit.template?.length || 0 }}
- + - + - + - + @@ -67,9 +67,9 @@ - + diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.scss b/frontend/src/app/components/blocks-list/blocks-list.component.scss index 39d6b55b0..3d3169a69 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.scss +++ b/frontend/src/app/components/blocks-list/blocks-list.component.scss @@ -173,6 +173,7 @@ tr, td, th { } .fee-delta { width: 6%; + padding-left: 0; @media (max-width: 991px) { display: none; } From 6b93e61b56247622230db2d270de7c546b13a492 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 14 Jun 2023 11:28:39 -0400 Subject: [PATCH 09/11] minor audit details fixes --- backend/src/api/database-migration.ts | 2 +- frontend/src/app/components/block/block.component.html | 4 ++-- .../src/app/components/blocks-list/blocks-list.component.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index a87fdc0fd..b1e54302a 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 61; + private static currentVersion = 62; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 7d2652829..b5bb7d5d3 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -392,7 +392,7 @@ -
Actual feesTotal fees @@ -425,7 +425,7 @@
Actual weightWeight @@ -434,7 +434,7 @@
Actual transactionsTransactions {{ block.tx_count }} diff --git a/frontend/src/app/components/block/block.component.scss b/frontend/src/app/components/block/block.component.scss index 48f094926..08091cb86 100644 --- a/frontend/src/app/components/block/block.component.scss +++ b/frontend/src/app/components/block/block.component.scss @@ -263,3 +263,10 @@ h1 { top: 11px; margin-left: 10px; } + +.audit-details-table { + margin-top: 1.25rem; + @media (max-width: 767.98px) { + margin-top: 0.75rem; + } +} diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index 727ca2782..5197ce3b4 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -18,7 +18,7 @@ Reward Feesvs Expected TXs Transactions + - {{ block.extras.feeDelta > 0 ? '+' : '' }}{{ (block.extras.feeDelta * 100) | amountShortener: 2 }}% + ({{ block.extras.feeDelta > 0 ? '+' : '' }}{{ (block.extras.feeDelta * 100) | amountShortener: 2 }}%) @@ -108,6 +108,9 @@ + +
+
@@ -413,7 +413,7 @@ -
Total fees
+
diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index 5197ce3b4..2ee611bc6 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -69,7 +69,7 @@
Total fees - ({{ block.extras.feeDelta > 0 ? '+' : '' }}{{ (block.extras.feeDelta * 100) | amountShortener: 2 }}%) + {{ block.extras.feeDelta > 0 ? '+' : '' }}{{ (block.extras.feeDelta * 100) | amountShortener: 2 }}% From c4f7b99978b101a5f301203c18767b810f7c59ba Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 14 Jun 2023 16:15:33 -0400 Subject: [PATCH 10/11] add backfilled audit stats to cached blocks --- backend/src/api/blocks.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index e080285ed..8d6a6b50d 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -480,6 +480,11 @@ class Blocks { totalWeight += (tx.vsize * 4); } await BlocksAuditsRepository.$setSummary(hash, totalFees, totalWeight); + const cachedBlock = this.blocks.find(block => block.id === hash); + if (cachedBlock) { + cachedBlock.extras.expectedFees = totalFees; + cachedBlock.extras.expectedWeight = totalWeight; + } indexedThisRun++; indexedTotal++; From 30f8d5cf9627f2e1806280af7e34c66c700e21e3 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 14 Jun 2023 16:15:58 -0400 Subject: [PATCH 11/11] add missing markForChecks in blocks list --- .../src/app/components/blocks-list/blocks-list.component.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.ts b/frontend/src/app/components/blocks-list/blocks-list.component.ts index 9ec09bfa9..8b4aa38e7 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.ts +++ b/frontend/src/app/components/blocks-list/blocks-list.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input } from '@angular/core'; +import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input, ChangeDetectorRef } from '@angular/core'; import { BehaviorSubject, combineLatest, concat, Observable, timer, EMPTY, Subscription, of } from 'rxjs'; import { catchError, delayWhen, map, retryWhen, scan, skip, switchMap, tap } from 'rxjs/operators'; import { BlockExtended } from '../../interfaces/node-api.interface'; @@ -39,6 +39,7 @@ export class BlocksList implements OnInit, OnDestroy { private apiService: ApiService, private websocketService: WebsocketService, public stateService: StateService, + private cd: ChangeDetectorRef, ) { } @@ -114,7 +115,6 @@ export class BlocksList implements OnInit, OnDestroy { return acc; }, []), switchMap((blocks) => { - console.log(blocks); blocks.forEach(block => { block.extras.feeDelta = block.extras.expectedFees ? (block.extras.totalFees - block.extras.expectedFees) / block.extras.expectedFees : 0; }); @@ -138,6 +138,7 @@ export class BlocksList implements OnInit, OnDestroy { this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null; }); this.loadingScores = false; + this.cd.markForCheck(); }); this.latestScoreSubscription = this.stateService.blocks$.pipe( @@ -162,6 +163,7 @@ export class BlocksList implements OnInit, OnDestroy { ).subscribe((score) => { if (score && score.hash) { this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null; + this.cd.markForCheck(); } }); }