diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 9e56db027..8d6a6b50d 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -282,10 +282,14 @@ class Blocks { } extras.matchRate = null; + extras.expectedFees = null; + extras.expectedWeight = null; if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); if (auditScore != null) { extras.matchRate = auditScore.matchRate; + extras.expectedFees = auditScore.expectedFees; + extras.expectedWeight = auditScore.expectedWeight; } } } @@ -455,6 +459,46 @@ class Blocks { } } + /** + * [INDEXING] Index expected fees & weight for all audited blocks + */ + public async $generateAuditStats(): Promise { + const blockIds = await BlocksAuditsRepository.$getBlocksWithoutSummaries(); + if (!blockIds?.length) { + return; + } + let timer = Date.now(); + let indexedThisRun = 0; + let indexedTotal = 0; + logger.debug(`Indexing ${blockIds.length} block audit details`); + for (const hash of blockIds) { + const summary = await BlocksSummariesRepository.$getTemplate(hash); + let totalFees = 0; + let totalWeight = 0; + for (const tx of summary?.transactions || []) { + totalFees += tx.fee; + totalWeight += (tx.vsize * 4); + } + await BlocksAuditsRepository.$setSummary(hash, totalFees, totalWeight); + const cachedBlock = this.blocks.find(block => block.id === hash); + if (cachedBlock) { + cachedBlock.extras.expectedFees = totalFees; + cachedBlock.extras.expectedWeight = totalWeight; + } + + indexedThisRun++; + indexedTotal++; + const elapsedSeconds = (Date.now() - timer) / 1000; + if (elapsedSeconds > 5) { + const blockPerSeconds = indexedThisRun / elapsedSeconds; + logger.debug(`Indexed ${indexedTotal} / ${blockIds.length} block audit details (${blockPerSeconds.toFixed(1)}/s)`); + timer = Date.now(); + indexedThisRun = 0; + } + } + logger.debug(`Indexing block audit details completed`); + } + /** * [INDEXING] Index all blocks metadata for the mining dashboard */ diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index ee208951f..22b42dac7 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 61; + private static currentVersion = 62; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -533,6 +533,12 @@ class DatabaseMigration { await this.updateToSchemaVersion(61); } + if (databaseSchemaVersion < 62 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_weight BIGINT UNSIGNED DEFAULT NULL'); + await this.updateToSchemaVersion(62); + } + } /** diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 557d751e4..8aaab5ab5 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -571,11 +571,18 @@ class WebsocketHandler { }; }) : []; + let totalFees = 0; + let totalWeight = 0; + for (const tx of stripped) { + totalFees += tx.fee; + totalWeight += (tx.vsize * 4); + } + BlocksSummariesRepository.$saveTemplate({ height: block.height, template: { id: block.id, - transactions: stripped + transactions: stripped, } }); @@ -588,10 +595,14 @@ class WebsocketHandler { freshTxs: fresh, sigopTxs: sigop, matchRate: matchRate, + expectedFees: totalFees, + expectedWeight: totalWeight, }); if (block.extras) { block.extras.matchRate = matchRate; + block.extras.expectedFees = totalFees; + block.extras.expectedWeight = totalWeight; block.extras.similarity = similarity; } } diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 3b16ad155..4b120867f 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -134,6 +134,7 @@ class Indexer { await mining.$generatePoolHashrateHistory(); await blocks.$generateBlocksSummariesDatabase(); await blocks.$generateCPFPDatabase(); + await blocks.$generateAuditStats(); } catch (e) { this.indexerRunning = false; logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 1ed8de8a3..3edd84cde 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -35,11 +35,15 @@ export interface BlockAudit { sigopTxs: string[], addedTxs: string[], matchRate: number, + expectedFees?: number, + expectedWeight?: number, } export interface AuditScore { hash: string, matchRate?: number, + expectedFees?: number + expectedWeight?: number } export interface MempoolBlock { @@ -182,6 +186,8 @@ export interface BlockExtension { feeRange: number[]; // fee rate percentiles reward: number; matchRate: number | null; + expectedFees: number | null; + expectedWeight: number | null; similarity?: number; pool: { id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id` diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 2401a65b3..1fa2b0209 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees, expected_weight) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -18,6 +18,19 @@ class BlocksAuditRepositories { } } + public async $setSummary(hash: string, expectedFees: number, expectedWeight: number) { + try { + await DB.query(` + UPDATE blocks_audits SET + expected_fees = ?, + expected_weight = ? + WHERE hash = ? + `, [expectedFees, expectedWeight, hash]); + } catch (e: any) { + logger.err(`Cannot update block audit in db. Reason: ` + (e instanceof Error ? e.message : e)); + } + } + public async $getBlocksHealthHistory(div: number, interval: string | null): Promise { try { let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`; @@ -51,7 +64,15 @@ class BlocksAuditRepositories { const [rows]: any[] = await DB.query( `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size, blocks.weight, blocks.tx_count, - transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, sigop_txs as sigopTxs, match_rate as matchRate + transactions, + template, + missing_txs as missingTxs, + added_txs as addedTxs, + fresh_txs as freshTxs, + sigop_txs as sigopTxs, + match_rate as matchRate, + expected_fees as expectedFees, + expected_weight as expectedWeight FROM blocks_audits JOIN blocks ON blocks.hash = blocks_audits.hash JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash @@ -81,7 +102,7 @@ class BlocksAuditRepositories { public async $getBlockAuditScore(hash: string): Promise { try { const [rows]: any[] = await DB.query( - `SELECT hash, match_rate as matchRate + `SELECT hash, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight FROM blocks_audits WHERE blocks_audits.hash = "${hash}" `); @@ -95,7 +116,7 @@ class BlocksAuditRepositories { public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise { try { const [rows]: any[] = await DB.query( - `SELECT hash, match_rate as matchRate + `SELECT hash, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight FROM blocks_audits WHERE blocks_audits.height BETWEEN ? AND ? `, [minHeight, maxHeight]); @@ -105,6 +126,32 @@ class BlocksAuditRepositories { throw e; } } + + public async $getBlocksWithoutSummaries(): Promise { + try { + const [fromRows]: any[] = await DB.query(` + SELECT height + FROM blocks_audits + WHERE expected_fees IS NULL + ORDER BY height DESC + LIMIT 1 + `); + if (!fromRows?.length) { + return []; + } + const fromHeight = fromRows[0].height; + const [idRows]: any[] = await DB.query(` + SELECT hash + FROM blocks_audits + WHERE height <= ? + ORDER BY height DESC + `, [fromHeight]); + return idRows.map(row => row.hash); + } catch (e: any) { + logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksAuditRepositories(); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 90cebf7e9..080de8480 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1018,10 +1018,14 @@ class BlocksRepository { // Match rate is not part of the blocks table, but it is part of APIs so we must include it extras.matchRate = null; + extras.expectedFees = null; + extras.expectedWeight = null; if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id); if (auditScore != null) { extras.matchRate = auditScore.matchRate; + extras.expectedFees = auditScore.expectedFees; + extras.expectedWeight = auditScore.expectedWeight; } } diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 2d2c23d07..09598db03 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -50,6 +50,21 @@ class BlocksSummariesRepository { } } + public async $getTemplate(id: string): Promise { + try { + const [templates]: any[] = await DB.query(`SELECT * from blocks_templates WHERE id = ?`, [id]); + if (templates.length > 0) { + return { + id: templates[0].id, + transactions: JSON.parse(templates[0].template), + }; + } + } catch (e) { + logger.err(`Cannot get block template for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e)); + } + return undefined; + } + public async $getIndexedSummariesId(): Promise { try { const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`); diff --git a/contributors/joostjager.txt b/contributors/joostjager.txt new file mode 100644 index 000000000..2a31f307d --- /dev/null +++ b/contributors/joostjager.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022. + +Signed: joostjager diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index c34a3e523..b5bb7d5d3 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -226,6 +226,9 @@ (txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)" [unavailable]="!isMobile && !showAudit"> + + +

Actual Block

@@ -235,6 +238,9 @@ (txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)" [unavailable]="isMobile && !showAudit">
+ + + @@ -385,5 +391,60 @@ + + + + + + + + + + + + + + + + +
Total fees + +
Weight
Transactions{{ blockAudit.template?.length || 0 }}
+ + + + + + + + + + + + + + + + + + +
Total fees + + + {{ blockAudit.feeDelta < 0 ? '+' : '' }}{{ (-blockAudit.feeDelta * 100) | amountShortener: 2 }}% + +
Weight + + + {{ blockAudit.weightDelta < 0 ? '+' : '' }}{{ (-blockAudit.weightDelta * 100) | amountShortener: 2 }}% + +
Transactions + {{ block.tx_count }} + + {{ blockAudit.txDelta < 0 ? '+' : '' }}{{ (-blockAudit.txDelta * 100) | amountShortener: 2 }}% + +
+ +

diff --git a/frontend/src/app/components/block/block.component.scss b/frontend/src/app/components/block/block.component.scss index 319f53804..08091cb86 100644 --- a/frontend/src/app/components/block/block.component.scss +++ b/frontend/src/app/components/block/block.component.scss @@ -38,6 +38,17 @@ color: rgba(255, 255, 255, 0.4); margin-left: 5px; } + + .difference { + margin-left: 0.5em; + + &.positive { + color: rgb(66, 183, 71); + } + &.negative { + color: rgb(183, 66, 66); + } + } } } @@ -252,3 +263,10 @@ h1 { top: 11px; margin-left: 10px; } + +.audit-details-table { + margin-top: 1.25rem; + @media (max-width: 767.98px) { + margin-top: 0.75rem; + } +} diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index f5fe1a469..be0e1318c 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -388,6 +388,11 @@ export class BlockComponent implements OnInit, OnDestroy { for (const tx of blockAudit.transactions) { inBlock[tx.txid] = true; } + + blockAudit.feeDelta = blockAudit.expectedFees > 0 ? (blockAudit.expectedFees - this.block.extras.totalFees) / blockAudit.expectedFees : 0; + blockAudit.weightDelta = blockAudit.expectedWeight > 0 ? (blockAudit.expectedWeight - this.block.weight) / blockAudit.expectedWeight : 0; + blockAudit.txDelta = blockAudit.template.length > 0 ? (blockAudit.template.length - this.block.tx_count) / blockAudit.template.length : 0; + this.setAuditAvailable(true); } else { this.setAuditAvailable(false); diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index d6c229846..2ee611bc6 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -13,12 +13,12 @@ Pool Timestamp - Mined Health Reward Fees + TXs Transactions @@ -42,9 +42,6 @@ ‎{{ block.timestamp * 1000 | date:'yyyy-MM-dd HH:mm' }} - - - + + + {{ block.extras.feeDelta > 0 ? '+' : '' }}{{ (block.extras.feeDelta * 100) | amountShortener: 2 }}% + + {{ block.tx_count | number }} @@ -106,6 +108,9 @@ + + + diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.scss b/frontend/src/app/components/blocks-list/blocks-list.component.scss index ea6e93347..3d3169a69 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.scss +++ b/frontend/src/app/components/blocks-list/blocks-list.component.scss @@ -23,6 +23,17 @@ tr, td, th { border: 0px; padding-top: 0.65rem !important; padding-bottom: 0.7rem !important; + + .difference { + margin-left: 0.5em; + + &.positive { + color: rgb(66, 183, 71); + } + &.negative { + color: rgb(183, 66, 66); + } + } } .clear-link { @@ -90,7 +101,7 @@ tr, td, th { } .timestamp { - width: 18%; + width: 10%; @media (max-width: 1100px) { display: none; } @@ -123,8 +134,8 @@ tr, td, th { } .txs { - padding-right: 40px; - width: 8%; + padding-right: 20px; + width: 6%; @media (max-width: 1100px) { padding-right: 10px; } @@ -160,6 +171,16 @@ tr, td, th { .fees.widget { width: 20%; } +.fee-delta { + width: 6%; + padding-left: 0; + @media (max-width: 991px) { + display: none; + } +} +.fee-delta.widget { + display: none; +} .reward { width: 8%; @@ -214,7 +235,7 @@ tr, td, th { .health { width: 10%; - @media (max-width: 1105px) { + @media (max-width: 1100px) { width: 13%; } @media (max-width: 560px) { diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.ts b/frontend/src/app/components/blocks-list/blocks-list.component.ts index 0086ff902..8b4aa38e7 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.ts +++ b/frontend/src/app/components/blocks-list/blocks-list.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input } from '@angular/core'; +import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input, ChangeDetectorRef } from '@angular/core'; import { BehaviorSubject, combineLatest, concat, Observable, timer, EMPTY, Subscription, of } from 'rxjs'; import { catchError, delayWhen, map, retryWhen, scan, skip, switchMap, tap } from 'rxjs/operators'; import { BlockExtended } from '../../interfaces/node-api.interface'; @@ -39,6 +39,7 @@ export class BlocksList implements OnInit, OnDestroy { private apiService: ApiService, private websocketService: WebsocketService, public stateService: StateService, + private cd: ChangeDetectorRef, ) { } @@ -112,7 +113,13 @@ export class BlocksList implements OnInit, OnDestroy { acc = acc.slice(0, this.widget ? 6 : 15); } return acc; - }, []) + }, []), + switchMap((blocks) => { + blocks.forEach(block => { + block.extras.feeDelta = block.extras.expectedFees ? (block.extras.totalFees - block.extras.expectedFees) / block.extras.expectedFees : 0; + }); + return of(blocks); + }) ); if (this.indexingAvailable && this.auditAvailable) { @@ -131,6 +138,7 @@ export class BlocksList implements OnInit, OnDestroy { this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null; }); this.loadingScores = false; + this.cd.markForCheck(); }); this.latestScoreSubscription = this.stateService.blocks$.pipe( @@ -155,6 +163,7 @@ export class BlocksList implements OnInit, OnDestroy { ).subscribe((score) => { if (score && score.hash) { this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null; + this.cd.markForCheck(); } }); } diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index a2e7b6537..2e58c79e4 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -133,6 +133,9 @@ export interface BlockExtension { reward?: number; coinbaseRaw?: string; matchRate?: number; + expectedFees?: number; + expectedWeight?: number; + feeDelta?: number; similarity?: number; pool?: { id: number; @@ -149,6 +152,11 @@ export interface BlockAudit extends BlockExtended { missingTxs: string[], addedTxs: string[], matchRate: number, + expectedFees: number, + expectedWeight: number, + feeDelta?: number, + weightDelta?: number, + txDelta?: number, template: TransactionStripped[], transactions: TransactionStripped[], }