diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index b1a2da18f..cecdee6bd 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -19,6 +19,9 @@ import HashratesRepository from '../repositories/HashratesRepository'; import indexer from '../indexer'; import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; +import mining from './mining'; +import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; +import difficultyAdjustment from './difficulty-adjustment'; class Blocks { private blocks: BlockExtended[] = []; @@ -449,7 +452,10 @@ class Blocks { const newBlock = await this.$indexBlock(lastBlock['height'] - i); await this.$getStrippedBlockTransactions(newBlock.id, true, true); } - logger.info(`Re-indexed 10 blocks and summaries`); + await mining.$indexDifficultyAdjustments(); + await DifficultyAdjustmentsRepository.$deleteLastAdjustment(); + logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`); + indexer.reindex(); } await blocksRepository.$saveBlockInDatabase(blockExtended); @@ -461,6 +467,15 @@ class Blocks { } if (block.height % 2016 === 0) { + if (Common.indexingEnabled()) { + await DifficultyAdjustmentsRepository.$saveAdjustments({ + time: block.timestamp, + height: block.height, + difficulty: block.difficulty, + adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise + }); + } + this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index f0f375309..8d9959cf2 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 21; + private static currentVersion = 22; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -226,6 +226,11 @@ class DatabaseMigration { await this.$executeQuery('DROP TABLE IF EXISTS `rates`'); await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices')); } + + if (databaseSchemaVersion < 22 && isBitcoin === true) { + await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`'); + await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments')); + } } catch (e) { throw e; } @@ -513,7 +518,7 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } - private getCreateRatesTableQuery(): string { + private getCreateRatesTableQuery(): string { // This table has been replaced by the prices table return `CREATE TABLE IF NOT EXISTS rates ( height int(10) unsigned NOT NULL, bisq_rates JSON NOT NULL, @@ -539,6 +544,17 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateDifficultyAdjustmentsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS difficulty_adjustments ( + time timestamp NOT NULL, + height int(10) unsigned NOT NULL, + difficulty double unsigned NOT NULL, + adjustment float NOT NULL, + PRIMARY KEY (height), + INDEX (time) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates', 'prices']; diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 977cbe4e8..672f0970f 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -1,4 +1,4 @@ -import { PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces'; +import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces'; import BlocksRepository from '../repositories/BlocksRepository'; import PoolsRepository from '../repositories/PoolsRepository'; import HashratesRepository from '../repositories/HashratesRepository'; @@ -7,6 +7,7 @@ import logger from '../logger'; import { Common } from './common'; import loadingIndicators from './loading-indicators'; import { escape } from 'mysql2'; +import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; class Mining { constructor() { @@ -377,6 +378,48 @@ class Mining { } } + /** + * Index difficulty adjustments + */ + public async $indexDifficultyAdjustments(): Promise { + const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights(); + const indexedHeights = {}; + for (const height of indexedHeightsArray) { + indexedHeights[height] = true; + } + + const blocks: any = await BlocksRepository.$getBlocksDifficulty(); + + let currentDifficulty = 0; + let totalIndexed = 0; + + for (const block of blocks) { + if (block.difficulty !== currentDifficulty) { + if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed + currentDifficulty = block.difficulty; + continue; + } + + let adjustment = block.difficulty / Math.max(1, currentDifficulty); + adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise + + await DifficultyAdjustmentsRepository.$saveAdjustments({ + time: block.time, + height: block.height, + difficulty: block.difficulty, + adjustment: adjustment, + }); + + totalIndexed++; + currentDifficulty = block.difficulty; + } + } + + if (totalIndexed > 0) { + logger.notice(`Indexed ${totalIndexed} difficulty adjustments`); + } + } + private getDateMidnight(date: Date): Date { date.setUTCHours(0); date.setUTCMinutes(0); diff --git a/backend/src/index.ts b/backend/src/index.ts index 566622055..a3f7d5d99 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -290,6 +290,7 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', routes.$getDifficultyAdjustments) ; } diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 96cca9f7f..cfdc4a7d0 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -36,6 +36,7 @@ class Indexer { return; } + await mining.$indexDifficultyAdjustments(); await this.$resetHashratesIndexingState(); await mining.$generateNetworkHashrateHistory(); await mining.$generatePoolHashrateHistory(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a35dc6d76..983434564 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -224,6 +224,13 @@ export interface IDifficultyAdjustment { timeOffset: number; } +export interface IndexedDifficultyAdjustment { + time: number; // UNIX timestamp + height: number; // Block height + difficulty: number; + adjustment: number; +} + export interface RewardStats { totalReward: number; totalFee: number; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 01b7622f3..e88ac7877 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -7,6 +7,7 @@ import PoolsRepository from './PoolsRepository'; import HashratesRepository from './HashratesRepository'; import { escape } from 'mysql2'; import BlocksSummariesRepository from './BlocksSummariesRepository'; +import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; class BlocksRepository { /** @@ -381,48 +382,9 @@ class BlocksRepository { /** * Return blocks difficulty */ - public async $getBlocksDifficulty(interval: string | null): Promise { - interval = Common.getSqlInterval(interval); - - // :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162 - // Basically, using temporary user defined fields, we are able to extract all - // difficulty adjustments from the blocks tables. - // This allow use to avoid indexing it in another table. - let query = ` - SELECT - * - FROM - ( - SELECT - UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, height, - IF(@prevStatus = YT.difficulty, @rn := @rn + 1, - IF(@prevStatus := YT.difficulty, @rn := 1, @rn := 1) - ) AS rn - FROM blocks YT - CROSS JOIN - ( - SELECT @prevStatus := -1, @rn := 1 - ) AS var - `; - - if (interval) { - query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; - } - - query += ` - ORDER BY YT.height - ) AS t - WHERE t.rn = 1 - ORDER BY t.height - `; - + public async $getBlocksDifficulty(): Promise { try { - const [rows]: any[] = await DB.query(query); - - for (const row of rows) { - delete row['rn']; - } - + const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`); return rows; } catch (e) { logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e)); @@ -452,26 +414,6 @@ class BlocksRepository { } } - /* - * Check if the last 10 blocks chain is valid - */ - public async $validateRecentBlocks(): Promise { - try { - const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`); - - for (let i = 0; i < lastBlocks.length - 1; ++i) { - if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) { - logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`); - return false; - } - } - - return true; - } catch (e) { - return true; // Don't do anything if there is a db error - } - } - /** * Check if the chain of block hash is valid and delete data from the stale branch if needed */ @@ -498,6 +440,7 @@ class BlocksRepository { await this.$deleteBlocksFrom(blocks[idx - 1].height); await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height); await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800); + await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height); return false; } ++idx; diff --git a/backend/src/repositories/DifficultyAdjustmentsRepository.ts b/backend/src/repositories/DifficultyAdjustmentsRepository.ts new file mode 100644 index 000000000..76324b5e6 --- /dev/null +++ b/backend/src/repositories/DifficultyAdjustmentsRepository.ts @@ -0,0 +1,88 @@ +import { Common } from '../api/common'; +import DB from '../database'; +import logger from '../logger'; +import { IndexedDifficultyAdjustment } from '../mempool.interfaces'; + +class DifficultyAdjustmentsRepository { + public async $saveAdjustments(adjustment: IndexedDifficultyAdjustment): Promise { + if (adjustment.height === 1) { + return; + } + + try { + const query = `INSERT INTO difficulty_adjustments(time, height, difficulty, adjustment) VALUE (FROM_UNIXTIME(?), ?, ?, ?)`; + const params: any[] = [ + adjustment.time, + adjustment.height, + adjustment.difficulty, + adjustment.adjustment, + ]; + await DB.query(query, params); + } catch (e: any) { + if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart + logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`); + } else { + logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + } + + public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise { + interval = Common.getSqlInterval(interval); + + let query = `SELECT UNIX_TIMESTAMP(time) as time, height, difficulty, adjustment + FROM difficulty_adjustments`; + + if (interval) { + query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; + } + + if (descOrder === true) { + query += ` ORDER BY time DESC`; + } else { + query += ` ORDER BY time`; + } + + try { + const [rows] = await DB.query(query); + return rows as IndexedDifficultyAdjustment[]; + } catch (e) { + logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + public async $getAdjustmentsHeights(): Promise { + try { + const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`); + return rows.map(block => block.height); + } catch (e: any) { + logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + + public async $deleteAdjustementsFromHeight(height: number): Promise { + try { + logger.info(`Delete newer difficulty adjustments from height ${height} from the database`); + await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]); + } catch (e: any) { + logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + + public async $deleteLastAdjustment(): Promise { + try { + logger.info(`Delete last difficulty adjustment from the database`); + await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`); + } catch (e: any) { + logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } +} + +export default new DifficultyAdjustmentsRepository(); + diff --git a/backend/src/routes.ts b/backend/src/routes.ts index b86187e4c..c0ba17a41 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -26,6 +26,7 @@ import mining from './api/mining'; import BlocksRepository from './repositories/BlocksRepository'; import HashratesRepository from './repositories/HashratesRepository'; import difficultyAdjustment from './api/difficulty-adjustment'; +import DifficultyAdjustmentsRepository from './repositories/DifficultyAdjustmentsRepository'; class Routes { constructor() {} @@ -653,7 +654,7 @@ class Routes { try { const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval); - const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval); + const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false); const blockCount = await BlocksRepository.$blockCount(null, null); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); @@ -730,6 +731,18 @@ class Routes { } } + public async $getDifficultyAdjustments(req: Request, res: Response) { + try { + const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, true); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment])); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + public async getBlock(req: Request, res: Response) { try { const block = await blocks.$getBlock(req.params.hash); diff --git a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html index 5f11c2608..1d6038070 100644 --- a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html +++ b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html @@ -9,7 +9,7 @@ - + {{ diffChange.height }} @@ -17,7 +17,7 @@ {{ diffChange.difficultyShorten }} - {{ diffChange.change >= 0 ? '+' : '' }}{{ diffChange.change | amountShortener }}% + {{ diffChange.change >= 0 ? '+' : '' }}{{ diffChange.change | amountShortener: 2 }}% diff --git a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts index fb7d5c8f7..32ad907d3 100644 --- a/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts +++ b/frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts @@ -30,27 +30,24 @@ export class DifficultyAdjustmentsTable implements OnInit { } ngOnInit(): void { - this.hashrateObservable$ = this.apiService.getHistoricalHashrate$('1y') + this.hashrateObservable$ = this.apiService.getDifficultyAdjustments$('3m') .pipe( map((response) => { const data = response.body; const tableData = []; - for (let i = data.difficulty.length - 1; i > 0; --i) { - const selectedPowerOfTen: any = selectPowerOfTen(data.difficulty[i].difficulty); - const change = (data.difficulty[i].difficulty / data.difficulty[i - 1].difficulty - 1) * 100; - - tableData.push(Object.assign(data.difficulty[i], { - change: Math.round(change * 100) / 100, + for (const adjustment of data) { + const selectedPowerOfTen: any = selectPowerOfTen(adjustment[2]); + tableData.push({ + height: adjustment[1], + timestamp: adjustment[0], + change: (adjustment[3] - 1) * 100, difficultyShorten: formatNumber( - data.difficulty[i].difficulty / selectedPowerOfTen.divider, + adjustment[2] / selectedPowerOfTen.divider, this.locale, '1.2-2') + selectedPowerOfTen.unit - })); + }); } this.isLoading = false; - - return { - difficulty: tableData.slice(0, 6), - }; + return tableData.slice(0, 6); }), ); } diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts index 900d6399d..1ee3c20c0 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts @@ -95,6 +95,7 @@ export class HashrateChartComponent implements OnInit { .pipe( tap((response) => { const data = response.body; + // We generate duplicated data point so the tooltip works nicely const diffFixed = []; let diffIndex = 1; @@ -112,7 +113,7 @@ export class HashrateChartComponent implements OnInit { } while (hashIndex < data.hashrates.length && diffIndex < data.difficulty.length && - data.hashrates[hashIndex].timestamp <= data.difficulty[diffIndex].timestamp + data.hashrates[hashIndex].timestamp <= data.difficulty[diffIndex].time ) { diffFixed.push({ timestamp: data.hashrates[hashIndex].timestamp, diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index bd32685fd..5efa745d1 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -140,7 +140,7 @@ export class ApiService { this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/pools` + (interval !== undefined ? `/${interval}` : ''), { observe: 'response' } ); - } + } getPoolStats$(slug: string): Observable { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/pool/${slug}`); @@ -172,6 +172,13 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/block/' + hash + '/summary'); } + getDifficultyAdjustments$(interval: string | undefined): Observable { + return this.httpClient.get( + this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/difficulty-adjustments` + + (interval !== undefined ? `/${interval}` : ''), { observe: 'response' } + ); + } + getHistoricalHashrate$(interval: string | undefined): Observable { return this.httpClient.get( this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/hashrate` + diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index 366a07345..f6ec8473d 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -67,6 +67,16 @@ do for url in / \ '/api/v1/mining/blocks/fee-rates/2y' \ '/api/v1/mining/blocks/fee-rates/3y' \ '/api/v1/mining/blocks/fee-rates/all' \ + '/api/v1/mining/difficulty-adjustments/24h' \ + '/api/v1/mining/difficulty-adjustments/3d' \ + '/api/v1/mining/difficulty-adjustments/1w' \ + '/api/v1/mining/difficulty-adjustments/1m' \ + '/api/v1/mining/difficulty-adjustments/3m' \ + '/api/v1/mining/difficulty-adjustments/6m' \ + '/api/v1/mining/difficulty-adjustments/1y' \ + '/api/v1/mining/difficulty-adjustments/2y' \ + '/api/v1/mining/difficulty-adjustments/3y' \ + '/api/v1/mining/difficulty-adjustments/all' \ do curl -s "https://${hostname}${url}" >/dev/null