From 900e02d9a539d4dac40dbe424edc5e880a1e8004 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 18 Apr 2022 17:49:22 +0900 Subject: [PATCH] Validate block hash chain after indexing and for new blocks --- backend/src/api/blocks.ts | 26 +++++++++---- backend/src/index.ts | 4 -- backend/src/repositories/BlocksRepository.ts | 39 ++++++++++++++++--- .../src/repositories/HashratesRepository.ts | 16 ++++++++ 4 files changed, 68 insertions(+), 17 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 4402f0d37..872627156 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -13,6 +13,8 @@ import blocksRepository from '../repositories/BlocksRepository'; import loadingIndicators from './loading-indicators'; import BitcoinApi from './bitcoin/bitcoin-api'; import { prepareBlock } from '../utils/blocks-utils'; +import BlocksRepository from '../repositories/BlocksRepository'; +import HashratesRepository from '../repositories/HashratesRepository'; class Blocks { private blocks: BlockExtended[] = []; @@ -23,7 +25,7 @@ class Blocks { private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; private blockIndexingStarted = false; public blockIndexingCompleted = false; - public reindexFlag = true; // Always re-index the latest indexed data in case the node went offline with an invalid block tip (reorg) + public reindexFlag = false; constructor() { } @@ -272,10 +274,13 @@ class Blocks { return; } - this.blockIndexingCompleted = true; + const chainValid = await BlocksRepository.$validateChain(); + this.reindexFlag = !chainValid; + this.blockIndexingCompleted = chainValid; } public async $updateBlocks() { + let fastForwarded = false; const blockHeightTip = await bitcoinApi.$getBlockHeightTip(); if (this.blocks.length === 0) { @@ -287,6 +292,7 @@ class Blocks { if (blockHeightTip - this.currentBlockHeight > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 2) { logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`); this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT; + fastForwarded = true; } if (!this.lastDifficultyAdjustmentTime) { @@ -324,12 +330,16 @@ class Blocks { const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions); if (Common.indexingEnabled()) { - await blocksRepository.$saveBlockInDatabase(blockExtended); - - // If the last 10 blocks chain is not valid, re-index them (reorg) - const chainValid = await blocksRepository.$validateRecentBlocks(); - if (!chainValid) { - this.reindexFlag = true; + if (!fastForwarded) { + const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1); + if (lastBlock !== null && blockExtended.id !== lastBlock['previousblockhash']) { + logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`); + await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 2); + await HashratesRepository.$deleteLastEntries(); + this.reindexFlag = true; + } else { + await blocksRepository.$saveBlockInDatabase(blockExtended); + } } } diff --git a/backend/src/index.ts b/backend/src/index.ts index 943448e3a..1b3273913 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -188,10 +188,6 @@ class Server { try { await poolsUpdater.updatePoolsJson(); - if (blocks.reindexFlag) { - await BlocksRepository.$deleteBlocks(10); - await HashratesRepository.$deleteLastEntries(); - } await blocks.$generateBlockDatabase(); await mining.$generateNetworkHashrateHistory(); await mining.$generatePoolHashrateHistory(); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 8e96a0c38..c8fbe428f 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -4,6 +4,7 @@ import logger from '../logger'; import { Common } from '../api/common'; import { prepareBlock } from '../utils/blocks-utils'; import PoolsRepository from './PoolsRepository'; +import HashratesRepository from './HashratesRepository'; class BlocksRepository { /** @@ -370,15 +371,43 @@ class BlocksRepository { } /** - * Delete $count blocks from the database + * Check if the chain of block hash is valid and delete data from the stale branch if needed */ - public async $deleteBlocks(count: number) { - logger.info(`Delete ${count} most recent indexed blocks from the database`); + public async $validateChain(): Promise { + try { + const start = new Date().getTime(); + const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash, + UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`); + + let currentHeight = 1; + while (currentHeight < blocks.length) { + if (blocks[currentHeight].previous_block_hash !== blocks[currentHeight - 1].hash) { + logger.warn(`Chain divergence detected at block ${blocks[currentHeight - 1].height}, re-indexing newer blocks and hashrates`); + await this.$deleteBlocksFrom(blocks[currentHeight - 1].height); + await HashratesRepository.$deleteHashratesFromTimestamp(blocks[currentHeight - 1].timestamp - 604800); + return false; + } + ++currentHeight; + } + + logger.info(`${currentHeight} blocks hash validated in ${new Date().getTime() - start} ms`); + return true; + } catch (e) { + logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e)); + return true; // Don't do anything if there is a db error + } + } + + /** + * Delete blocks from the database from blockHeight + */ + public async $deleteBlocksFrom(blockHeight: number) { + logger.info(`Delete newer blocks from height ${blockHeight} from the database`); try { - await DB.query(`DELETE FROM blocks ORDER BY height DESC LIMIT ${count};`); + await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`); } catch (e) { - logger.err('Cannot delete recent indexed blocks. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot delete indexed blocks. Reason: ' + (e instanceof Error ? e.message : e)); } } diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 8388b9122..c78f3ea42 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -195,6 +195,22 @@ class HashratesRepository { logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e)); } } + + /** + * Delete hashrates from the database from timestamp + */ + public async $deleteHashratesFromTimestamp(timestamp: number) { + logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`); + + try { + await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]); + // Re-run the hashrate indexing to fill up missing data + await this.$setLatestRunTimestamp('last_hashrates_indexing', 0); + await this.$setLatestRunTimestamp('last_weekly_hashrates_indexing', 0); + } catch (e) { + logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e)); + } + } } export default new HashratesRepository();