From ae59f95ba9bdbfde43714c173a5af5ae3b846715 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 19:27:19 +0900 Subject: [PATCH] Refactoring price update config. Fixing last price time. --- backend/mempool-config.sample.json | 8 ++------ .../src/__fixtures__/mempool-config.template.json | 12 ++++-------- backend/src/__tests__/config.test.ts | 9 +-------- backend/src/config.ts | 14 ++------------ backend/src/indexer.ts | 8 ++++++-- backend/src/repositories/PricesRepository.ts | 2 +- backend/src/tasks/price-updater.ts | 10 +++++----- docker/backend/mempool-config.json | 8 ++------ production/mempool-config.mainnet.json | 6 ++---- 9 files changed, 25 insertions(+), 52 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index fddd44781..7f8bb4e22 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -32,7 +32,8 @@ "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": 3600 }, "CORE_RPC": { "HOST": "127.0.0.1", @@ -115,11 +116,6 @@ "USERNAME": "", "PASSWORD": "" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": 300, - "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "https://mempool.space/api/v1", "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 5ed888962..583f8fb2d 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -23,8 +23,8 @@ "USER_AGENT": "__MEMPOOL_USER_AGENT__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": 14, - "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__POOLS_JSON_URL__", + "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "AUDIT": true, "ADVANCED_GBT_AUDIT": true, "ADVANCED_GBT_MEMPOOL": true, @@ -33,7 +33,8 @@ "MAX_BLOCKS_BULK_QUERY": 999, "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": "__MEMPOOL_PRICE_UPDATE_FREQUENCY__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -92,11 +93,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index cb564a334..c42ccd223 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,6 +47,7 @@ describe('Mempool Backend Config', () => { DISK_CACHE_BLOCK_INTERVAL: 6, MAX_PUSH_TX_SIZE_WEIGHT: 400000, ALLOW_UNREACHABLE: true, + PRICE_UPDATE_FREQUENCY: 3600, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); @@ -101,12 +102,6 @@ describe('Mempool Backend Config', () => { PASSWORD: '' }); - expect(config.PRICE_DATA_SERVER).toStrictEqual({ - UPDATE_FREQUENCY: 3600, - TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices' - }); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({ MEMPOOL_API: 'https://mempool.space/api/v1', MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -164,8 +159,6 @@ describe('Mempool Backend Config', () => { expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY); - expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); expect(config.REDIS).toStrictEqual(fixture.REDIS); diff --git a/backend/src/config.ts b/backend/src/config.ts index 5117983fb..1c59bcc46 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,6 +38,7 @@ interface IConfig { DISK_CACHE_BLOCK_INTERVAL: number; MAX_PUSH_TX_SIZE_WEIGHT: number; ALLOW_UNREACHABLE: boolean; + PRICE_UPDATE_FREQUENCY: number; }; ESPLORA: { REST_API_URL: string; @@ -115,11 +116,6 @@ interface IConfig { USERNAME: string; PASSWORD: string; }; - PRICE_DATA_SERVER: { - UPDATE_FREQUENCY: number; - TOR_URL: string; - CLEARNET_URL: string; - }; EXTERNAL_DATA_SERVER: { MEMPOOL_API: string; MEMPOOL_ONION: string; @@ -182,6 +178,7 @@ const defaults: IConfig = { 'DISK_CACHE_BLOCK_INTERVAL': 6, 'MAX_PUSH_TX_SIZE_WEIGHT': 400000, 'ALLOW_UNREACHABLE': true, + 'PRICE_UPDATE_FREQUENCY': 3600, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', @@ -259,11 +256,6 @@ const defaults: IConfig = { 'USERNAME': '', 'PASSWORD': '' }, - 'PRICE_DATA_SERVER': { - 'UPDATE_FREQUENCY': 3600, - 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' - }, 'EXTERNAL_DATA_SERVER': { 'MEMPOOL_API': 'https://mempool.space/api/v1', 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -304,7 +296,6 @@ class Config implements IConfig { LND: IConfig['LND']; CLIGHTNING: IConfig['CLIGHTNING']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; - PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; REPLICATION: IConfig['REPLICATION']; @@ -325,7 +316,6 @@ class Config implements IConfig { this.LND = configs.LND; this.CLIGHTNING = configs.CLIGHTNING; this.SOCKS5PROXY = configs.SOCKS5PROXY; - this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; this.REPLICATION = configs.REPLICATION; diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index d89a2647f..7ec65d9c9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -105,6 +105,12 @@ class Indexer { return; } + try { + await priceUpdater.$run(); + } catch (e) { + logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); + } + // Do not attempt to index anything unless Bitcoin Core is fully synced const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks !== blockchainInfo.headers) { @@ -119,8 +125,6 @@ class Indexer { await this.checkAvailableCoreIndexes(); try { - await priceUpdater.$run(); - const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index ed9d1fd72..7d77834f5 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -65,7 +65,7 @@ class PricesRepository { try { await DB.query(` - INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) + INSERT IGNORE INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] ); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 335a61065..91a36d04b 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.PRICE_DATA_SERVER.UPDATE_FREQUENCY) { - // Refresh every UPDATE_FREQUENCY seconds at most + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.MEMPOOL.PRICE_UPDATE_FREQUENCY) { + // Refresh every PRICE_UPDATE_FREQUENCY seconds at most return; } @@ -146,14 +146,11 @@ class PriceUpdater { } } - logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); - if (config.DATABASE.ENABLED === true) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 - this.latestPrices.time = nowRounded.getTime() / 1000; await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { this.lastRun = previousRun + 5 * 60; @@ -161,6 +158,9 @@ class PriceUpdater { } } + this.latestPrices.time = Math.round(new Date().getTime() / 1000); + logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); + if (this.ratesChangedCallback) { this.ratesChangedCallback(this.latestPrices); } diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8b8a79969..19d7965fd 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -33,7 +33,8 @@ "MAX_PUSH_TX_SIZE_WEIGHT": __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__, "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__" + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", + "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -111,11 +112,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 769dd8019..02078cf78 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -18,7 +18,8 @@ "USE_SECOND_NODE_FOR_MINFEE": true, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": 300 }, "SYSLOG" : { "MIN_PRIORITY": "debug" @@ -49,9 +50,6 @@ "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": 300 - }, "REPLICATION": { "ENABLED": true, "AUDIT": true,