From 172c77328be75d13389f5cb2251269f702355a60 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 3 Aug 2023 17:42:41 +0900 Subject: [PATCH] Changing setting to per hour --- backend/mempool-config.sample.json | 2 +- .../__fixtures__/mempool-config.template.json | 2 +- backend/src/__tests__/config.test.ts | 2 +- backend/src/config.ts | 4 +- backend/src/indexer.ts | 2 +- backend/src/repositories/PricesRepository.ts | 2 +- backend/src/tasks/price-updater.ts | 56 ++++++++++++++----- docker/README.md | 4 +- docker/backend/mempool-config.json | 2 +- docker/backend/start.sh | 4 +- production/mempool-config.mainnet.json | 2 +- 11 files changed, 55 insertions(+), 27 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 7f8bb4e22..8fd59a430 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -33,7 +33,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 3600 }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index e3e8f5b14..471b7bb47 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -34,7 +34,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index c42ccd223..9cd547b90 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,7 +47,7 @@ describe('Mempool Backend Config', () => { DISK_CACHE_BLOCK_INTERVAL: 6, MAX_PUSH_TX_SIZE_WEIGHT: 400000, ALLOW_UNREACHABLE: true, - PRICE_UPDATE_FREQUENCY: 3600, + PRICE_UPDATES_PER_HOUR: 1, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 1c59bcc46..2e89bd525 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,7 +38,7 @@ interface IConfig { DISK_CACHE_BLOCK_INTERVAL: number; MAX_PUSH_TX_SIZE_WEIGHT: number; ALLOW_UNREACHABLE: boolean; - PRICE_UPDATE_FREQUENCY: number; + PRICE_UPDATES_PER_HOUR: number; }; ESPLORA: { REST_API_URL: string; @@ -178,7 +178,7 @@ const defaults: IConfig = { 'DISK_CACHE_BLOCK_INTERVAL': 6, 'MAX_PUSH_TX_SIZE_WEIGHT': 400000, 'ALLOW_UNREACHABLE': true, - 'PRICE_UPDATE_FREQUENCY': 3600, + 'PRICE_UPDATES_PER_HOUR': 1, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 392174ecf..7ec65d9c9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -106,7 +106,7 @@ class Indexer { } try { - await priceUpdater.$run(true); + await priceUpdater.$run(); } catch (e) { logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 7d77834f5..ed9d1fd72 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -65,7 +65,7 @@ class PricesRepository { try { await DB.query(` - INSERT IGNORE INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) + INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] ); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 05d6f0bec..bbfb7d167 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -25,7 +25,9 @@ export interface PriceHistory { class PriceUpdater { public historyInserted = false; - private lastRun = 0; + private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + private cyclePosition = -1; + private firstRun = true; private lastHistoricalRun = 0; private running = false; private feeds: PriceFeed[] = []; @@ -41,6 +43,8 @@ class PriceUpdater { this.feeds.push(new CoinbaseApi()); this.feeds.push(new BitfinexApi()); this.feeds.push(new GeminiApi()); + + this.setCyclePosition(); } public getLatestPrices(): ApiPrice { @@ -72,7 +76,7 @@ class PriceUpdater { this.latestPrices = await PricesRepository.$getLatestConversionRates(); } - public async $run(storeInDb: boolean = false): Promise { + public async $run(): Promise { if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') { // Coins have no value on testnet/signet, so we want to always show 0 return; @@ -89,7 +93,7 @@ class PriceUpdater { } try { - await this.$updatePrice(storeInDb); + await this.$updatePrice(); if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); } @@ -100,22 +104,42 @@ class PriceUpdater { this.running = false; } + private getMillisecondsSinceBeginningOfHour(): number { + const now = new Date(); + const beginningOfHour = new Date(now); + beginningOfHour.setMinutes(0, 0, 0); + return now.getTime() - beginningOfHour.getTime(); + } + + private setCyclePosition(): void { + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + for (let i = 0; i < config.MEMPOOL.PRICE_UPDATES_PER_HOUR; i++) { + if (this.timeBetweenUpdatesMs * i > millisecondsSinceBeginningOfHour) { + this.cyclePosition = i; + return; + } + } + this.cyclePosition = config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + } + /** * Fetch last BTC price from exchanges, average them, and save it in the database once every hour */ - private async $updatePrice(storeInDb: boolean): Promise { - if (this.lastRun === 0 && config.DATABASE.ENABLED === true) { - this.lastRun = await PricesRepository.$getLatestPriceTime(); + private async $updatePrice(): Promise { + let forceUpdate = false; + if (this.firstRun === true && config.DATABASE.ENABLED === true) { + const lastUpdate = await PricesRepository.$getLatestPriceTime(); + if (new Date().getTime() / 1000 - lastUpdate > this.timeBetweenUpdatesMs / 1000) { + forceUpdate = true; + } + this.firstRun = false; } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < Math.min(config.MEMPOOL.PRICE_UPDATE_FREQUENCY, 3600)) { - // Refresh at least every hour or PRICE_UPDATE_FREQUENCY seconds + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate) { return; } - const previousRun = this.lastRun; - this.lastRun = new Date().getTime() / 1000; - for (const currency of this.currencies) { let prices: number[] = []; @@ -146,14 +170,13 @@ class PriceUpdater { } } - if (config.DATABASE.ENABLED === true && storeInDb) { + if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { - this.lastRun = previousRun + 5 * 60; logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`); } } @@ -165,7 +188,12 @@ class PriceUpdater { this.ratesChangedCallback(this.latestPrices); } - this.lastRun = new Date().getTime() / 1000; + if (!forceUpdate) { + this.cyclePosition++; + if (this.cyclePosition > config.MEMPOOL.PRICE_UPDATES_PER_HOUR) { + this.cyclePosition = 0; + } + } if (this.latestPrices.USD === -1) { this.latestPrices = await PricesRepository.$getLatestConversionRates(); diff --git a/docker/README.md b/docker/README.md index 4270d11c8..13bda7ec6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -114,7 +114,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, "DISK_CACHE_BLOCK_INTERVAL": 6, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, ``` @@ -147,7 +147,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" - MEMPOOL_PRICE_UPDATE_FREQUENCY: "" + MEMPOOL_PRICE_UPDATES_PER_HOUR: "" ... ``` diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 19d7965fd..216ad95c2 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -34,7 +34,7 @@ "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", - "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__ + "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 8f8c79b41..30d2ac332 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} -__MEMPOOL_PRICE_UPDATE_FREQUENCY__=${MEMPOOL_PRICE_UPDATE_FREQUENCY:=3600} +__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=3600} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -173,7 +173,7 @@ sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__} sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json -sed -i "s!__MEMPOOL_PRICE_UPDATE_FREQUENCY__!${__MEMPOOL_PRICE_UPDATE_FREQUENCY__}!g" mempool-config.json +sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 02078cf78..f54635415 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -19,7 +19,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 300 + "PRICE_UPDATES_PER_HOUR": 12 }, "SYSLOG" : { "MIN_PRIORITY": "debug"