From 0d92779971cd41a72be1ba04ef7c3ba2c63a91ea Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Thu, 1 Dec 2022 08:14:10 -0800 Subject: [PATCH 1/7] Update ignore rules and frequency of dependabot updates --- .github/dependabot.yml | 44 +++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 001ea3cb3..1e2574971 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,20 +1,28 @@ version: 2 updates: -- package-ecosystem: npm - directory: "/backend" - schedule: - interval: daily - open-pull-requests-limit: 10 -- package-ecosystem: npm - directory: "/frontend" - schedule: - interval: daily - open-pull-requests-limit: 10 -- package-ecosystem: docker - directory: "/docker/backend" - schedule: - interval: weekly -- package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" + - package-ecosystem: npm + directory: "/backend" + schedule: + interval: daily + open-pull-requests-limit: 10 + ignore: + - update-types: ["version-update:semver-major"] + - package-ecosystem: npm + directory: "/frontend" + schedule: + interval: daily + open-pull-requests-limit: 10 + ignore: + - update-types: ["version-update:semver-major"] + - package-ecosystem: docker + directory: "/docker/backend" + schedule: + interval: daily + ignore: + - update-types: ["version-update:semver-major"] + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: daily + ignore: + - update-types: ["version-update:semver-major"] From 132e848fdc7bf2457f44f19f50c33933e3209d7a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Dec 2022 10:49:10 +0900 Subject: [PATCH 2/7] Fix block summaries repo upsert race condition --- backend/src/api/websocket-handler.ts | 2 +- .../repositories/BlocksSummariesRepository.ts | 41 +++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 31224fc0c..246b5b90d 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -439,7 +439,7 @@ class WebsocketHandler { }; }) : []; - BlocksSummariesRepository.$saveSummary({ + BlocksSummariesRepository.$saveTemplate({ height: block.height, template: { id: block.id, diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 28b3cc7eb..1406a1d07 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -17,19 +17,16 @@ class BlocksSummariesRepository { return undefined; } - public async $saveSummary(params: { height: number, mined?: BlockSummary, template?: BlockSummary}) { - const blockId = params.mined?.id ?? params.template?.id; + public async $saveSummary(params: { height: number, mined?: BlockSummary}) { + const blockId = params.mined?.id; try { - const [dbSummary]: any[] = await DB.query(`SELECT * FROM blocks_summaries WHERE id = "${blockId}"`); - if (dbSummary.length === 0) { // First insertion - await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?, ?)`, [ - params.height, blockId, JSON.stringify(params.mined?.transactions ?? []), JSON.stringify(params.template?.transactions ?? []) - ]); - } else if (params.mined !== undefined) { // Update mined block summary - await DB.query(`UPDATE blocks_summaries SET transactions = ? WHERE id = "${params.mined.id}"`, [JSON.stringify(params.mined.transactions)]); - } else if (params.template !== undefined) { // Update template block summary - await DB.query(`UPDATE blocks_summaries SET template = ? WHERE id = "${params.template.id}"`, [JSON.stringify(params.template?.transactions)]); - } + const transactions = JSON.stringify(params.mined?.transactions || []); + await DB.query(` + INSERT INTO blocks_summaries (height, id, transactions, template) + VALUE (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + transactions = ? + `, [params.height, blockId, transactions, '[]', transactions]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`); @@ -40,6 +37,26 @@ class BlocksSummariesRepository { } } + public async $saveTemplate(params: { height: number, template: BlockSummary}) { + const blockId = params.template?.id; + try { + const transactions = JSON.stringify(params.template?.transactions || []); + await DB.query(` + INSERT INTO blocks_summaries (height, id, transactions, template) + VALUE (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + template = ? + `, [params.height, blockId, '[]', transactions, transactions]); + } catch (e: any) { + if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart + logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`); + } else { + logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + } + public async $getIndexedSummariesId(): Promise { try { const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`); From 3126a559a03794cddd0d412b44293e8c25d8a6cb Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Dec 2022 11:17:53 +0900 Subject: [PATCH 3/7] Make forensics backend call rate limiting configurable --- backend/mempool-config.sample.json | 3 ++- backend/src/__fixtures__/mempool-config.template.json | 3 ++- backend/src/config.ts | 2 ++ backend/src/tasks/lightning/forensics.service.ts | 7 +++---- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 6de690ad8..dbbc8412d 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -85,7 +85,8 @@ "STATS_REFRESH_INTERVAL": 600, "GRAPH_REFRESH_INTERVAL": 600, "LOGGER_UPDATE_INTERVAL": 30, - "FORENSICS_INTERVAL": 43200 + "FORENSICS_INTERVAL": 43200, + "FORENSICS_RATE_LIMIT": 20 }, "LND": { "TLS_CERT_PATH": "tls.cert", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 7a988a70d..2e9221c7a 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -101,7 +101,8 @@ "STATS_REFRESH_INTERVAL": 600, "GRAPH_REFRESH_INTERVAL": 600, "LOGGER_UPDATE_INTERVAL": 30, - "FORENSICS_INTERVAL": 43200 + "FORENSICS_INTERVAL": 43200, + "FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__" }, "LND": { "TLS_CERT_PATH": "", diff --git a/backend/src/config.ts b/backend/src/config.ts index 3a3d2b56d..e97deb5e5 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -44,6 +44,7 @@ interface IConfig { GRAPH_REFRESH_INTERVAL: number; LOGGER_UPDATE_INTERVAL: number; FORENSICS_INTERVAL: number; + FORENSICS_RATE_LIMIT: number; }; LND: { TLS_CERT_PATH: string; @@ -205,6 +206,7 @@ const defaults: IConfig = { 'GRAPH_REFRESH_INTERVAL': 600, 'LOGGER_UPDATE_INTERVAL': 30, 'FORENSICS_INTERVAL': 43200, + 'FORENSICS_RATE_LIMIT': 20, }, 'LND': { 'TLS_CERT_PATH': '', diff --git a/backend/src/tasks/lightning/forensics.service.ts b/backend/src/tasks/lightning/forensics.service.ts index 7acb36e89..c62639411 100644 --- a/backend/src/tasks/lightning/forensics.service.ts +++ b/backend/src/tasks/lightning/forensics.service.ts @@ -7,7 +7,6 @@ import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import { Common } from '../../api/common'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; -const throttleDelay = 20; //ms const tempCacheSize = 10000; class ForensicsService { @@ -91,7 +90,7 @@ class ForensicsService { let outspends: IEsploraApi.Outspend[] | undefined; try { outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id); - await Common.sleep$(throttleDelay); + await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); } catch (e) { logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); continue; @@ -340,7 +339,7 @@ class ForensicsService { let outspends: IEsploraApi.Outspend[] | undefined; try { outspends = await bitcoinApi.$getOutspends(input.txid); - await Common.sleep$(throttleDelay); + await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); } catch (e) { logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); } @@ -429,7 +428,7 @@ class ForensicsService { if (temp) { this.tempCached.push(txid); } - await Common.sleep$(throttleDelay); + await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); } catch (e) { logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); return null; From e54e896e569e09b6070e24792e5b1aa198fc8d43 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Dec 2022 12:10:54 +0900 Subject: [PATCH 4/7] fix skipped descendant updates on tx inclusion --- backend/src/api/tx-selection-worker.ts | 54 +++++++++++++------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/backend/src/api/tx-selection-worker.ts b/backend/src/api/tx-selection-worker.ts index 206d26fe3..ca40af84f 100644 --- a/backend/src/api/tx-selection-worker.ts +++ b/backend/src/api/tx-selection-worker.ts @@ -106,44 +106,44 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: if (nextTx && !nextTx?.used) { // Check if the package fits into this block if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { - blockWeight += nextTx.ancestorWeight; const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values()); const descendants: AuditTransaction[] = []; // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4); - + const used: AuditTransaction[] = []; while (sortedTxSet.length) { const ancestor = sortedTxSet.pop(); const mempoolTx = mempool[ancestor.txid]; - if (ancestor && !ancestor?.used) { - ancestor.used = true; - // update original copy of this tx with effective fee rate & relatives data - mempoolTx.effectiveFeePerVsize = effectiveFeeRate; - mempoolTx.ancestors = sortedTxSet.map((a) => { - return { - txid: a.txid, - fee: a.fee, - weight: a.weight, - }; - }).reverse(); - mempoolTx.descendants = descendants.map((a) => { - return { - txid: a.txid, - fee: a.fee, - weight: a.weight, - }; - }); - descendants.push(ancestor); - mempoolTx.cpfpChecked = true; - transactions.push(ancestor); - blockSize += ancestor.size; - } + ancestor.used = true; + ancestor.usedBy = nextTx.txid; + // update original copy of this tx with effective fee rate & relatives data + mempoolTx.effectiveFeePerVsize = effectiveFeeRate; + mempoolTx.ancestors = sortedTxSet.map((a) => { + return { + txid: a.txid, + fee: a.fee, + weight: a.weight, + }; + }).reverse(); + mempoolTx.descendants = descendants.map((a) => { + return { + txid: a.txid, + fee: a.fee, + weight: a.weight, + }; + }); + descendants.push(ancestor); + mempoolTx.cpfpChecked = true; + transactions.push(ancestor); + blockSize += ancestor.size; + blockWeight += ancestor.weight; + used.push(ancestor); } // remove these as valid package ancestors for any descendants remaining in the mempool - if (sortedTxSet.length) { - sortedTxSet.forEach(tx => { + if (used.length) { + used.forEach(tx => { updateDescendants(tx, auditPool, modified); }); } From 79f6ae3b6ff2f2839704622316e05fb625169e6c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Dec 2022 12:11:21 +0900 Subject: [PATCH 5/7] fix post-block advanced gbt call --- backend/src/api/websocket-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 31224fc0c..c9b7ff233 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -468,7 +468,7 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - await mempoolBlocks.makeBlockTemplates(_memPool, 2); + await mempoolBlocks.makeBlockTemplates(_memPool, 8, null, true); } else { mempoolBlocks.updateMempoolBlocks(_memPool); } From 685433fe4cc59ef77ee4683e235b188f71ef2397 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 5 Dec 2022 08:11:46 +0100 Subject: [PATCH 6/7] Handle ISP with no nodes --- backend/src/api/explorer/nodes.api.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 28241ad20..cf1c77686 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -538,6 +538,10 @@ class NodesApi { const IPSIds = ISPId.split(','); const [rows]: any = await DB.query(query, [IPSIds, IPSIds]); + if (!rows || rows.length === 0) { + return []; + } + const nodes = {}; const intISPIds: number[] = []; From 7a7172bb64d4addb257528e52235e5b25ed61513 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Mon, 5 Dec 2022 21:56:42 -0800 Subject: [PATCH 7/7] Change Bisq staging host to fra so tests can pass --- frontend/proxy.conf.staging.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/proxy.conf.staging.js b/frontend/proxy.conf.staging.js index a99ead7a4..0281b66ce 100644 --- a/frontend/proxy.conf.staging.js +++ b/frontend/proxy.conf.staging.js @@ -5,7 +5,7 @@ let PROXY_CONFIG = require('./proxy.conf'); PROXY_CONFIG.forEach(entry => { entry.target = entry.target.replace("mempool.space", "mempool-staging.tk7.mempool.space"); entry.target = entry.target.replace("liquid.network", "liquid-staging.tk7.mempool.space"); - entry.target = entry.target.replace("bisq.markets", "bisq-staging.tk7.mempool.space"); + entry.target = entry.target.replace("bisq.markets", "bisq-staging.fra.mempool.space"); }); module.exports = PROXY_CONFIG;