From e0952a4c1d7fd1fd41ccd2360c1ba34e6eb07a05 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 16 Jul 2022 09:22:45 +0200 Subject: [PATCH 01/53] Wait for the price updater to complete before saving blocks prices --- backend/src/api/blocks.ts | 15 ++++++++++ backend/src/api/mining/mining.ts | 14 +++++----- backend/src/indexer.ts | 29 +++++++++++++++++--- backend/src/repositories/PricesRepository.ts | 5 ++++ backend/src/tasks/price-updater.ts | 9 +++--- 5 files changed, 57 insertions(+), 15 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 30f9fbf78..45ffd6079 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -22,6 +22,8 @@ import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; +import PricesRepository from '../repositories/PricesRepository'; +import priceUpdater from '../tasks/price-updater'; class Blocks { private blocks: BlockExtended[] = []; @@ -457,6 +459,19 @@ class Blocks { } await blocksRepository.$saveBlockInDatabase(blockExtended); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === true && lastestPriceId !== null) { + await blocksRepository.$saveBlockPrices([{ + height: blockExtended.height, + priceId: lastestPriceId, + }]); + } else { + logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`) + setTimeout(() => { + indexer.runSingleTask('blocksPrices'); + }, 10000); + } + // Save blocks summary for visualization if it's enabled if (Common.blocksSummariesIndexingEnabled() === true) { await this.$getStrippedBlockTransactions(blockExtended.id, true); diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 55e749596..55cd33bd3 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -473,7 +473,7 @@ class Mining { for (const block of blocksWithoutPrices) { // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks - if (block.height < 68951) { + if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) { blocksPrices.push({ height: block.height, priceId: prices[0].id, @@ -492,11 +492,11 @@ class Mining { if (blocksPrices.length >= 100000) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); blocksPrices.length = 0; } @@ -504,11 +504,11 @@ class Mining { if (blocksPrices.length > 0) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); } } catch (e) { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index e452a42f4..26a407291 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -6,13 +6,12 @@ import logger from './logger'; import HashratesRepository from './repositories/HashratesRepository'; import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; +import PricesRepository from './repositories/PricesRepository'; class Indexer { runIndexer = true; indexerRunning = false; - - constructor() { - } + tasksRunning: string[] = []; public reindex() { if (Common.indexingEnabled()) { @@ -20,6 +19,28 @@ class Indexer { } } + public async runSingleTask(task: 'blocksPrices') { + if (!Common.indexingEnabled()) { + return; + } + + if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) { + this.tasksRunning.push(task); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === false || lastestPriceId === null) { + logger.debug(`Blocks prices indexer is waiting for the price updater to complete`) + setTimeout(() => { + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.runSingleTask('blocksPrices'); + }, 10000); + } else { + logger.debug(`Blocks prices indexer will run now`) + await mining.$indexBlockPrices(); + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + } + } + } + public async $run() { if (!Common.indexingEnabled() || this.runIndexer === false || this.indexerRunning === true || mempool.hasPriority() @@ -50,7 +71,7 @@ class Indexer { return; } - await mining.$indexBlockPrices(); + this.runSingleTask('blocksPrices'); await mining.$indexDifficultyAdjustments(); await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient await mining.$generateNetworkHashrateHistory(); diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 92fb4860f..cc79ff2a6 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -27,6 +27,11 @@ class PricesRepository { return oldestRow[0] ? oldestRow[0].time : 0; } + public async $getLatestPriceId(): Promise { + const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); + return oldestRow[0] ? oldestRow[0].id : null; + } + public async $getLatestPriceTime(): Promise { const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); return oldestRow[0] ? oldestRow[0].time : 0; diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index a5901d7f7..81066efb2 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import { Common } from '../api/common'; import config from '../config'; import logger from '../logger'; import PricesRepository from '../repositories/PricesRepository'; @@ -34,10 +35,10 @@ export interface Prices { } class PriceUpdater { - historyInserted: boolean = false; - lastRun: number = 0; - lastHistoricalRun: number = 0; - running: boolean = false; + public historyInserted = false; + lastRun = 0; + lastHistoricalRun = 0; + running = false; feeds: PriceFeed[] = []; currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; latestPrices: Prices; From eaf7da9acbae0dfb19f0d0fd6619623e7270019e Mon Sep 17 00:00:00 2001 From: Stephan Oeste Date: Fri, 29 Jul 2022 20:13:48 +0200 Subject: [PATCH 02/53] Add Unfurl to the prod installer --- production/install | 173 +++++++++++++++++++------- production/mempool-config.unfurl.json | 13 ++ production/unfurl-build | 62 +++++++++ production/unfurl-kill | 2 + production/unfurl-start | 6 + 5 files changed, 211 insertions(+), 45 deletions(-) create mode 100644 production/mempool-config.unfurl.json create mode 100755 production/unfurl-build create mode 100755 production/unfurl-kill create mode 100755 production/unfurl-start diff --git a/production/install b/production/install index 729ff33e0..053ffef70 100755 --- a/production/install +++ b/production/install @@ -39,6 +39,9 @@ BITCOIN_INSTALL=ON BISQ_INSTALL=ON ELEMENTS_INSTALL=ON +# install UNFURL +UNFURL_INSTALL=ON + # configure 4 network instances BITCOIN_MAINNET_ENABLE=ON BITCOIN_MAINNET_MINFEE_ENABLE=ON @@ -48,6 +51,9 @@ BISQ_MAINNET_ENABLE=ON ELEMENTS_LIQUID_ENABLE=ON ELEMENTS_LIQUIDTESTNET_ENABLE=ON +# install Electrs +ELECTRS_INSTALL=ON + # enable lightmode and disable compaction to fit on 1TB SSD drive BITCOIN_ELECTRS_LIGHT_MODE=ON BITCOIN_ELECTRS_COMPACTION=OFF @@ -278,6 +284,12 @@ BISQ_GROUP=bisq # bisq home folder, needs about 1GB BISQ_HOME=/bisq +# Unfurl user/group +UNFURL_USER=unfurl +UNFURL_GROUP=unfurl +# Unfurl home folder +UNFURL_HOME=/unfurl + # liquid user/group ELEMENTS_USER=elements ELEMENTS_GROUP=elements @@ -315,6 +327,13 @@ BISQ_REPO_BRANCH=master BISQ_LATEST_RELEASE=master echo -n '.' +UNFURL_REPO_URL=https://github.com/mempool/mempool +UNFURL_REPO_NAME=unfurl +UNFURL_REPO_BRANCH=master +#UNFURL_LATEST_RELEASE=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4) +UNFURL_LATEST_RELEASE=master +echo -n '.' + ELEMENTS_REPO_URL=https://github.com/ElementsProject/elements ELEMENTS_REPO_NAME=elements ELEMENTS_REPO_BRANCH=master @@ -351,6 +370,10 @@ DEBIAN_PKG+=(libboost-system-dev libboost-filesystem-dev libboost-chrono-dev lib DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python3-certbot-nginx rsync ufw) DEBIAN_PKG+=(geoipupdate) +DEBIAN_UNFURL_PKG=() +DEBIAN_UNFURL_PKG+=(cups chromium-bsu libatk1.0 libatk-bridge2.0 libxkbcommon-dev libxcomposite-dev) +DEBIAN_UNFURL_PKG+=(libxdamage-dev libxrandr-dev libgbm-dev libpango1.0-dev libasound-dev) + # packages needed for mempool ecosystem FREEBSD_PKG=() FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim) @@ -712,6 +735,7 @@ Signet:Enable Bitcoin Signet:ON Liquid:Enable Elements Liquid:ON Liquidtestnet:Enable Elements Liquidtestnet:ON Bisq:Enable Bisq:ON +Unfurl:Enable Unfurl:ON EOF cat $input | sed -e 's/^/"/' -e 's/:/" "/g' -e 's/$/"/' >$output @@ -785,6 +809,12 @@ else ELEMENTS_INSTALL=OFF fi +if [ "${BITCOIN_INSTALL}" = ON -o "${ELEMENTS_INSTALL}" = ON ];then + ELECTRS_INSTALL=ON +else + ELECTRS_INSTALL=OFF +fi + if grep Bisq $tempfile >/dev/null 2>&1;then BISQ_INSTALL=ON BISQ_MAINNET_ENABLE=ON @@ -793,6 +823,12 @@ else BISQ_MAINNET_ENABLE=OFF fi +if grep Unfurl $tempfile >/dev/null 2>&1;then + UNFURL_INSTALL=ON +else + UNFURL_INSTALL=OFF +fi + ################## ## dialog part 2 # ################## @@ -1074,52 +1110,55 @@ fi # Bitcoin -> Electrs installation # ################################### -echo "[*] Creating Bitcoin Electrs data folder" -osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}" -osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_ELECTRS_HOME}" -if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then - osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_MAINNET_DATA}" +if [ "${ELECTRS_INSTALL}" = ON ];then + + echo "[*] Creating Bitcoin Electrs data folder" + osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}" + osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_ELECTRS_HOME}" + if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then + osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_MAINNET_DATA}" + fi + if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then + osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_TESTNET_DATA}" + fi + if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then + osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_SIGNET_DATA}" + fi + + echo "[*] Cloning Bitcoin Electrs repo from ${BITCOIN_ELECTRS_REPO_URL}" + osSudo "${BITCOIN_USER}" git config --global advice.detachedHead false + osSudo "${BITCOIN_USER}" git clone --branch "${BITCOIN_ELECTRS_REPO_BRANCH}" "${BITCOIN_ELECTRS_REPO_URL}" "${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME}" + + echo "[*] Checking out Electrs ${BITCOIN_ELECTRS_LATEST_RELEASE}" + osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME} && git checkout ${BITCOIN_ELECTRS_LATEST_RELEASE}" + + case $OS in + FreeBSD) + echo "[*] Installing Rust from pkg install" + ;; + Debian) + echo "[*] Installing Rust from rustup.rs" + osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y" + ;; + esac + + echo "[*] Building Bitcoin Electrs release binary" + osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" || true + + case $OS in + FreeBSD) + echo "[*] Patching Bitcoin Electrs code for FreeBSD" + osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\"" + osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/new_index/\" && sed -i.bak -e s/Snappy/None/ db.rs && rm db.rs.bak" + osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i.bak -e 's/from_secs(5)/from_secs(1)/' electrs.rs && rm electrs.rs.bak" + ;; + Debian) + ;; + esac + + echo "[*] Building Bitcoin Electrs release binary" + osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" fi -if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then - osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_TESTNET_DATA}" -fi -if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then - osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_SIGNET_DATA}" -fi - -echo "[*] Cloning Bitcoin Electrs repo from ${BITCOIN_ELECTRS_REPO_URL}" -osSudo "${BITCOIN_USER}" git config --global advice.detachedHead false -osSudo "${BITCOIN_USER}" git clone --branch "${BITCOIN_ELECTRS_REPO_BRANCH}" "${BITCOIN_ELECTRS_REPO_URL}" "${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME}" - -echo "[*] Checking out Electrs ${BITCOIN_ELECTRS_LATEST_RELEASE}" -osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME} && git checkout ${BITCOIN_ELECTRS_LATEST_RELEASE}" - -case $OS in - FreeBSD) - echo "[*] Installing Rust from pkg install" - ;; - Debian) - echo "[*] Installing Rust from rustup.rs" - osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y" - ;; -esac - -echo "[*] Building Bitcoin Electrs release binary" -osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" || true - -case $OS in - FreeBSD) - echo "[*] Patching Bitcoin Electrs code for FreeBSD" - osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\"" - osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/new_index/\" && sed -i.bak -e s/Snappy/None/ db.rs && rm db.rs.bak" - osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i.bak -e 's/from_secs(5)/from_secs(1)/' electrs.rs && rm electrs.rs.bak" - ;; - Debian) - ;; -esac - -echo "[*] Building Bitcoin Electrs release binary" -osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" ################################## # Liquid -> Electrs installation # @@ -1246,6 +1285,50 @@ if [ "${BISQ_INSTALL}" = ON ];then esac fi +####################### +# Unfurl installation # +####################### + +if [ "${UNFURL_INSTALL}" = ON ];then + + echo "[*] Creating Unfurl user" + osGroupCreate "${UNFURL_GROUP}" + osUserCreate "${UNFURL_USER}" "${UNFURL_HOME}" "${UNFURL_GROUP}" + osSudo "${ROOT_USER}" chsh -s `which zsh` "${UNFURL_USER}" + + echo "[*] Creating Unfurl folder" + osSudo "${ROOT_USER}" mkdir -p "${UNFURL_HOME}" + osSudo "${ROOT_USER}" chown -R "${UNFURL_USER}:${UNFURL_GROUP}" "${UNFURL_HOME}" + osSudo "${UNFURL_USER}" touch "${UNFURL_HOME}/.zshrc" + + echo "[*] Insalling Unfurl source" + case $OS in + + FreeBSD) + echo "[*] FIXME: Unfurl must be installed manually on FreeBSD" + ;; + + Debian) + echo "[*] Installing packages for Unfurl" + osPackageInstall ${DEBIAN_UNFURL_PKG[@]} + echo "[*] Cloning Mempool (Unfurl) repo from ${UNFURL_REPO_URL}" + osSudo "${UNFURL_USER}" git config --global pull.rebase true + osSudo "${UNFURL_USER}" git config --global advice.detachedHead false + osSudo "${UNFURL_USER}" git clone --branch "${UNFURL_REPO_BRANCH}" "${UNFURL_REPO_URL}" "${UNFURL_HOME}/${UNFURL_REPO_NAME}" + osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-build upgrade + osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-kill stop + osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-start start + echo "[*] Installing nvm.sh from GitHub" + osSudo "${UNFURL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh' + + echo "[*] Building NodeJS via nvm.sh" + osSudo "${UNFURL_USER}" zsh -c 'source ~/.zshrc ; nvm install v16.16.0 --shared-zlib' + + ;; + esac + +fi + ################################ # Bitcoin instance for Mainnet # ################################ diff --git a/production/mempool-config.unfurl.json b/production/mempool-config.unfurl.json new file mode 100644 index 000000000..5cf67d5ac --- /dev/null +++ b/production/mempool-config.unfurl.json @@ -0,0 +1,13 @@ +{ + "SERVER": { + "HOST": "https://mempool.space", + "HTTP_PORT": 8001 + }, + "MEMPOOL": { + "HTTP_HOST": "https://mempool.space", + "HTTP_PORT": 443 + }, + "PUPPETEER": { + "CLUSTER_SIZE": 8 + } +} diff --git a/production/unfurl-build b/production/unfurl-build new file mode 100755 index 000000000..5b838e0ae --- /dev/null +++ b/production/unfurl-build @@ -0,0 +1,62 @@ +#!/usr/bin/env zsh +PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin +HOSTNAME=$(hostname) +LOCATION=$(hostname|cut -d . -f2) +LOCKFILE="${HOME}/lock" +REF=$(echo "${1:=origin/master}"|sed -e 's!:!/!') + +if [ -f "${LOCKFILE}" ];then + echo "upgrade already running? check lockfile ${LOCKFILE}" + exit 1 +fi + +# on exit, remove lockfile but preserve exit code +trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT + +# create lockfile +touch "${LOCKFILE}" + +# notify logged in users +echo "Upgrading unfurler to ${REF}" | wall + +update_repo() +{ + echo "[*] Upgrading unfurler to ${REF}" + cd "$HOME/unfurl/unfurler" || exit 1 + + git fetch origin || exit 1 + for remote in origin;do + git remote add "${remote}" "https://github.com/${remote}/mempool" >/dev/null 2>&1 + git fetch "${remote}" || exit 1 + done + + if [ $(git tag -l "${REF}") ];then + git reset --hard "tags/${REF}" || exit 1 + elif [ $(git branch -r -l "origin/${REF}") ];then + git reset --hard "origin/${REF}" || exit 1 + else + git reset --hard "${REF}" || exit 1 + fi + export HASH=$(git rev-parse HEAD) +} + +build_backend() +{ + echo "[*] Building backend for unfurler" + [ -z "${HASH}" ] && exit 1 + cd "$HOME/unfurl/unfurler" || exit 1 + if [ ! -e "config.json" ];then + cp "${HOME}/unfurl/production/mempool-config.unfurl.json" "config.json" + fi + npm install || exit 1 + npm run build || exit 1 +} + +update_repo +build_backend + +# notify everyone +echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev +echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}" + +exit 0 diff --git a/production/unfurl-kill b/production/unfurl-kill new file mode 100755 index 000000000..ae48552c2 --- /dev/null +++ b/production/unfurl-kill @@ -0,0 +1,2 @@ +#!/usr/bin/env zsh +killall sh node diff --git a/production/unfurl-start b/production/unfurl-start new file mode 100755 index 000000000..29b5ddf3e --- /dev/null +++ b/production/unfurl-start @@ -0,0 +1,6 @@ +#!/usr/bin/env zsh +export NVM_DIR="$HOME/.nvm" +source "$NVM_DIR/nvm.sh" + +cd "${HOME}/unfurl/unfurler/" && \ +screen -dmS "unfurl" sh -c 'while true;do npm run start-production;sleep 1;done' From 21b6c6158ad44300caa37c92025d7587a2dcf5f4 Mon Sep 17 00:00:00 2001 From: Stephan Oeste Date: Sat, 30 Jul 2022 13:58:41 +0200 Subject: [PATCH 03/53] Fix tor config for FreeBSD on prod installer --- production/install | 39 +++++++++++++++++++++++++++++++-------- production/torrc | 8 -------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/production/install b/production/install index 729ff33e0..4cb37e6af 100755 --- a/production/install +++ b/production/install @@ -178,7 +178,7 @@ case $OS in ROOT_GROUP=wheel ROOT_HOME=/root TOR_HOME=/var/db/tor - TOR_CONFIGURATION=/usr/local/etc/tor/torrc + TOR_CONFIGURATION=/var/db/tor/torrc TOR_RESOURCES=/var/db/tor TOR_PKG=tor TOR_USER=_tor @@ -277,6 +277,8 @@ BISQ_USER=bisq BISQ_GROUP=bisq # bisq home folder, needs about 1GB BISQ_HOME=/bisq +# tor HS folder +BISQ_TOR_HS=bisq # liquid user/group ELEMENTS_USER=elements @@ -287,6 +289,8 @@ ELEMENTS_HOME=/elements ELECTRS_HOME=/electrs # elements electrs source/binaries ELEMENTS_ELECTRS_HOME=${ELEMENTS_HOME}/electrs +# tor HS folder +LIQUID_TOR_HS=liquid # minfee user/group MINFEE_USER=minfee @@ -941,14 +945,33 @@ if [ "${TOR_INSTALL}" = ON ];then echo "[*] Installing Tor base configuration" osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/torrc" "${TOR_HOME}/torrc" + osSudo "${ROOT_USER}" sed -i.orig "s!__TOR_RESOURCES__!${TOR_RESOURCES}!" "${TOR_CONFIGURATION}" - echo "[*] Adding Tor HS configuration" - if ! grep "${MEMPOOL_TOR_HS}" /etc/tor/torrc >/dev/null 2>&1;then - osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${MEMPOOL_TOR_HS}/ >> ${TOR_CONFIGURATION}" - osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:81 >> ${TOR_CONFIGURATION}" - osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}" - else - osSudo "${ROOT_USER}" sed -i.orig "s!__TOR_RESOURCES__!${TOR_RESOURCES}!" "${TOR_CONFIGURATION}" + echo "[*] Adding Tor HS configuration for Mempool" + if [ "${MEMPOOL_ENABLE}" = "ON" ];then + if ! grep "${MEMPOOL_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then + osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${MEMPOOL_TOR_HS}/ >> ${TOR_CONFIGURATION}" + osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:81 >> ${TOR_CONFIGURATION}" + osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}" + fi + fi + + echo "[*] Adding Tor HS configuration for Bisq" + if [ "${BISQ_ENABLE}" = "ON" ];then + if ! grep "${BISQ_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then + osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${BISQ_TOR_HS}/ >> ${TOR_CONFIGURATION}" + osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:82 >> ${TOR_CONFIGURATION}" + osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}" + fi + fi + + echo "[*] Adding Tor HS configuration for Liquid" + if [ "${LIQUID_ENABLE}" = "ON" ];then + if ! grep "${LIQUID_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then + osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${LIQUID_TOR_HS}/ >> ${TOR_CONFIGURATION}" + osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:83 >> ${TOR_CONFIGURATION}" + osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}" + fi fi case $OS in diff --git a/production/torrc b/production/torrc index 454cafba0..344ebc6e4 100644 --- a/production/torrc +++ b/production/torrc @@ -13,11 +13,3 @@ CookieAuthFileGroupReadable 1 HiddenServiceDir __TOR_RESOURCES__/mempool HiddenServicePort 80 127.0.0.1:81 HiddenServiceVersion 3 - -HiddenServiceDir __TOR_RESOURCES__/bisq -HiddenServicePort 80 127.0.0.1:82 -HiddenServiceVersion 3 - -HiddenServiceDir __TOR_RESOURCES__/liquid -HiddenServicePort 80 127.0.0.1:83 -HiddenServiceVersion 3 From a413c6ebb86b2da7e9c34fe721d8a6575c7ec41c Mon Sep 17 00:00:00 2001 From: wiz Date: Sat, 30 Jul 2022 15:25:02 +0200 Subject: [PATCH 04/53] Separate electrs into bitcoin electrs and elements electrs --- production/install | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/production/install b/production/install index 053ffef70..6022a9a5e 100755 --- a/production/install +++ b/production/install @@ -51,12 +51,11 @@ BISQ_MAINNET_ENABLE=ON ELEMENTS_LIQUID_ENABLE=ON ELEMENTS_LIQUIDTESTNET_ENABLE=ON -# install Electrs -ELECTRS_INSTALL=ON - # enable lightmode and disable compaction to fit on 1TB SSD drive +BITCOIN_ELECTRS_INSTALL=ON BITCOIN_ELECTRS_LIGHT_MODE=ON BITCOIN_ELECTRS_COMPACTION=OFF +ELEMENTS_ELECTRS_INSTALL=ON ELEMENTS_ELECTRS_LIGHT_MODE=ON ELEMENTS_ELECTRS_COMPACTION=OFF @@ -809,10 +808,16 @@ else ELEMENTS_INSTALL=OFF fi -if [ "${BITCOIN_INSTALL}" = ON -o "${ELEMENTS_INSTALL}" = ON ];then - ELECTRS_INSTALL=ON +if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then + BITCOIN_ELECTRS_INSTALL=ON else - ELECTRS_INSTALL=OFF + BITCOIN_ELECTRS_INSTALL=OFF +fi + +if [ "${ELEMENTS_LIQUID_ENABLE}" = ON -o "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then + ELEMENTS_ELECTRS_INSTALL=ON +else + ELEMENTS_ELECTRS_INSTALL=OFF fi if grep Bisq $tempfile >/dev/null 2>&1;then @@ -1110,7 +1115,7 @@ fi # Bitcoin -> Electrs installation # ################################### -if [ "${ELECTRS_INSTALL}" = ON ];then +if [ "${BITCOIN_ELECTRS_INSTALL}" = ON ];then echo "[*] Creating Bitcoin Electrs data folder" osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}" @@ -1164,14 +1169,18 @@ fi # Liquid -> Electrs installation # ################################## -if [ "${ELEMENTS_INSTALL}" = ON ;then +if [ "${ELEMENTS_ELECTRS_INSTALL}" = ON ];then echo "[*] Creating Liquid Electrs data folder" osSudo "${ROOT_USER}" mkdir -p "${ELEMENTS_ELECTRS_HOME}" osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}" osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_ELECTRS_HOME}" - osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUID_DATA}" - osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUIDTESTNET_DATA}" + if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then + osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUID_DATA}" + fi + if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then + osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUIDTESTNET_DATA}" + fi echo "[*] Cloning Liquid Electrs repo from ${ELEMENTS_ELECTRS_REPO_URL}" osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false From 8f183945c0e7713b24fb4594716c5d60c32fdf2d Mon Sep 17 00:00:00 2001 From: wiz Date: Sat, 30 Jul 2022 15:30:55 +0200 Subject: [PATCH 05/53] Fix FreeBSD path for torrc --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index 44488add4..8612e0f98 100755 --- a/production/install +++ b/production/install @@ -183,7 +183,7 @@ case $OS in ROOT_GROUP=wheel ROOT_HOME=/root TOR_HOME=/var/db/tor - TOR_CONFIGURATION=/var/db/tor/torrc + TOR_CONFIGURATION=/usr/local/etc/tor/torrc TOR_RESOURCES=/var/db/tor TOR_PKG=tor TOR_USER=_tor From 8d1624476f44bbb4b55bbb59df46821291565ad2 Mon Sep 17 00:00:00 2001 From: wiz Date: Sat, 30 Jul 2022 15:32:51 +0200 Subject: [PATCH 06/53] Remove TOR_HOME variable in prod/install --- production/install | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/production/install b/production/install index 8612e0f98..94c9936bf 100755 --- a/production/install +++ b/production/install @@ -182,7 +182,6 @@ case $OS in ROOT_USER=root ROOT_GROUP=wheel ROOT_HOME=/root - TOR_HOME=/var/db/tor TOR_CONFIGURATION=/usr/local/etc/tor/torrc TOR_RESOURCES=/var/db/tor TOR_PKG=tor @@ -198,7 +197,6 @@ case $OS in ROOT_USER=root ROOT_GROUP=root ROOT_HOME=/root - TOR_HOME=/etc/tor TOR_CONFIGURATION=/etc/tor/torrc TOR_RESOURCES=/var/lib/tor TOR_PKG=tor @@ -985,7 +983,7 @@ if [ "${TOR_INSTALL}" = ON ];then osPackageInstall "${TOR_PKG}" echo "[*] Installing Tor base configuration" - osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/torrc" "${TOR_HOME}/torrc" + osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/torrc" "${TOR_CONFIGURATION}" osSudo "${ROOT_USER}" sed -i.orig "s!__TOR_RESOURCES__!${TOR_RESOURCES}!" "${TOR_CONFIGURATION}" echo "[*] Adding Tor HS configuration for Mempool" From 887fb13f347e5b319b6035665cfd50781f90aed7 Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Fri, 29 Jul 2022 20:53:19 +0200 Subject: [PATCH 07/53] use lnd rest api --- backend/mempool-config.sample.json | 4 +- backend/package-lock.json | 892 +----------------- backend/package.json | 5 +- .../api/lightning/lightning-api.interface.ts | 104 +- backend/src/api/lightning/lnd/lnd-api.ts | 46 +- backend/src/config.ts | 4 +- .../src/tasks/lightning/node-sync.service.ts | 115 ++- .../tasks/lightning/stats-updater.service.ts | 25 +- 8 files changed, 197 insertions(+), 998 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index b544a3f9b..312d9d18d 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -79,8 +79,8 @@ }, "LND": { "TLS_CERT_PATH": "tls.cert", - "MACAROON_PATH": "admin.macaroon", - "SOCKET": "localhost:10009" + "MACAROON_PATH": "readonly.macaroon", + "REST_API_URL": "https://localhost:8080" }, "SOCKS5PROXY": { "ENABLED": false, diff --git a/backend/package-lock.json b/backend/package-lock.json index e724ac35b..b23a7f874 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -13,10 +13,8 @@ "@types/node": "^16.11.41", "axios": "~0.27.2", "bitcoinjs-lib": "6.0.1", - "bolt07": "^1.8.1", "crypto-js": "^4.0.0", "express": "^4.18.0", - "lightning": "^5.16.3", "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", @@ -97,36 +95,6 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, - "node_modules/@grpc/grpc-js": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.7.tgz", - "integrity": "sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw==", - "dependencies": { - "@grpc/proto-loader": "^0.6.4", - "@types/node": ">=12.12.47" - }, - "engines": { - "node": "^8.13.0 || >=10.10.0" - } - }, - "node_modules/@grpc/proto-loader": { - "version": "0.6.13", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.6.13.tgz", - "integrity": "sha512-FjxPYDRTn6Ec3V0arm1FtSpmP6V50wuph2yILpyvTKzjc76oDdoihXqM1DzOW5ubvCC8GivfCnNtfaRE8myJ7g==", - "dependencies": { - "@types/long": "^4.0.1", - "lodash.camelcase": "^4.3.0", - "long": "^4.0.0", - "protobufjs": "^6.11.3", - "yargs": "^16.2.0" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -213,74 +181,16 @@ "node": ">= 8" } }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" - }, "node_modules/@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", "integrity": "sha512-W98JrE0j2K78swW4ukqMleo8R7h/pFETjM2DQ90MF6XK2i4LO4W3gQ71Lt4w3bfm2EvVSyWHplECvB5sK22yFQ==", + "dev": true, "dependencies": { "@types/connect": "*", "@types/node": "*" } }, - "node_modules/@types/caseless": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", - "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" - }, "node_modules/@types/compression": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.2.tgz", @@ -294,6 +204,7 @@ "version": "3.4.34", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.34.tgz", "integrity": "sha512-ePPA/JuI+X0vb+gSWlPKOY0NdNAie/rPUqX2GUPpbZwiKTkSPhjXWuee47E4MtE54QVzGCQMQkAL6JhV2E1+cQ==", + "dev": true, "dependencies": { "@types/node": "*" } @@ -308,6 +219,7 @@ "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "dev": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.18", @@ -319,6 +231,7 @@ "version": "4.17.28", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", + "dev": true, "dependencies": { "@types/node": "*", "@types/qs": "*", @@ -331,15 +244,11 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, - "node_modules/@types/long": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", - "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" - }, "node_modules/@types/mime": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", - "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==" + "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==", + "dev": true }, "node_modules/@types/node": { "version": "16.11.41", @@ -349,55 +258,30 @@ "node_modules/@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", + "dev": true }, "node_modules/@types/range-parser": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "node_modules/@types/request": { - "version": "2.48.8", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.8.tgz", - "integrity": "sha512-whjk1EDJPcAR2kYHRbFl/lKeeKYTi05A15K9bnLInCVroNDCtXce57xKdI0/rQaA3K+6q0eFyUBPmqfSndUZdQ==", - "dependencies": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - } - }, - "node_modules/@types/request/node_modules/form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", + "dev": true }, "node_modules/@types/serve-static": { "version": "1.13.8", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.8.tgz", "integrity": "sha512-MoJhSQreaVoL+/hurAZzIm8wafFR6ajiTM1m4A0kv6AGeVBl4r4pOV8bGFrjjq1sGxDTnCoF8i22o0/aE5XCyA==", + "dev": true, "dependencies": { "@types/mime": "*", "@types/node": "*" } }, - "node_modules/@types/tough-cookie": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", - "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==" - }, "node_modules/@types/ws": { "version": "8.5.3", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", + "dev": true, "dependencies": { "@types/node": "*" } @@ -838,6 +722,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, "engines": { "node": ">=8" } @@ -846,6 +731,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -870,24 +756,6 @@ "node": ">=8" } }, - "node_modules/async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" - }, - "node_modules/asyncjs-util": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/asyncjs-util/-/asyncjs-util-1.2.9.tgz", - "integrity": "sha512-U9imS8ehJA6DPNdBdvoLcIRDFh7yzI9J93CC8/2obk8gUSIy8KKhmCqYe+3NlISJhxLLi8aWmVL1Gkb3dz1xhg==", - "dependencies": { - "async": "3.2.3" - } - }, - "node_modules/asyncjs-util/node_modules/async": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", - "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -929,19 +797,6 @@ "node": ">=8.0.0" } }, - "node_modules/bip66": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/bip66/-/bip66-1.1.5.tgz", - "integrity": "sha512-nemMHz95EmS38a26XbbdxIYj5csHd3RMP3H5bwQknX0WYHF01qhpufP42mLOwVICuH2JmhIhXiWs89MfUGL7Xw==", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/bitcoin-ops": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/bitcoin-ops/-/bitcoin-ops-1.4.1.tgz", - "integrity": "sha512-pef6gxZFztEhaE9RY9HmWVmiIHqCb2OyS4HPKkpc6CIiiOa3Qmuoylxc5P2EkU3w+5eTSifI9SEZC88idAIGow==" - }, "node_modules/bitcoinjs-lib": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-6.0.1.tgz", @@ -959,11 +814,6 @@ "node": ">=8.0.0" } }, - "node_modules/bn.js": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", - "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" - }, "node_modules/body-parser": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", @@ -987,22 +837,6 @@ "npm": "1.2.8000 || >= 1.4.16" } }, - "node_modules/bolt07": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/bolt07/-/bolt07-1.8.2.tgz", - "integrity": "sha512-jq1b/ZdMambhh+yi+pm+1PJBAnlYvQYljaBgSajvVAINHrHg32ovCBra8d0ADE3BAoj6G/tK7OSV4t/yT9A+/g==", - "dependencies": { - "bn.js": "5.2.1" - } - }, - "node_modules/bolt09": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/bolt09/-/bolt09-0.2.3.tgz", - "integrity": "sha512-xEt5GE6pXB8wMIWHAoyF28k0Yt2rFqIou1LCyIeNadAOQhu/F7GTjZwreFwLl07YYkhOH23avewRt5PD8JnKKg==", - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1072,17 +906,6 @@ "node": ">=6" } }, - "node_modules/cbor": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/cbor/-/cbor-8.1.0.tgz", - "integrity": "sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==", - "dependencies": { - "nofilter": "^3.1.0" - }, - "engines": { - "node": ">=12.19" - } - }, "node_modules/cipher-base": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", @@ -1092,20 +915,11 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -1116,7 +930,8 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "node_modules/combined-stream": { "version": "1.0.8", @@ -1269,29 +1084,11 @@ "node": ">=6.0.0" } }, - "node_modules/ecpair": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/ecpair/-/ecpair-2.0.1.tgz", - "integrity": "sha512-iT3wztQMeE/nDTlfnAg8dAFUfBS7Tq2BXzq3ae6L+pWgFU0fQ3l0woTzdTBrJV3OxBjxbzjq8EQhAbEmJNWFSw==", - "dependencies": { - "randombytes": "^2.1.0", - "typeforce": "^1.18.0", - "wif": "^2.0.6" - }, - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1300,14 +1097,6 @@ "node": ">= 0.8" } }, - "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "engines": { - "node": ">=6" - } - }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -1849,14 +1638,6 @@ "is-property": "^1.0.2" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, "node_modules/get-intrinsic": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", @@ -2047,22 +1828,6 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, - "node_modules/invoices": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/invoices/-/invoices-2.0.7.tgz", - "integrity": "sha512-2kpjok/83zOTnb4tbV+RbJz7LuGVzj/GZ+jwsC7FxMqwLAf4Sf6OESNM3uuamX9oeFRo44Vip3wn1aX+9D2m8w==", - "dependencies": { - "bech32": "2.0.0", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "bolt07": "1.8.2", - "bolt09": "0.2.3", - "tiny-secp256k1": "2.2.1" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", @@ -2085,14 +1850,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -2150,57 +1907,6 @@ "node": ">= 0.8.0" } }, - "node_modules/lightning": { - "version": "5.16.3", - "resolved": "https://registry.npmjs.org/lightning/-/lightning-5.16.3.tgz", - "integrity": "sha512-ghban3KbqkbzahwIp4NAtuhc8xIurVcCXAd7tV6qGkFYKZAy9loIvFrhZqoWF4A4jnaKbRnJPCaxzJ8JwPl3EA==", - "dependencies": { - "@grpc/grpc-js": "1.6.7", - "@grpc/proto-loader": "0.6.13", - "@types/express": "4.17.13", - "@types/node": "17.0.41", - "@types/request": "2.48.8", - "@types/ws": "8.5.3", - "async": "3.2.4", - "asyncjs-util": "1.2.9", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "body-parser": "1.20.0", - "bolt07": "1.8.2", - "bolt09": "0.2.3", - "cbor": "8.1.0", - "ecpair": "2.0.1", - "express": "4.18.1", - "invoices": "2.0.7", - "psbt": "2.6.0", - "tiny-secp256k1": "2.2.1", - "type-fest": "2.13.0" - }, - "engines": { - "node": ">=12.20" - } - }, - "node_modules/lightning/node_modules/@types/node": { - "version": "17.0.41", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.41.tgz", - "integrity": "sha512-xA6drNNeqb5YyV5fO3OAEsnXLfO7uF0whiOfPTz5AeDo8KeZFmODKnvwPymMNO8qE/an8pVY/O50tig2SQCrGw==" - }, - "node_modules/lightning/node_modules/type-fest": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.13.0.tgz", - "integrity": "sha512-lPfAm42MxE4/456+QyIaaVBAwgpJb6xZ8PRu09utnhPdWwcyj9vgy6Sq0Z5yNbJ21EdxB5dRU/Qg8bsyAMtlcw==", - "engines": { - "node": ">=12.20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" - }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -2418,14 +2124,6 @@ "resolved": "https://registry.npmjs.org/node-worker-threads-pool/-/node-worker-threads-pool-1.5.1.tgz", "integrity": "sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw==" }, - "node_modules/nofilter": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", - "integrity": "sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==", - "engines": { - "node": ">=12.19" - } - }, "node_modules/object-inspect": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", @@ -2559,31 +2257,6 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, - "node_modules/protobufjs": { - "version": "6.11.3", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz", - "integrity": "sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==", - "hasInstallScript": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": ">=13.7.0", - "long": "^4.0.0" - }, - "bin": { - "pbjs": "bin/pbjs", - "pbts": "bin/pbts" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2596,22 +2269,6 @@ "node": ">= 0.10" } }, - "node_modules/psbt": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/psbt/-/psbt-2.6.0.tgz", - "integrity": "sha512-z2ca00AMwZ6PfVETQNvXRumZdRwGuQzApIH/hKNp2o6Qo8N8TW7Ug2V+aSH2w/eC1b/bOOMZIE57V3jYN+kB4A==", - "dependencies": { - "bip66": "1.1.5", - "bitcoin-ops": "1.4.1", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "pushdata-bitcoin": "1.0.1", - "varuint-bitcoin": "1.1.2" - }, - "engines": { - "node": ">=12.20" - } - }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -2626,14 +2283,6 @@ "node": ">=6" } }, - "node_modules/pushdata-bitcoin": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pushdata-bitcoin/-/pushdata-bitcoin-1.0.1.tgz", - "integrity": "sha512-hw7rcYTJRAl4olM8Owe8x0fBuJJ+WGbMhQuLWOXEMN3PxPCKQHRkhfL+XG0+iXUmSHjkMmb3Ba55Mt21cZc9kQ==", - "dependencies": { - "bitcoin-ops": "^1.3.0" - } - }, "node_modules/qs": { "version": "6.10.3", "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", @@ -2668,14 +2317,6 @@ } ] }, - "node_modules/randombytes": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -2723,14 +2364,6 @@ "url": "https://github.com/sponsors/mysticatea" } }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -3008,23 +2641,11 @@ "safe-buffer": "~5.2.0" } }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -3058,17 +2679,6 @@ "node": ">=6" } }, - "node_modules/tiny-secp256k1": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz", - "integrity": "sha512-/U4xfVqnVxJXN4YVsru0E6t5wVncu2uunB8+RVR40fYUxkKYUPS10f+ePQZgFBoE/Jbf9H1NBveupF2VmB58Ng==", - "dependencies": { - "uint8array-tools": "0.0.7" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -3148,14 +2758,6 @@ "node": ">=4.2.0" } }, - "node_modules/uint8array-tools": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/uint8array-tools/-/uint8array-tools-0.0.7.tgz", - "integrity": "sha512-vrrNZJiusLWoFWBqz5Y5KMCgP9W9hnjZHzZiZRT8oNAkq3d5Z5Oe76jAvVVSRh4U8GGR90N2X1dWtrhvx6L8UQ==", - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -3240,22 +2842,6 @@ "node": ">=0.10.0" } }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -3282,43 +2868,10 @@ } } }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "engines": { - "node": ">=10" - } - }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "node_modules/yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", - "engines": { - "node": ">=10" - } } }, "dependencies": { @@ -3371,27 +2924,6 @@ } } }, - "@grpc/grpc-js": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.7.tgz", - "integrity": "sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw==", - "requires": { - "@grpc/proto-loader": "^0.6.4", - "@types/node": ">=12.12.47" - } - }, - "@grpc/proto-loader": { - "version": "0.6.13", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.6.13.tgz", - "integrity": "sha512-FjxPYDRTn6Ec3V0arm1FtSpmP6V50wuph2yILpyvTKzjc76oDdoihXqM1DzOW5ubvCC8GivfCnNtfaRE8myJ7g==", - "requires": { - "@types/long": "^4.0.1", - "lodash.camelcase": "^4.3.0", - "long": "^4.0.0", - "protobufjs": "^6.11.3", - "yargs": "^16.2.0" - } - }, "@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -3457,74 +2989,16 @@ "fastq": "^1.6.0" } }, - "@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" - }, - "@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" - }, - "@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "requires": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" - }, - "@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" - }, - "@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" - }, - "@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" - }, - "@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" - }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", "integrity": "sha512-W98JrE0j2K78swW4ukqMleo8R7h/pFETjM2DQ90MF6XK2i4LO4W3gQ71Lt4w3bfm2EvVSyWHplECvB5sK22yFQ==", + "dev": true, "requires": { "@types/connect": "*", "@types/node": "*" } }, - "@types/caseless": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", - "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" - }, "@types/compression": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.2.tgz", @@ -3538,6 +3012,7 @@ "version": "3.4.34", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.34.tgz", "integrity": "sha512-ePPA/JuI+X0vb+gSWlPKOY0NdNAie/rPUqX2GUPpbZwiKTkSPhjXWuee47E4MtE54QVzGCQMQkAL6JhV2E1+cQ==", + "dev": true, "requires": { "@types/node": "*" } @@ -3552,6 +3027,7 @@ "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "dev": true, "requires": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.18", @@ -3563,6 +3039,7 @@ "version": "4.17.28", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", + "dev": true, "requires": { "@types/node": "*", "@types/qs": "*", @@ -3575,15 +3052,11 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, - "@types/long": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", - "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" - }, "@types/mime": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", - "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==" + "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==", + "dev": true }, "@types/node": { "version": "16.11.41", @@ -3593,54 +3066,30 @@ "@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", + "dev": true }, "@types/range-parser": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "@types/request": { - "version": "2.48.8", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.8.tgz", - "integrity": "sha512-whjk1EDJPcAR2kYHRbFl/lKeeKYTi05A15K9bnLInCVroNDCtXce57xKdI0/rQaA3K+6q0eFyUBPmqfSndUZdQ==", - "requires": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - }, - "dependencies": { - "form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - } - } + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", + "dev": true }, "@types/serve-static": { "version": "1.13.8", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.8.tgz", "integrity": "sha512-MoJhSQreaVoL+/hurAZzIm8wafFR6ajiTM1m4A0kv6AGeVBl4r4pOV8bGFrjjq1sGxDTnCoF8i22o0/aE5XCyA==", + "dev": true, "requires": { "@types/mime": "*", "@types/node": "*" } }, - "@types/tough-cookie": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", - "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==" - }, "@types/ws": { "version": "8.5.3", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", + "dev": true, "requires": { "@types/node": "*" } @@ -3913,12 +3362,14 @@ "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -3934,26 +3385,6 @@ "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true }, - "async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" - }, - "asyncjs-util": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/asyncjs-util/-/asyncjs-util-1.2.9.tgz", - "integrity": "sha512-U9imS8ehJA6DPNdBdvoLcIRDFh7yzI9J93CC8/2obk8gUSIy8KKhmCqYe+3NlISJhxLLi8aWmVL1Gkb3dz1xhg==", - "requires": { - "async": "3.2.3" - }, - "dependencies": { - "async": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", - "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==" - } - } - }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -3992,19 +3423,6 @@ "resolved": "https://registry.npmjs.org/bip174/-/bip174-2.0.1.tgz", "integrity": "sha512-i3X26uKJOkDTAalYAp0Er+qGMDhrbbh2o93/xiPyAN2s25KrClSpe3VXo/7mNJoqA5qfko8rLS2l3RWZgYmjKQ==" }, - "bip66": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/bip66/-/bip66-1.1.5.tgz", - "integrity": "sha512-nemMHz95EmS38a26XbbdxIYj5csHd3RMP3H5bwQknX0WYHF01qhpufP42mLOwVICuH2JmhIhXiWs89MfUGL7Xw==", - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "bitcoin-ops": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/bitcoin-ops/-/bitcoin-ops-1.4.1.tgz", - "integrity": "sha512-pef6gxZFztEhaE9RY9HmWVmiIHqCb2OyS4HPKkpc6CIiiOa3Qmuoylxc5P2EkU3w+5eTSifI9SEZC88idAIGow==" - }, "bitcoinjs-lib": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-6.0.1.tgz", @@ -4019,11 +3437,6 @@ "wif": "^2.0.1" } }, - "bn.js": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", - "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" - }, "body-parser": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", @@ -4043,19 +3456,6 @@ "unpipe": "1.0.0" } }, - "bolt07": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/bolt07/-/bolt07-1.8.2.tgz", - "integrity": "sha512-jq1b/ZdMambhh+yi+pm+1PJBAnlYvQYljaBgSajvVAINHrHg32ovCBra8d0ADE3BAoj6G/tK7OSV4t/yT9A+/g==", - "requires": { - "bn.js": "5.2.1" - } - }, - "bolt09": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/bolt09/-/bolt09-0.2.3.tgz", - "integrity": "sha512-xEt5GE6pXB8wMIWHAoyF28k0Yt2rFqIou1LCyIeNadAOQhu/F7GTjZwreFwLl07YYkhOH23avewRt5PD8JnKKg==" - }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -4113,14 +3513,6 @@ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true }, - "cbor": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/cbor/-/cbor-8.1.0.tgz", - "integrity": "sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==", - "requires": { - "nofilter": "^3.1.0" - } - }, "cipher-base": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", @@ -4130,20 +3522,11 @@ "safe-buffer": "^5.0.1" } }, - "cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "requires": { "color-name": "~1.1.4" } @@ -4151,7 +3534,8 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "combined-stream": { "version": "1.0.8", @@ -4270,36 +3654,16 @@ "esutils": "^2.0.2" } }, - "ecpair": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/ecpair/-/ecpair-2.0.1.tgz", - "integrity": "sha512-iT3wztQMeE/nDTlfnAg8dAFUfBS7Tq2BXzq3ae6L+pWgFU0fQ3l0woTzdTBrJV3OxBjxbzjq8EQhAbEmJNWFSw==", - "requires": { - "randombytes": "^2.1.0", - "typeforce": "^1.18.0", - "wif": "^2.0.6" - } - }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, - "escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" - }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -4716,11 +4080,6 @@ "is-property": "^1.0.2" } }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" - }, "get-intrinsic": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", @@ -4857,19 +4216,6 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, - "invoices": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/invoices/-/invoices-2.0.7.tgz", - "integrity": "sha512-2kpjok/83zOTnb4tbV+RbJz7LuGVzj/GZ+jwsC7FxMqwLAf4Sf6OESNM3uuamX9oeFRo44Vip3wn1aX+9D2m8w==", - "requires": { - "bech32": "2.0.0", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "bolt07": "1.8.2", - "bolt09": "0.2.3", - "tiny-secp256k1": "2.2.1" - } - }, "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", @@ -4886,11 +4232,6 @@ "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, "is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -4939,50 +4280,6 @@ "type-check": "~0.4.0" } }, - "lightning": { - "version": "5.16.3", - "resolved": "https://registry.npmjs.org/lightning/-/lightning-5.16.3.tgz", - "integrity": "sha512-ghban3KbqkbzahwIp4NAtuhc8xIurVcCXAd7tV6qGkFYKZAy9loIvFrhZqoWF4A4jnaKbRnJPCaxzJ8JwPl3EA==", - "requires": { - "@grpc/grpc-js": "1.6.7", - "@grpc/proto-loader": "0.6.13", - "@types/express": "4.17.13", - "@types/node": "17.0.41", - "@types/request": "2.48.8", - "@types/ws": "8.5.3", - "async": "3.2.4", - "asyncjs-util": "1.2.9", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "body-parser": "1.20.0", - "bolt07": "1.8.2", - "bolt09": "0.2.3", - "cbor": "8.1.0", - "ecpair": "2.0.1", - "express": "4.18.1", - "invoices": "2.0.7", - "psbt": "2.6.0", - "tiny-secp256k1": "2.2.1", - "type-fest": "2.13.0" - }, - "dependencies": { - "@types/node": { - "version": "17.0.41", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.41.tgz", - "integrity": "sha512-xA6drNNeqb5YyV5fO3OAEsnXLfO7uF0whiOfPTz5AeDo8KeZFmODKnvwPymMNO8qE/an8pVY/O50tig2SQCrGw==" - }, - "type-fest": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.13.0.tgz", - "integrity": "sha512-lPfAm42MxE4/456+QyIaaVBAwgpJb6xZ8PRu09utnhPdWwcyj9vgy6Sq0Z5yNbJ21EdxB5dRU/Qg8bsyAMtlcw==" - } - } - }, - "lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" - }, "lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -5154,11 +4451,6 @@ "resolved": "https://registry.npmjs.org/node-worker-threads-pool/-/node-worker-threads-pool-1.5.1.tgz", "integrity": "sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw==" }, - "nofilter": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", - "integrity": "sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==" - }, "object-inspect": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", @@ -5250,26 +4542,6 @@ "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", "dev": true }, - "protobufjs": { - "version": "6.11.3", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz", - "integrity": "sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==", - "requires": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": ">=13.7.0", - "long": "^4.0.0" - } - }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -5279,19 +4551,6 @@ "ipaddr.js": "1.9.1" } }, - "psbt": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/psbt/-/psbt-2.6.0.tgz", - "integrity": "sha512-z2ca00AMwZ6PfVETQNvXRumZdRwGuQzApIH/hKNp2o6Qo8N8TW7Ug2V+aSH2w/eC1b/bOOMZIE57V3jYN+kB4A==", - "requires": { - "bip66": "1.1.5", - "bitcoin-ops": "1.4.1", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "pushdata-bitcoin": "1.0.1", - "varuint-bitcoin": "1.1.2" - } - }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -5303,14 +4562,6 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, - "pushdata-bitcoin": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pushdata-bitcoin/-/pushdata-bitcoin-1.0.1.tgz", - "integrity": "sha512-hw7rcYTJRAl4olM8Owe8x0fBuJJ+WGbMhQuLWOXEMN3PxPCKQHRkhfL+XG0+iXUmSHjkMmb3Ba55Mt21cZc9kQ==", - "requires": { - "bitcoin-ops": "^1.3.0" - } - }, "qs": { "version": "6.10.3", "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", @@ -5325,14 +4576,6 @@ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true }, - "randombytes": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "requires": { - "safe-buffer": "^5.1.0" - } - }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -5365,11 +4608,6 @@ "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", "dev": true }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" - }, "resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -5564,20 +4802,11 @@ "safe-buffer": "~5.2.0" } }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "requires": { "ansi-regex": "^5.0.1" } @@ -5599,14 +4828,6 @@ "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==" }, - "tiny-secp256k1": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz", - "integrity": "sha512-/U4xfVqnVxJXN4YVsru0E6t5wVncu2uunB8+RVR40fYUxkKYUPS10f+ePQZgFBoE/Jbf9H1NBveupF2VmB58Ng==", - "requires": { - "uint8array-tools": "0.0.7" - } - }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -5661,11 +4882,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==" }, - "uint8array-tools": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/uint8array-tools/-/uint8array-tools-0.0.7.tgz", - "integrity": "sha512-vrrNZJiusLWoFWBqz5Y5KMCgP9W9hnjZHzZiZRT8oNAkq3d5Z5Oe76jAvVVSRh4U8GGR90N2X1dWtrhvx6L8UQ==" - }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -5732,16 +4948,6 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", "dev": true }, - "wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -5754,34 +4960,10 @@ "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==", "requires": {} }, - "y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" - }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "requires": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - } - }, - "yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==" } } } diff --git a/backend/package.json b/backend/package.json index b8930d6e5..6345e89da 100644 --- a/backend/package.json +++ b/backend/package.json @@ -16,7 +16,8 @@ "mempool", "blockchain", "explorer", - "liquid" + "liquid", + "lightning" ], "main": "index.ts", "scripts": { @@ -34,10 +35,8 @@ "@types/node": "^16.11.41", "axios": "~0.27.2", "bitcoinjs-lib": "6.0.1", - "bolt07": "^1.8.1", "crypto-js": "^4.0.0", "express": "^4.18.0", - "lightning": "^5.16.3", "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", diff --git a/backend/src/api/lightning/lightning-api.interface.ts b/backend/src/api/lightning/lightning-api.interface.ts index 9b83b5473..283f34a5a 100644 --- a/backend/src/api/lightning/lightning-api.interface.ts +++ b/backend/src/api/lightning/lightning-api.interface.ts @@ -1,71 +1,85 @@ export namespace ILightningApi { export interface NetworkInfo { - average_channel_size: number; - channel_count: number; - max_channel_size: number; - median_channel_size: number; - min_channel_size: number; - node_count: number; - not_recently_updated_policy_count: number; - total_capacity: number; + graph_diameter: number; + avg_out_degree: number; + max_out_degree: number; + num_nodes: number; + num_channels: number; + total_network_capacity: string; + avg_channel_size: number; + min_channel_size: string; + max_channel_size: string; + median_channel_size_sat: string; + num_zombie_chans: string; } export interface NetworkGraph { - channels: Channel[]; nodes: Node[]; + edges: Channel[]; } export interface Channel { - id: string; - capacity: number; - policies: Policy[]; - transaction_id: string; - transaction_vout: number; - updated_at?: string; + channel_id: string; + chan_point: string; + last_update: number; + node1_pub: string; + node2_pub: string; + capacity: string; + node1_policy: RoutingPolicy | null; + node2_policy: RoutingPolicy | null; } - interface Policy { - public_key: string; - base_fee_mtokens?: string; - cltv_delta?: number; - fee_rate?: number; - is_disabled?: boolean; - max_htlc_mtokens?: string; - min_htlc_mtokens?: string; - updated_at?: string; + export interface RoutingPolicy { + time_lock_delta: number; + min_htlc: string; + fee_base_msat: string; + fee_rate_milli_msat: string; + disabled: boolean; + max_htlc_msat: string; + last_update: number; } export interface Node { + last_update: number; + pub_key: string; alias: string; + addresses: { + network: string; + addr: string; + }[]; color: string; - features: Feature[]; - public_key: string; - sockets: string[]; - updated_at?: string; + features: { [key: number]: Feature }; } export interface Info { - chains: string[]; - color: string; - active_channels_count: number; + identity_pubkey: string; alias: string; - current_block_hash: string; - current_block_height: number; - features: Feature[]; - is_synced_to_chain: boolean; - is_synced_to_graph: boolean; - latest_block_at: string; - peers_count: number; - pending_channels_count: number; - public_key: string; - uris: any[]; + num_pending_channels: number; + num_active_channels: number; + num_peers: number; + block_height: number; + block_hash: string; + synced_to_chain: boolean; + testnet: boolean; + uris: string[]; + best_header_timestamp: string; version: string; + num_inactive_channels: number; + chains: { + chain: string; + network: string; + }[]; + color: string; + synced_to_graph: boolean; + features: { [key: number]: Feature }; + commit_hash: string; + /** Available on LND since v0.15.0-beta */ + require_htlc_interceptor?: boolean; } - + export interface Feature { - bit: number; - is_known: boolean; + name: string; is_required: boolean; - type?: string; + is_known: boolean; } } diff --git a/backend/src/api/lightning/lnd/lnd-api.ts b/backend/src/api/lightning/lnd/lnd-api.ts index 19d98744d..1480f9b8f 100644 --- a/backend/src/api/lightning/lnd/lnd-api.ts +++ b/backend/src/api/lightning/lnd/lnd-api.ts @@ -1,44 +1,40 @@ +import axios, { AxiosRequestConfig } from 'axios'; +import { Agent } from 'https'; +import * as fs from 'fs'; import { AbstractLightningApi } from '../lightning-api-abstract-factory'; import { ILightningApi } from '../lightning-api.interface'; -import * as fs from 'fs'; -import { authenticatedLndGrpc, getWalletInfo, getNetworkGraph, getNetworkInfo } from 'lightning'; import config from '../../../config'; -import logger from '../../../logger'; class LndApi implements AbstractLightningApi { - private lnd: any; + axiosConfig: AxiosRequestConfig = {}; + constructor() { - if (!config.LIGHTNING.ENABLED) { - return; - } - try { - const tls = fs.readFileSync(config.LND.TLS_CERT_PATH).toString('base64'); - const macaroon = fs.readFileSync(config.LND.MACAROON_PATH).toString('base64'); - - const { lnd } = authenticatedLndGrpc({ - cert: tls, - macaroon: macaroon, - socket: config.LND.SOCKET, - }); - - this.lnd = lnd; - } catch (e) { - logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e)); - process.exit(1); + if (config.LIGHTNING.ENABLED) { + this.axiosConfig = { + headers: { + 'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex') + }, + httpsAgent: new Agent({ + ca: fs.readFileSync(config.LND.TLS_CERT_PATH) + }), + timeout: 10000 + }; } } async $getNetworkInfo(): Promise { - return await getNetworkInfo({ lnd: this.lnd }); + return axios.get(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig) + .then((response) => response.data); } async $getInfo(): Promise { - // @ts-ignore - return await getWalletInfo({ lnd: this.lnd }); + return axios.get(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig) + .then((response) => response.data); } async $getNetworkGraph(): Promise { - return await getNetworkGraph({ lnd: this.lnd }); + return axios.get(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig) + .then((response) => response.data); } } diff --git a/backend/src/config.ts b/backend/src/config.ts index 0e3382517..5560a25a7 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -35,7 +35,7 @@ interface IConfig { LND: { TLS_CERT_PATH: string; MACAROON_PATH: string; - SOCKET: string; + REST_API_URL: string; }; ELECTRUM: { HOST: string; @@ -182,7 +182,7 @@ const defaults: IConfig = { 'LND': { 'TLS_CERT_PATH': '', 'MACAROON_PATH': '', - 'SOCKET': 'localhost:10009', + 'REST_API_URL': 'https://localhost:8080', }, 'SOCKS5PROXY': { 'ENABLED': false, diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index f45473aba..10cd2d744 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -1,4 +1,3 @@ -import { chanNumber } from 'bolt07'; import DB from '../../database'; import logger from '../../logger'; import channelsApi from '../../api/explorer/channels.api'; @@ -39,9 +38,9 @@ class NodeSyncService { } const graphChannelsIds: string[] = []; - for (const channel of networkGraph.channels) { + for (const channel of networkGraph.edges) { await this.$saveChannel(channel); - graphChannelsIds.push(channel.id); + graphChannelsIds.push(channel.channel_id); } await this.$setChannelsInactive(graphChannelsIds); @@ -56,7 +55,7 @@ class NodeSyncService { } } catch (e) { - logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e)); + logger.err('$runUpdater() error: ' + (e instanceof Error ? e.message : e)); } } @@ -107,8 +106,7 @@ class NodeSyncService { logger.info(`Running inactive channels scan...`); try { - // @ts-ignore - const [channels]: [ILightningApi.Channel[]] = await DB.query(` + const [channels]: [{ id: string }[]] = await DB.query(` SELECT channels.id FROM channels WHERE channels.status = 1 @@ -266,7 +264,10 @@ class NodeSyncService { } private async $saveChannel(channel: ILightningApi.Channel): Promise { - const fromChannel = chanNumber({ channel: channel.id }).number; + const [ txid, vout ] = channel.chan_point.split(':'); + + const policy1: Partial = channel.node1_policy || {}; + const policy2: Partial = channel.node2_policy || {}; try { const query = `INSERT INTO channels @@ -319,55 +320,55 @@ class NodeSyncService { ;`; await DB.query(query, [ - fromChannel, - channel.id, + channel.channel_id, + this.toShortId(channel.channel_id), channel.capacity, - channel.transaction_id, - channel.transaction_vout, - channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0, - channel.policies[0].public_key, - channel.policies[0].base_fee_mtokens, - channel.policies[0].cltv_delta, - channel.policies[0].fee_rate, - channel.policies[0].is_disabled, - channel.policies[0].max_htlc_mtokens, - channel.policies[0].min_htlc_mtokens, - channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0, - channel.policies[1].public_key, - channel.policies[1].base_fee_mtokens, - channel.policies[1].cltv_delta, - channel.policies[1].fee_rate, - channel.policies[1].is_disabled, - channel.policies[1].max_htlc_mtokens, - channel.policies[1].min_htlc_mtokens, - channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0, + txid, + vout, + this.utcDateToMysql(channel.last_update), + channel.node1_pub, + policy1.fee_base_msat, + policy1.time_lock_delta, + policy1.fee_rate_milli_msat, + policy1.disabled, + policy1.max_htlc_msat, + policy1.min_htlc, + this.utcDateToMysql(policy1.last_update), + channel.node2_pub, + policy2.fee_base_msat, + policy2.time_lock_delta, + policy2.fee_rate_milli_msat, + policy2.disabled, + policy2.max_htlc_msat, + policy2.min_htlc, + this.utcDateToMysql(policy2.last_update), channel.capacity, - channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0, - channel.policies[0].public_key, - channel.policies[0].base_fee_mtokens, - channel.policies[0].cltv_delta, - channel.policies[0].fee_rate, - channel.policies[0].is_disabled, - channel.policies[0].max_htlc_mtokens, - channel.policies[0].min_htlc_mtokens, - channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0, - channel.policies[1].public_key, - channel.policies[1].base_fee_mtokens, - channel.policies[1].cltv_delta, - channel.policies[1].fee_rate, - channel.policies[1].is_disabled, - channel.policies[1].max_htlc_mtokens, - channel.policies[1].min_htlc_mtokens, - channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0, + this.utcDateToMysql(channel.last_update), + channel.node1_pub, + policy1.fee_base_msat, + policy1.time_lock_delta, + policy1.fee_rate_milli_msat, + policy1.disabled, + policy1.max_htlc_msat, + policy1.min_htlc, + this.utcDateToMysql(policy1.last_update), + channel.node2_pub, + policy2.fee_base_msat, + policy2.time_lock_delta, + policy2.fee_rate_milli_msat, + policy2.disabled, + policy2.max_htlc_msat, + policy2.min_htlc, + this.utcDateToMysql(policy2.last_update) ]); } catch (e) { logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e)); } } - private async $updateChannelStatus(channelShortId: string, status: number): Promise { + private async $updateChannelStatus(channelId: string, status: number): Promise { try { - await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]); + await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelId]); } catch (e) { logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e)); } @@ -390,8 +391,8 @@ class NodeSyncService { private async $saveNode(node: ILightningApi.Node): Promise { try { - const updatedAt = node.updated_at ? this.utcDateToMysql(node.updated_at) : '0000-00-00 00:00:00'; - const sockets = node.sockets.join(','); + const updatedAt = this.utcDateToMysql(node.last_update); + const sockets = node.addresses.map(a => a.addr).join(','); const query = `INSERT INTO nodes( public_key, first_seen, @@ -403,7 +404,7 @@ class NodeSyncService { VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`; await DB.query(query, [ - node.public_key, + node.pub_key, updatedAt, node.alias, node.color, @@ -418,8 +419,18 @@ class NodeSyncService { } } - private utcDateToMysql(dateString: string): string { - const d = new Date(Date.parse(dateString)); + /** Decodes a channel id returned by lnd as uint64 to a short channel id */ + private toShortId(id: string): string { + const n = BigInt(id); + return [ + n >> 40n, // nth block + (n >> 16n) & 0xffffffn, // nth tx of the block + n & 0xffffn // nth output of the tx + ].join('x'); + } + + private utcDateToMysql(date?: number): string { + const d = new Date((date || 0) * 1000); return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; } } diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index f30da9e96..0a3ade614 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -52,7 +52,7 @@ class LightningStatsUpdater { private async $lightningIsSynced(): Promise { const nodeInfo = await lightningApi.$getInfo(); - return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph; + return nodeInfo.synced_to_chain && nodeInfo.synced_to_graph; } private async $runTasks(): Promise { @@ -66,13 +66,13 @@ class LightningStatsUpdater { private async $logLightningStatsDaily() { try { - logger.info(`Running lightning daily stats log...`); + logger.info(`Running lightning daily stats log...`); const networkGraph = await lightningApi.$getNetworkGraph(); let total_capacity = 0; - for (const channel of networkGraph.channels) { + for (const channel of networkGraph.edges) { if (channel.capacity) { - total_capacity += channel.capacity; + total_capacity += parseInt(channel.capacity); } } @@ -80,20 +80,17 @@ class LightningStatsUpdater { let torNodes = 0; let unannouncedNodes = 0; for (const node of networkGraph.nodes) { - let isUnnanounced = true; - for (const socket of node.sockets) { - const hasOnion = socket.indexOf('.onion') !== -1; + for (const socket of node.addresses) { + const hasOnion = socket.addr.indexOf('.onion') !== -1; if (hasOnion) { torNodes++; - isUnnanounced = false; } - const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0])); + const hasClearnet = [4, 6].includes(net.isIP(socket.addr.split(':')[0])); if (hasClearnet) { clearnetNodes++; - isUnnanounced = false; } } - if (isUnnanounced) { + if (node.addresses.length === 0) { unannouncedNodes++; } } @@ -118,7 +115,7 @@ class LightningStatsUpdater { VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; await DB.query(query, [ - networkGraph.channels.length, + networkGraph.edges.length, networkGraph.nodes.length, total_capacity, torNodes, @@ -292,7 +289,7 @@ class LightningStatsUpdater { for (const node of nodes) { const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); - + const date: Date = new Date(this.hardCodedStartTime); const currentDate = new Date(); this.setDateMidnight(currentDate); @@ -322,7 +319,7 @@ class LightningStatsUpdater { lastTotalCapacity = totalCapacity; lastChannelsCount = channelsCount; - + const query = `INSERT INTO node_stats( public_key, added, From 7570603b377c002b3079951f7c8c3ca282686af4 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 31 Jul 2022 23:25:28 +0200 Subject: [PATCH 08/53] Redirect with path --- frontend/src/app/services/enterprise.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/services/enterprise.service.ts b/frontend/src/app/services/enterprise.service.ts index be34576f9..b125739d6 100644 --- a/frontend/src/app/services/enterprise.service.ts +++ b/frontend/src/app/services/enterprise.service.ts @@ -49,7 +49,7 @@ export class EnterpriseService { }, (error) => { if (error.status === 404) { - window.location.href = 'https://mempool.space'; + window.location.href = 'https://mempool.space' + window.location.pathname; } }); } From f4670156a5c0a4491e11d6e2a3e1b5d32f43fb8c Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 1 Aug 2022 09:59:20 +0200 Subject: [PATCH 09/53] Fix missing pub key, capacity and channel count for node lists --- backend/src/api/explorer/nodes.api.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 55b0ba5cb..f05c4cf0a 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -163,8 +163,8 @@ class NodesApi { public async $getNodesPerCountry(countryId: string) { try { const query = ` - SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias, - UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, + SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city FROM node_stats JOIN ( @@ -172,7 +172,7 @@ class NodesApi { FROM node_stats GROUP BY public_key ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added - JOIN nodes ON nodes.public_key = node_stats.public_key + RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' WHERE geo_names_country.id = ? @@ -193,8 +193,8 @@ class NodesApi { public async $getNodesPerISP(ISPId: string) { try { const query = ` - SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias, - UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, + SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city, geo_names_country.names as country FROM node_stats JOIN ( From b61ef6814bab193b11502b6aecdb71850f0c4cea Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 28 Jul 2022 15:07:50 +0200 Subject: [PATCH 10/53] Re-enabled channels world map click event --- .../nodes-channels-map/nodes-channels-map.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index 16accda94..61f7afe85 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -98,7 +98,7 @@ export class NodesChannelsMap implements OnInit, OnDestroy { } this.chartOptions = { - silent: true, + silent: this.style === 'widget' ? true : false, title: title ?? undefined, geo3D: { map: 'world', From 4335ee815716d286609f1638a020286ce32ec359 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 1 Aug 2022 18:41:31 +0200 Subject: [PATCH 11/53] Set default values when pubkey, capacity and channels are missing from top nodes --- backend/src/api/explorer/nodes.api.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index f05c4cf0a..4c7028136 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -66,7 +66,15 @@ class NodesApi { public async $getTopCapacityNodes(): Promise { try { - const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.capacity DESC LIMIT 10`; + const query = ` + SELECT IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, nodes.public_key, + CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, + CAST(COALESCE(node_stats.channels, 0) as INT) as channels + FROM nodes + LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key + ORDER BY node_stats.added DESC, node_stats.capacity DESC + LIMIT 10 + `; const [rows]: any = await DB.query(query); return rows; } catch (e) { @@ -77,7 +85,15 @@ class NodesApi { public async $getTopChannelsNodes(): Promise { try { - const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.channels DESC LIMIT 10`; + const query = ` + SELECT IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, nodes.public_key, + CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, + CAST(COALESCE(node_stats.channels, 0) as INT) as channels + FROM nodes + LEFT JOIN node_stats + ON node_stats.public_key = nodes.public_key + ORDER BY node_stats.added DESC, node_stats.channels DESC + LIMIT 10`; const [rows]: any = await DB.query(query); return rows; } catch (e) { From 04d7265a86cac8f1972c149eff11a0ec48ebc23a Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 1 Aug 2022 18:55:35 +0200 Subject: [PATCH 12/53] Fix UX interaction with channels map --- .../nodes-channels-map/nodes-channels-map.component.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index 61f7afe85..c71ff88ad 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -112,13 +112,15 @@ export class NodesChannelsMap implements OnInit, OnDestroy { }, viewControl: { center: this.style === 'widget' ? [0, 0, -10] : undefined, - minDistance: this.style === 'widget' ? 22 : 0.1, - maxDistance: this.style === 'widget' ? 22 : 60, + minDistance: 1, + maxDistance: 60, distance: this.style === 'widget' ? 22 : 60, alpha: 90, - panMouseButton: 'left', + rotateSensitivity: 0, + panSensitivity: this.style === 'widget' ? 0 : 1, + zoomSensitivity: this.style === 'widget' ? 0 : 0.5, + panMouseButton: this.style === 'widget' ? null : 'left', rotateMouseButton: undefined, - zoomSensivity: 0.5, }, itemStyle: { color: 'white', From 54eb11a2a7589bfe5ea5c8f91f3bb010db7428e3 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 1 Aug 2022 20:08:53 +0200 Subject: [PATCH 13/53] Limit matomo to mempool.space --- frontend/src/app/services/enterprise.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/services/enterprise.service.ts b/frontend/src/app/services/enterprise.service.ts index b125739d6..41a6194a1 100644 --- a/frontend/src/app/services/enterprise.service.ts +++ b/frontend/src/app/services/enterprise.service.ts @@ -24,7 +24,7 @@ export class EnterpriseService { this.subdomain = subdomain; this.fetchSubdomainInfo(); this.disableSubnetworks(); - } else { + } else if (document.location.hostname === 'mempool.space') { this.insertMatomo(); } } From b8c82c8f2cec0faa3019a991fc883a8176f3e75f Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 2 Aug 2022 00:07:35 +0200 Subject: [PATCH 14/53] [ops] Fix syslog permissions for /var/log/mempool --- production/newsyslog-mempool-backend.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/production/newsyslog-mempool-backend.conf b/production/newsyslog-mempool-backend.conf index 5c96da47a..df0ae9c47 100644 --- a/production/newsyslog-mempool-backend.conf +++ b/production/newsyslog-mempool-backend.conf @@ -1,2 +1,2 @@ -/var/log/mempool 640 10 * @T00 C -/var/log/mempool.debug 640 10 * @T00 C +/var/log/mempool 644 10 * @T00 C +/var/log/mempool.debug 644 10 * @T00 C From 82f981443898041e92f02823cb2140d3c1838340 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 2 Aug 2022 01:00:06 +0200 Subject: [PATCH 15/53] [ops] Fix cron jobs to update liquid assets hourly --- production/elements.crontab | 8 ++++++-- production/mempool.crontab | 11 ++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/production/elements.crontab b/production/elements.crontab index 4459a8c5b..5ba8151a3 100644 --- a/production/elements.crontab +++ b/production/elements.crontab @@ -1,6 +1,10 @@ +# start elements on reboot @reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidv1 >/dev/null 2>&1 @reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidtestnet >/dev/null 2>&1 + +# start electrs on reboot @reboot sleep 90 ; screen -dmS liquidv1 /elements/electrs/electrs-start-liquid @reboot sleep 90 ; screen -dmS liquidtestnet /elements/electrs/electrs-start-liquidtestnet -6 * * * * cd $HOME/asset_registry_db && git pull origin master >/dev/null 2>&1 -6 * * * * cd $HOME/asset_registry_testnet_db && git pull origin master >/dev/null 2>&1 + +# hourly asset update and electrs restart +6 * * * * cd $HOME/asset_registry_db && git pull --quiet origin master && cd $HOME/asset_registry_testnet_db && git pull --quiet origin master && killall electrs diff --git a/production/mempool.crontab b/production/mempool.crontab index d953feac4..08639362f 100644 --- a/production/mempool.crontab +++ b/production/mempool.crontab @@ -1,3 +1,12 @@ +# start on reboot @reboot sleep 10 ; $HOME/start -37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 & + +# start cache warmer on reboot @reboot sleep 180 ; /mempool/mempool/production/nginx-cache-warmer >/dev/null 2>&1 & + +# daily backup +37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 & + +# hourly liquid asset update +6 * * * * cd $HOME/liquid/frontend && npm run sync-assets && rsync -av $HOME/liquid/frontend/dist/mempool/browser/en-US/resources/assets* $HOME/public_html/liquid/en-US/resources/ >/dev/null 2>&1 + From 4ea1e98547c09db50fade30756a7a60ed4064d60 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 1 Aug 2022 17:25:44 +0200 Subject: [PATCH 16/53] Import LN historical statistics (network wide + per node) --- backend/package-lock.json | 38 +++ backend/package.json | 1 + backend/src/api/database-migration.ts | 6 +- backend/src/config.ts | 4 +- .../tasks/lightning/stats-updater.service.ts | 183 +---------- .../sync-tasks/funding-tx-fetcher.ts | 104 +++++++ .../lightning/sync-tasks/stats-importer.ts | 287 ++++++++++++++++++ 7 files changed, 440 insertions(+), 183 deletions(-) create mode 100644 backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts create mode 100644 backend/src/tasks/lightning/sync-tasks/stats-importer.ts diff --git a/backend/package-lock.json b/backend/package-lock.json index b23a7f874..968cb953b 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -31,6 +31,7 @@ "@typescript-eslint/parser": "^5.30.5", "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", + "fast-xml-parser": "^4.0.9", "prettier": "^2.7.1" } }, @@ -1496,6 +1497,22 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.9.tgz", + "integrity": "sha512-4G8EzDg2Nb1Qurs3f7BpFV4+jpMVsdgLVuG1Uv8O2OHJfVCg7gcA53obuKbmVqzd4Y7YXVBK05oJG7hzGIdyzg==", + "dev": true, + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + }, + "funding": { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + }, "node_modules/fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -2665,6 +2682,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", + "dev": true + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -3973,6 +3996,15 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-xml-parser": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.9.tgz", + "integrity": "sha512-4G8EzDg2Nb1Qurs3f7BpFV4+jpMVsdgLVuG1Uv8O2OHJfVCg7gcA53obuKbmVqzd4Y7YXVBK05oJG7hzGIdyzg==", + "dev": true, + "requires": { + "strnum": "^1.0.5" + } + }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -4817,6 +4849,12 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", + "dev": true + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", diff --git a/backend/package.json b/backend/package.json index 6345e89da..750380156 100644 --- a/backend/package.json +++ b/backend/package.json @@ -53,6 +53,7 @@ "@typescript-eslint/parser": "^5.30.5", "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", + "fast-xml-parser": "^4.0.9", "prettier": "^2.7.1" } } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index d26bfd6cc..816efc7cc 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 33; + private static currentVersion = 34; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -311,6 +311,10 @@ class DatabaseMigration { if (databaseSchemaVersion < 33 && isBitcoin == true) { await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL'); } + + if (databaseSchemaVersion < 34 && isBitcoin == true) { + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"'); + } } /** diff --git a/backend/src/config.ts b/backend/src/config.ts index 5560a25a7..d480e6c51 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -31,6 +31,7 @@ interface IConfig { LIGHTNING: { ENABLED: boolean; BACKEND: 'lnd' | 'cln' | 'ldk'; + TOPOLOGY_FOLDER: string; }; LND: { TLS_CERT_PATH: string; @@ -177,7 +178,8 @@ const defaults: IConfig = { }, 'LIGHTNING': { 'ENABLED': false, - 'BACKEND': 'lnd' + 'BACKEND': 'lnd', + 'TOPOLOGY_FOLDER': '', }, 'LND': { 'TLS_CERT_PATH': '', diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 0a3ade614..c48b683cd 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -3,7 +3,7 @@ import DB from '../../database'; import logger from '../../logger'; import lightningApi from '../../api/lightning/lightning-api-factory'; import channelsApi from '../../api/explorer/channels.api'; -import * as net from 'net'; +import { isIP } from 'net'; class LightningStatsUpdater { hardCodedStartTime = '2018-01-12'; @@ -28,9 +28,6 @@ class LightningStatsUpdater { return; } - await this.$populateHistoricalStatistics(); - await this.$populateHistoricalNodeStatistics(); - setTimeout(() => { this.$runTasks(); }, this.timeUntilMidnight()); @@ -85,7 +82,7 @@ class LightningStatsUpdater { if (hasOnion) { torNodes++; } - const hasClearnet = [4, 6].includes(net.isIP(socket.addr.split(':')[0])); + const hasClearnet = [4, 6].includes(isIP(socket.split(':')[0])); if (hasClearnet) { clearnetNodes++; } @@ -167,182 +164,6 @@ class LightningStatsUpdater { logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e)); } } - - // We only run this on first launch - private async $populateHistoricalStatistics() { - try { - const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`); - // Only run if table is empty - if (rows[0]['COUNT(*)'] > 0) { - return; - } - logger.info(`Running historical stats population...`); - - const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`); - const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`); - - const date: Date = new Date(this.hardCodedStartTime); - const currentDate = new Date(); - this.setDateMidnight(currentDate); - - while (date < currentDate) { - let totalCapacity = 0; - let channelsCount = 0; - - for (const channel of channels) { - if (new Date(channel.created) > date) { - break; - } - if (channel.closing_date === null || new Date(channel.closing_date) > date) { - totalCapacity += channel.capacity; - channelsCount++; - } - } - - let nodeCount = 0; - let clearnetNodes = 0; - let torNodes = 0; - let unannouncedNodes = 0; - - for (const node of nodes) { - if (new Date(node.first_seen) > date) { - break; - } - nodeCount++; - - const sockets = node.sockets.split(','); - let isUnnanounced = true; - for (const socket of sockets) { - const hasOnion = socket.indexOf('.onion') !== -1; - if (hasOnion) { - torNodes++; - isUnnanounced = false; - } - const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':')))); - if (hasClearnet) { - clearnetNodes++; - isUnnanounced = false; - } - } - if (isUnnanounced) { - unannouncedNodes++; - } - } - - const query = `INSERT INTO lightning_stats( - added, - channel_count, - node_count, - total_capacity, - tor_nodes, - clearnet_nodes, - unannounced_nodes, - avg_capacity, - avg_fee_rate, - avg_base_fee_mtokens, - med_capacity, - med_fee_rate, - med_base_fee_mtokens - ) - VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; - - const rowTimestamp = date.getTime() / 1000; // Save timestamp for the row insertion down below - - date.setUTCDate(date.getUTCDate() + 1); - - // Last iteration, save channels stats - const channelStats = (date >= currentDate ? await channelsApi.$getChannelsStats() : undefined); - - await DB.query(query, [ - rowTimestamp, - channelsCount, - nodeCount, - totalCapacity, - torNodes, - clearnetNodes, - unannouncedNodes, - channelStats?.avgCapacity ?? 0, - channelStats?.avgFeeRate ?? 0, - channelStats?.avgBaseFee ?? 0, - channelStats?.medianCapacity ?? 0, - channelStats?.medianFeeRate ?? 0, - channelStats?.medianBaseFee ?? 0, - ]); - } - - logger.info('Historical stats populated.'); - } catch (e) { - logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $populateHistoricalNodeStatistics() { - try { - const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`); - // Only run if table is empty - if (rows[0]['COUNT(*)'] > 0) { - return; - } - logger.info(`Running historical node stats population...`); - - const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`); - - for (const node of nodes) { - const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); - - const date: Date = new Date(this.hardCodedStartTime); - const currentDate = new Date(); - this.setDateMidnight(currentDate); - - let lastTotalCapacity = 0; - let lastChannelsCount = 0; - - while (date < currentDate) { - let totalCapacity = 0; - let channelsCount = 0; - for (const channel of channels) { - if (new Date(channel.created) > date) { - break; - } - if (channel.closing_date !== null && new Date(channel.closing_date) < date) { - date.setUTCDate(date.getUTCDate() + 1); - continue; - } - totalCapacity += channel.capacity; - channelsCount++; - } - - if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) { - date.setUTCDate(date.getUTCDate() + 1); - continue; - } - - lastTotalCapacity = totalCapacity; - lastChannelsCount = channelsCount; - - const query = `INSERT INTO node_stats( - public_key, - added, - capacity, - channels - ) - VALUES (?, FROM_UNIXTIME(?), ?, ?)`; - - await DB.query(query, [ - node.public_key, - date.getTime() / 1000, - totalCapacity, - channelsCount, - ]); - date.setUTCDate(date.getUTCDate() + 1); - } - logger.debug('Updated node_stats for: ' + node.alias); - } - logger.info('Historical stats populated.'); - } catch (e) { - logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e)); - } - } } export default new LightningStatsUpdater(); diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts new file mode 100644 index 000000000..b9407c44d --- /dev/null +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -0,0 +1,104 @@ +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import bitcoinClient from '../../../api/bitcoin/bitcoin-client'; +import config from '../../../config'; +import logger from '../../../logger'; + +const BLOCKS_CACHE_MAX_SIZE = 100; +const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json'; + +class FundingTxFetcher { + private running = false; + private blocksCache = {}; + private channelNewlyProcessed = 0; + public fundingTxCache = {}; + + async $fetchChannelsFundingTxs(channelIds: string[]): Promise { + if (this.running) { + return; + } + this.running = true; + + // Load funding tx disk cache + if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) { + try { + this.fundingTxCache = JSON.parse(readFileSync(CACHE_FILE_NAME, 'utf-8')); + } catch (e) { + logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`); + this.fundingTxCache = {}; + } + logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`); + } + + const globalTimer = new Date().getTime() / 1000; + let cacheTimer = new Date().getTime() / 1000; + let loggerTimer = new Date().getTime() / 1000; + let channelProcessed = 0; + this.channelNewlyProcessed = 0; + for (const channelId of channelIds) { + await this.$fetchChannelOpenTx(channelId); + ++channelProcessed; + + let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer); + if (elapsedSeconds > 10) { + elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer); + logger.debug(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` + + `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` + + `elapsed: ${elapsedSeconds} seconds` + ); + loggerTimer = new Date().getTime() / 1000; + } + + elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer); + if (elapsedSeconds > 60) { + logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`); + writeFileSync(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)); + cacheTimer = new Date().getTime() / 1000; + } + } + + if (this.channelNewlyProcessed > 0) { + logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`); + logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`); + writeFileSync(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)); + } + + this.running = false; + } + + public async $fetchChannelOpenTx(channelId: string): Promise { + if (this.fundingTxCache[channelId]) { + return this.fundingTxCache[channelId]; + } + + const parts = channelId.split('x'); + const blockHeight = parts[0]; + const txIdx = parts[1]; + const outputIdx = parts[2]; + + let block = this.blocksCache[blockHeight]; + if (!block) { + const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10)); + block = await bitcoinClient.getBlock(blockHash, 2); + this.blocksCache[block.height] = block; + } + + const blocksCacheHashes = Object.keys(this.blocksCache).sort(); + if (blocksCacheHashes.length > BLOCKS_CACHE_MAX_SIZE) { + for (let i = 0; i < 10; ++i) { + delete this.blocksCache[blocksCacheHashes[i]]; + } + } + + this.fundingTxCache[channelId] = { + timestamp: block.time, + txid: block.tx[txIdx].txid, + value: block.tx[txIdx].vout[outputIdx].value, + }; + + ++this.channelNewlyProcessed; + + return this.fundingTxCache[channelId]; + } +} + +export default new FundingTxFetcher; \ No newline at end of file diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts new file mode 100644 index 000000000..a0a256457 --- /dev/null +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -0,0 +1,287 @@ +import DB from '../../../database'; +import { readdirSync, readFileSync } from 'fs'; +import { XMLParser } from 'fast-xml-parser'; +import logger from '../../../logger'; +import fundingTxFetcher from './funding-tx-fetcher'; +import config from '../../../config'; + +interface Node { + id: string; + timestamp: number; + features: string; + rgb_color: string; + alias: string; + addresses: string; + out_degree: number; + in_degree: number; +} + +interface Channel { + scid: string; + source: string; + destination: string; + timestamp: number; + features: string; + fee_base_msat: number; + fee_proportional_millionths: number; + htlc_minimim_msat: number; + cltv_expiry_delta: number; + htlc_maximum_msat: number; +} + +const topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER; +const parser = new XMLParser(); + +let latestNodeCount = 1; // Ignore gap in the data + +async function $run(): Promise { + // const [channels]: any[] = await DB.query('SELECT short_id from channels;'); + // logger.info('Caching funding txs for currently existing channels'); + // await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); + + await $importHistoricalLightningStats(); +} + +/** + * Parse the file content into XML, and return a list of nodes and channels + */ +function parseFile(fileContent): any { + const graph = parser.parse(fileContent); + if (Object.keys(graph).length === 0) { + return null; + } + + const nodes: Node[] = []; + const channels: Channel[] = []; + + // If there is only one entry, the parser does not return an array, so we override this + if (!Array.isArray(graph.graphml.graph.node)) { + graph.graphml.graph.node = [graph.graphml.graph.node]; + } + if (!Array.isArray(graph.graphml.graph.edge)) { + graph.graphml.graph.edge = [graph.graphml.graph.edge]; + } + + for (const node of graph.graphml.graph.node) { + if (!node.data) { + continue; + } + nodes.push({ + id: node.data[0], + timestamp: node.data[1], + features: node.data[2], + rgb_color: node.data[3], + alias: node.data[4], + addresses: node.data[5], + out_degree: node.data[6], + in_degree: node.data[7], + }); + } + + for (const channel of graph.graphml.graph.edge) { + if (!channel.data) { + continue; + } + channels.push({ + scid: channel.data[0], + source: channel.data[1], + destination: channel.data[2], + timestamp: channel.data[3], + features: channel.data[4], + fee_base_msat: channel.data[5], + fee_proportional_millionths: channel.data[6], + htlc_minimim_msat: channel.data[7], + cltv_expiry_delta: channel.data[8], + htlc_maximum_msat: channel.data[9], + }); + } + + return { + nodes: nodes, + channels: channels, + }; +} + +/** + * Generate LN network stats for one day + */ +async function computeNetworkStats(timestamp: number, networkGraph): Promise { + // Node counts and network shares + let clearnetNodes = 0; + let torNodes = 0; + let clearnetTorNodes = 0; + let unannouncedNodes = 0; + + for (const node of networkGraph.nodes) { + let hasOnion = false; + let hasClearnet = false; + let isUnnanounced = true; + + const sockets = node.addresses.split(','); + for (const socket of sockets) { + hasOnion = hasOnion || (socket.indexOf('torv3://') !== -1); + hasClearnet = hasClearnet || (socket.indexOf('ipv4://') !== -1 || socket.indexOf('ipv6://') !== -1); + } + if (hasOnion && hasClearnet) { + clearnetTorNodes++; + isUnnanounced = false; + } else if (hasOnion) { + torNodes++; + isUnnanounced = false; + } else if (hasClearnet) { + clearnetNodes++; + isUnnanounced = false; + } + if (isUnnanounced) { + unannouncedNodes++; + } + } + + // Channels and node historical stats + const nodeStats = {}; + let capacity = 0; + let avgFeeRate = 0; + let avgBaseFee = 0; + const capacities: number[] = []; + const feeRates: number[] = []; + const baseFees: number[] = []; + for (const channel of networkGraph.channels) { + const tx = await fundingTxFetcher.$fetchChannelOpenTx(channel.scid.slice(0, -2)); + if (!tx) { + logger.err(`Unable to fetch funding tx for channel ${channel.scid}. Capacity and creation date will stay unknown.`); + continue; + } + + if (!nodeStats[channel.source]) { + nodeStats[channel.source] = { + capacity: 0, + channels: 0, + }; + } + if (!nodeStats[channel.destination]) { + nodeStats[channel.destination] = { + capacity: 0, + channels: 0, + }; + } + + nodeStats[channel.source].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.source].channels++; + nodeStats[channel.destination].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.destination].channels++; + + capacity += Math.round(tx.value * 100000000); + avgFeeRate += channel.fee_proportional_millionths; + avgBaseFee += channel.fee_base_msat; + capacities.push(Math.round(tx.value * 100000000)); + feeRates.push(channel.fee_proportional_millionths); + baseFees.push(channel.fee_base_msat); + } + + avgFeeRate /= networkGraph.channels.length; + avgBaseFee /= networkGraph.channels.length; + const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; + const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; + const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; + + let query = `INSERT INTO lightning_stats( + added, + channel_count, + node_count, + total_capacity, + tor_nodes, + clearnet_nodes, + unannounced_nodes, + clearnet_tor_nodes, + avg_capacity, + avg_fee_rate, + avg_base_fee_mtokens, + med_capacity, + med_fee_rate, + med_base_fee_mtokens + ) + VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + + await DB.query(query, [ + timestamp, + networkGraph.channels.length, + networkGraph.nodes.length, + capacity, + torNodes, + clearnetNodes, + unannouncedNodes, + clearnetTorNodes, + Math.round(capacity / networkGraph.channels.length), + avgFeeRate, + avgBaseFee, + medCapacity, + medFeeRate, + medBaseFee, + ]); + + for (const public_key of Object.keys(nodeStats)) { + query = `INSERT INTO node_stats( + public_key, + added, + capacity, + channels + ) + VALUES (?, FROM_UNIXTIME(?), ?, ?)`; + + await DB.query(query, [ + public_key, + timestamp, + nodeStats[public_key].capacity, + nodeStats[public_key].channels, + ]); + } +} + +export async function $importHistoricalLightningStats(): Promise { + const fileList = readdirSync(topologiesFolder); + fileList.sort().reverse(); + + const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) as added FROM lightning_stats'); + const existingStatsTimestamps = {}; + for (const row of rows) { + existingStatsTimestamps[row.added] = true; + } + + for (const filename of fileList) { + const timestamp = parseInt(filename.split('_')[1], 10); + const fileContent = readFileSync(`${topologiesFolder}/${filename}`, 'utf8'); + + const graph = parseFile(fileContent); + if (!graph) { + continue; + } + + // Ignore drop of more than 90% of the node count as it's probably a missing data point + const diffRatio = graph.nodes.length / latestNodeCount; + if (diffRatio < 0.90) { + continue; + } + latestNodeCount = graph.nodes.length; + + // Stats exist already, don't calculate/insert them + if (existingStatsTimestamps[timestamp] === true) { + continue; + } + + logger.debug(`Processing ${topologiesFolder}/${filename}`); + + const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; + logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.channels.length} channels`); + + // Cache funding txs + logger.debug(`Caching funding txs for ${datestr}`); + await fundingTxFetcher.$fetchChannelsFundingTxs(graph.channels.map(channel => channel.scid.slice(0, -2))); + + logger.debug(`Generating LN network stats for ${datestr}`); + await computeNetworkStats(timestamp, graph); + } + + logger.info(`Lightning network stats historical import completed`); +} + +$run().then(() => process.exit(0)); \ No newline at end of file From 91ada9ce751dea183c592f5bad4e4d3a5b1a036e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 1 Aug 2022 17:48:04 +0200 Subject: [PATCH 17/53] Integrate LN stats importer into the main process --- backend/src/index.ts | 10 +- .../tasks/lightning/stats-updater.service.ts | 5 +- .../lightning/sync-tasks/stats-importer.ts | 472 +++++++++--------- 3 files changed, 246 insertions(+), 241 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index b7159afaf..fa80fb2ad 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -29,11 +29,11 @@ import channelsRoutes from './api/explorer/channels.routes'; import generalLightningRoutes from './api/explorer/general.routes'; import lightningStatsUpdater from './tasks/lightning/stats-updater.service'; import nodeSyncService from './tasks/lightning/node-sync.service'; -import statisticsRoutes from "./api/statistics/statistics.routes"; -import miningRoutes from "./api/mining/mining-routes"; -import bisqRoutes from "./api/bisq/bisq.routes"; -import liquidRoutes from "./api/liquid/liquid.routes"; -import bitcoinRoutes from "./api/bitcoin/bitcoin.routes"; +import statisticsRoutes from './api/statistics/statistics.routes'; +import miningRoutes from './api/mining/mining-routes'; +import bisqRoutes from './api/bisq/bisq.routes'; +import liquidRoutes from './api/liquid/liquid.routes'; +import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; class Server { private wss: WebSocket.Server | undefined; diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index c48b683cd..c5ca55cd8 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -4,11 +4,12 @@ import logger from '../../logger'; import lightningApi from '../../api/lightning/lightning-api-factory'; import channelsApi from '../../api/explorer/channels.api'; import { isIP } from 'net'; +import LightningStatsImporter from './sync-tasks/stats-importer'; class LightningStatsUpdater { hardCodedStartTime = '2018-01-12'; - public async $startService() { + public async $startService(): Promise { logger.info('Starting Lightning Stats service'); let isInSync = false; let error: any; @@ -28,6 +29,8 @@ class LightningStatsUpdater { return; } + LightningStatsImporter.$run(); + setTimeout(() => { this.$runTasks(); }, this.timeUntilMidnight()); diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index a0a256457..9dd5751b9 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -29,259 +29,261 @@ interface Channel { htlc_maximum_msat: number; } -const topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER; -const parser = new XMLParser(); +class LightningStatsImporter { + topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER; + parser = new XMLParser(); -let latestNodeCount = 1; // Ignore gap in the data + latestNodeCount = 1; // Ignore gap in the data -async function $run(): Promise { - // const [channels]: any[] = await DB.query('SELECT short_id from channels;'); - // logger.info('Caching funding txs for currently existing channels'); - // await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); - - await $importHistoricalLightningStats(); -} - -/** - * Parse the file content into XML, and return a list of nodes and channels - */ -function parseFile(fileContent): any { - const graph = parser.parse(fileContent); - if (Object.keys(graph).length === 0) { - return null; + async $run(): Promise { + // const [channels]: any[] = await DB.query('SELECT short_id from channels;'); + // logger.info('Caching funding txs for currently existing channels'); + // await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); + + await this.$importHistoricalLightningStats(); } - const nodes: Node[] = []; - const channels: Channel[] = []; + /** + * Parse the file content into XML, and return a list of nodes and channels + */ + parseFile(fileContent): any { + const graph = this.parser.parse(fileContent); + if (Object.keys(graph).length === 0) { + return null; + } - // If there is only one entry, the parser does not return an array, so we override this - if (!Array.isArray(graph.graphml.graph.node)) { - graph.graphml.graph.node = [graph.graphml.graph.node]; - } - if (!Array.isArray(graph.graphml.graph.edge)) { - graph.graphml.graph.edge = [graph.graphml.graph.edge]; + const nodes: Node[] = []; + const channels: Channel[] = []; + + // If there is only one entry, the parser does not return an array, so we override this + if (!Array.isArray(graph.graphml.graph.node)) { + graph.graphml.graph.node = [graph.graphml.graph.node]; + } + if (!Array.isArray(graph.graphml.graph.edge)) { + graph.graphml.graph.edge = [graph.graphml.graph.edge]; + } + + for (const node of graph.graphml.graph.node) { + if (!node.data) { + continue; + } + nodes.push({ + id: node.data[0], + timestamp: node.data[1], + features: node.data[2], + rgb_color: node.data[3], + alias: node.data[4], + addresses: node.data[5], + out_degree: node.data[6], + in_degree: node.data[7], + }); + } + + for (const channel of graph.graphml.graph.edge) { + if (!channel.data) { + continue; + } + channels.push({ + scid: channel.data[0], + source: channel.data[1], + destination: channel.data[2], + timestamp: channel.data[3], + features: channel.data[4], + fee_base_msat: channel.data[5], + fee_proportional_millionths: channel.data[6], + htlc_minimim_msat: channel.data[7], + cltv_expiry_delta: channel.data[8], + htlc_maximum_msat: channel.data[9], + }); + } + + return { + nodes: nodes, + channels: channels, + }; } - for (const node of graph.graphml.graph.node) { - if (!node.data) { - continue; - } - nodes.push({ - id: node.data[0], - timestamp: node.data[1], - features: node.data[2], - rgb_color: node.data[3], - alias: node.data[4], - addresses: node.data[5], - out_degree: node.data[6], - in_degree: node.data[7], - }); - } + /** + * Generate LN network stats for one day + */ + async computeNetworkStats(timestamp: number, networkGraph): Promise { + // Node counts and network shares + let clearnetNodes = 0; + let torNodes = 0; + let clearnetTorNodes = 0; + let unannouncedNodes = 0; - for (const channel of graph.graphml.graph.edge) { - if (!channel.data) { - continue; - } - channels.push({ - scid: channel.data[0], - source: channel.data[1], - destination: channel.data[2], - timestamp: channel.data[3], - features: channel.data[4], - fee_base_msat: channel.data[5], - fee_proportional_millionths: channel.data[6], - htlc_minimim_msat: channel.data[7], - cltv_expiry_delta: channel.data[8], - htlc_maximum_msat: channel.data[9], - }); - } + for (const node of networkGraph.nodes) { + let hasOnion = false; + let hasClearnet = false; + let isUnnanounced = true; - return { - nodes: nodes, - channels: channels, - }; -} - -/** - * Generate LN network stats for one day - */ -async function computeNetworkStats(timestamp: number, networkGraph): Promise { - // Node counts and network shares - let clearnetNodes = 0; - let torNodes = 0; - let clearnetTorNodes = 0; - let unannouncedNodes = 0; - - for (const node of networkGraph.nodes) { - let hasOnion = false; - let hasClearnet = false; - let isUnnanounced = true; - - const sockets = node.addresses.split(','); - for (const socket of sockets) { - hasOnion = hasOnion || (socket.indexOf('torv3://') !== -1); - hasClearnet = hasClearnet || (socket.indexOf('ipv4://') !== -1 || socket.indexOf('ipv6://') !== -1); - } - if (hasOnion && hasClearnet) { - clearnetTorNodes++; - isUnnanounced = false; - } else if (hasOnion) { - torNodes++; - isUnnanounced = false; - } else if (hasClearnet) { - clearnetNodes++; - isUnnanounced = false; - } - if (isUnnanounced) { - unannouncedNodes++; - } - } - - // Channels and node historical stats - const nodeStats = {}; - let capacity = 0; - let avgFeeRate = 0; - let avgBaseFee = 0; - const capacities: number[] = []; - const feeRates: number[] = []; - const baseFees: number[] = []; - for (const channel of networkGraph.channels) { - const tx = await fundingTxFetcher.$fetchChannelOpenTx(channel.scid.slice(0, -2)); - if (!tx) { - logger.err(`Unable to fetch funding tx for channel ${channel.scid}. Capacity and creation date will stay unknown.`); - continue; + const sockets = node.addresses.split(','); + for (const socket of sockets) { + hasOnion = hasOnion || (socket.indexOf('torv3://') !== -1); + hasClearnet = hasClearnet || (socket.indexOf('ipv4://') !== -1 || socket.indexOf('ipv6://') !== -1); + } + if (hasOnion && hasClearnet) { + clearnetTorNodes++; + isUnnanounced = false; + } else if (hasOnion) { + torNodes++; + isUnnanounced = false; + } else if (hasClearnet) { + clearnetNodes++; + isUnnanounced = false; + } + if (isUnnanounced) { + unannouncedNodes++; + } } - if (!nodeStats[channel.source]) { - nodeStats[channel.source] = { - capacity: 0, - channels: 0, - }; - } - if (!nodeStats[channel.destination]) { - nodeStats[channel.destination] = { - capacity: 0, - channels: 0, - }; + // Channels and node historical stats + const nodeStats = {}; + let capacity = 0; + let avgFeeRate = 0; + let avgBaseFee = 0; + const capacities: number[] = []; + const feeRates: number[] = []; + const baseFees: number[] = []; + for (const channel of networkGraph.channels) { + const tx = await fundingTxFetcher.$fetchChannelOpenTx(channel.scid.slice(0, -2)); + if (!tx) { + logger.err(`Unable to fetch funding tx for channel ${channel.scid}. Capacity and creation date will stay unknown.`); + continue; + } + + if (!nodeStats[channel.source]) { + nodeStats[channel.source] = { + capacity: 0, + channels: 0, + }; + } + if (!nodeStats[channel.destination]) { + nodeStats[channel.destination] = { + capacity: 0, + channels: 0, + }; + } + + nodeStats[channel.source].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.source].channels++; + nodeStats[channel.destination].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.destination].channels++; + + capacity += Math.round(tx.value * 100000000); + avgFeeRate += channel.fee_proportional_millionths; + avgBaseFee += channel.fee_base_msat; + capacities.push(Math.round(tx.value * 100000000)); + feeRates.push(channel.fee_proportional_millionths); + baseFees.push(channel.fee_base_msat); } - nodeStats[channel.source].capacity += Math.round(tx.value * 100000000); - nodeStats[channel.source].channels++; - nodeStats[channel.destination].capacity += Math.round(tx.value * 100000000); - nodeStats[channel.destination].channels++; - - capacity += Math.round(tx.value * 100000000); - avgFeeRate += channel.fee_proportional_millionths; - avgBaseFee += channel.fee_base_msat; - capacities.push(Math.round(tx.value * 100000000)); - feeRates.push(channel.fee_proportional_millionths); - baseFees.push(channel.fee_base_msat); - } - - avgFeeRate /= networkGraph.channels.length; - avgBaseFee /= networkGraph.channels.length; - const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; - const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; - const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; - - let query = `INSERT INTO lightning_stats( - added, - channel_count, - node_count, - total_capacity, - tor_nodes, - clearnet_nodes, - unannounced_nodes, - clearnet_tor_nodes, - avg_capacity, - avg_fee_rate, - avg_base_fee_mtokens, - med_capacity, - med_fee_rate, - med_base_fee_mtokens - ) - VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; - - await DB.query(query, [ - timestamp, - networkGraph.channels.length, - networkGraph.nodes.length, - capacity, - torNodes, - clearnetNodes, - unannouncedNodes, - clearnetTorNodes, - Math.round(capacity / networkGraph.channels.length), - avgFeeRate, - avgBaseFee, - medCapacity, - medFeeRate, - medBaseFee, - ]); - - for (const public_key of Object.keys(nodeStats)) { - query = `INSERT INTO node_stats( - public_key, + avgFeeRate /= networkGraph.channels.length; + avgBaseFee /= networkGraph.channels.length; + const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; + const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; + const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; + + let query = `INSERT INTO lightning_stats( added, - capacity, - channels + channel_count, + node_count, + total_capacity, + tor_nodes, + clearnet_nodes, + unannounced_nodes, + clearnet_tor_nodes, + avg_capacity, + avg_fee_rate, + avg_base_fee_mtokens, + med_capacity, + med_fee_rate, + med_base_fee_mtokens ) - VALUES (?, FROM_UNIXTIME(?), ?, ?)`; - + VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + await DB.query(query, [ - public_key, timestamp, - nodeStats[public_key].capacity, - nodeStats[public_key].channels, + networkGraph.channels.length, + networkGraph.nodes.length, + capacity, + torNodes, + clearnetNodes, + unannouncedNodes, + clearnetTorNodes, + Math.round(capacity / networkGraph.channels.length), + avgFeeRate, + avgBaseFee, + medCapacity, + medFeeRate, + medBaseFee, ]); + + for (const public_key of Object.keys(nodeStats)) { + query = `INSERT INTO node_stats( + public_key, + added, + capacity, + channels + ) + VALUES (?, FROM_UNIXTIME(?), ?, ?)`; + + await DB.query(query, [ + public_key, + timestamp, + nodeStats[public_key].capacity, + nodeStats[public_key].channels, + ]); + } + } + + async $importHistoricalLightningStats(): Promise { + const fileList = readdirSync(this.topologiesFolder); + fileList.sort().reverse(); + + const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) as added FROM lightning_stats'); + const existingStatsTimestamps = {}; + for (const row of rows) { + existingStatsTimestamps[row.added] = true; + } + + for (const filename of fileList) { + const timestamp = parseInt(filename.split('_')[1], 10); + const fileContent = readFileSync(`${this.topologiesFolder}/${filename}`, 'utf8'); + + const graph = this.parseFile(fileContent); + if (!graph) { + continue; + } + + // Ignore drop of more than 90% of the node count as it's probably a missing data point + const diffRatio = graph.nodes.length / this.latestNodeCount; + if (diffRatio < 0.90) { + continue; + } + this.latestNodeCount = graph.nodes.length; + + // Stats exist already, don't calculate/insert them + if (existingStatsTimestamps[timestamp] === true) { + continue; + } + + logger.debug(`Processing ${this.topologiesFolder}/${filename}`); + + const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; + logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.channels.length} channels`); + + // Cache funding txs + logger.debug(`Caching funding txs for ${datestr}`); + await fundingTxFetcher.$fetchChannelsFundingTxs(graph.channels.map(channel => channel.scid.slice(0, -2))); + + logger.debug(`Generating LN network stats for ${datestr}`); + await this.computeNetworkStats(timestamp, graph); + } + + logger.info(`Lightning network stats historical import completed`); } } -export async function $importHistoricalLightningStats(): Promise { - const fileList = readdirSync(topologiesFolder); - fileList.sort().reverse(); - - const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) as added FROM lightning_stats'); - const existingStatsTimestamps = {}; - for (const row of rows) { - existingStatsTimestamps[row.added] = true; - } - - for (const filename of fileList) { - const timestamp = parseInt(filename.split('_')[1], 10); - const fileContent = readFileSync(`${topologiesFolder}/${filename}`, 'utf8'); - - const graph = parseFile(fileContent); - if (!graph) { - continue; - } - - // Ignore drop of more than 90% of the node count as it's probably a missing data point - const diffRatio = graph.nodes.length / latestNodeCount; - if (diffRatio < 0.90) { - continue; - } - latestNodeCount = graph.nodes.length; - - // Stats exist already, don't calculate/insert them - if (existingStatsTimestamps[timestamp] === true) { - continue; - } - - logger.debug(`Processing ${topologiesFolder}/${filename}`); - - const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; - logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.channels.length} channels`); - - // Cache funding txs - logger.debug(`Caching funding txs for ${datestr}`); - await fundingTxFetcher.$fetchChannelsFundingTxs(graph.channels.map(channel => channel.scid.slice(0, -2))); - - logger.debug(`Generating LN network stats for ${datestr}`); - await computeNetworkStats(timestamp, graph); - } - - logger.info(`Lightning network stats historical import completed`); -} - -$run().then(() => process.exit(0)); \ No newline at end of file +export default new LightningStatsImporter; \ No newline at end of file From 2daf94f65a5a3687140190800ca780e95737358a Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 1 Aug 2022 18:21:45 +0200 Subject: [PATCH 18/53] Re-use LN stats importer code to log daily LN stats --- .../tasks/lightning/stats-updater.service.ts | 111 ++---------------- .../lightning/sync-tasks/stats-importer.ts | 4 +- 2 files changed, 11 insertions(+), 104 deletions(-) diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index c5ca55cd8..d093892bb 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -56,116 +56,21 @@ class LightningStatsUpdater { } private async $runTasks(): Promise { - await this.$logLightningStatsDaily(); - await this.$logNodeStatsDaily(); + await this.$logStatsDaily(); setTimeout(() => { this.$runTasks(); }, this.timeUntilMidnight()); } - private async $logLightningStatsDaily() { - try { - logger.info(`Running lightning daily stats log...`); + private async $logStatsDaily(): Promise { + const date = new Date(); + this.setDateMidnight(date); + date.setUTCHours(24); - const networkGraph = await lightningApi.$getNetworkGraph(); - let total_capacity = 0; - for (const channel of networkGraph.edges) { - if (channel.capacity) { - total_capacity += parseInt(channel.capacity); - } - } - - let clearnetNodes = 0; - let torNodes = 0; - let unannouncedNodes = 0; - for (const node of networkGraph.nodes) { - for (const socket of node.addresses) { - const hasOnion = socket.addr.indexOf('.onion') !== -1; - if (hasOnion) { - torNodes++; - } - const hasClearnet = [4, 6].includes(isIP(socket.split(':')[0])); - if (hasClearnet) { - clearnetNodes++; - } - } - if (node.addresses.length === 0) { - unannouncedNodes++; - } - } - - const channelStats = await channelsApi.$getChannelsStats(); - - const query = `INSERT INTO lightning_stats( - added, - channel_count, - node_count, - total_capacity, - tor_nodes, - clearnet_nodes, - unannounced_nodes, - avg_capacity, - avg_fee_rate, - avg_base_fee_mtokens, - med_capacity, - med_fee_rate, - med_base_fee_mtokens - ) - VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; - - await DB.query(query, [ - networkGraph.edges.length, - networkGraph.nodes.length, - total_capacity, - torNodes, - clearnetNodes, - unannouncedNodes, - channelStats.avgCapacity, - channelStats.avgFeeRate, - channelStats.avgBaseFee, - channelStats.medianCapacity, - channelStats.medianFeeRate, - channelStats.medianBaseFee, - ]); - logger.info(`Lightning daily stats done.`); - } catch (e) { - logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $logNodeStatsDaily() { - try { - logger.info(`Running daily node stats update...`); - - const query = ` - SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left, - c2.channels_capacity_right - FROM nodes - LEFT JOIN ( - SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left - FROM channels - WHERE channels.status = 1 - GROUP BY node1_public_key - ) c1 ON c1.node1_public_key = nodes.public_key - LEFT JOIN ( - SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right - FROM channels WHERE channels.status = 1 GROUP BY node2_public_key - ) c2 ON c2.node2_public_key = nodes.public_key - `; - - const [nodes]: any = await DB.query(query); - - for (const node of nodes) { - await DB.query( - `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW() - INTERVAL 1 DAY, ?, ?)`, - [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)), - node.channels_count_left + node.channels_count_right]); - } - logger.info('Daily node stats has updated.'); - } catch (e) { - logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e)); - } + logger.info(`Running lightning daily stats log...`); + const networkGraph = await lightningApi.$getNetworkGraph(); + LightningStatsImporter.computeNetworkStats(date.getTime(), networkGraph); } } diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 9dd5751b9..f6d70df7d 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -36,6 +36,8 @@ class LightningStatsImporter { latestNodeCount = 1; // Ignore gap in the data async $run(): Promise { + logger.info(`Importing historical lightning stats`); + // const [channels]: any[] = await DB.query('SELECT short_id from channels;'); // logger.info('Caching funding txs for currently existing channels'); // await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); @@ -106,7 +108,7 @@ class LightningStatsImporter { /** * Generate LN network stats for one day */ - async computeNetworkStats(timestamp: number, networkGraph): Promise { + public async computeNetworkStats(timestamp: number, networkGraph): Promise { // Node counts and network shares let clearnetNodes = 0; let torNodes = 0; From b246c6f4c3a28ce7946b3113a9e7ac848a2752d9 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 1 Aug 2022 19:50:55 +0200 Subject: [PATCH 19/53] We don't need a synced node to import historical data --- .../tasks/lightning/stats-updater.service.ts | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index d093892bb..f364629b9 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -1,9 +1,6 @@ - import DB from '../../database'; import logger from '../../logger'; import lightningApi from '../../api/lightning/lightning-api-factory'; -import channelsApi from '../../api/explorer/channels.api'; -import { isIP } from 'net'; import LightningStatsImporter from './sync-tasks/stats-importer'; class LightningStatsUpdater { @@ -11,23 +8,6 @@ class LightningStatsUpdater { public async $startService(): Promise { logger.info('Starting Lightning Stats service'); - let isInSync = false; - let error: any; - try { - error = null; - isInSync = await this.$lightningIsSynced(); - } catch (e) { - error = e; - } - if (!isInSync) { - if (error) { - logger.warn('Was not able to fetch Lightning Node status: ' + (error instanceof Error ? error.message : error) + '. Retrying in 1 minute...'); - } else { - logger.notice('The Lightning graph is not yet in sync. Retrying in 1 minute...'); - } - setTimeout(() => this.$startService(), 60 * 1000); - return; - } LightningStatsImporter.$run(); @@ -50,11 +30,6 @@ class LightningStatsUpdater { date.setUTCMilliseconds(0); } - private async $lightningIsSynced(): Promise { - const nodeInfo = await lightningApi.$getInfo(); - return nodeInfo.synced_to_chain && nodeInfo.synced_to_graph; - } - private async $runTasks(): Promise { await this.$logStatsDaily(); From 528749089455e667407910c79575d53be5162003 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 12:19:57 +0200 Subject: [PATCH 20/53] Make sure to not count channels twice --- .../tasks/lightning/stats-updater.service.ts | 1 - .../sync-tasks/funding-tx-fetcher.ts | 34 ++++++++-- .../lightning/sync-tasks/stats-importer.ts | 65 +++++++++++-------- 3 files changed, 66 insertions(+), 34 deletions(-) diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index f364629b9..5701ef22a 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -1,4 +1,3 @@ -import DB from '../../database'; import logger from '../../logger'; import lightningApi from '../../api/lightning/lightning-api-factory'; import LightningStatsImporter from './sync-tasks/stats-importer'; diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts index b9407c44d..4068de8f1 100644 --- a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -1,8 +1,11 @@ -import { existsSync, readFileSync, writeFileSync } from 'fs'; +import { existsSync, promises } from 'fs'; import bitcoinClient from '../../../api/bitcoin/bitcoin-client'; import config from '../../../config'; +import DB from '../../../database'; import logger from '../../../logger'; +const fsPromises = promises; + const BLOCKS_CACHE_MAX_SIZE = 100; const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json'; @@ -21,7 +24,7 @@ class FundingTxFetcher { // Load funding tx disk cache if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) { try { - this.fundingTxCache = JSON.parse(readFileSync(CACHE_FILE_NAME, 'utf-8')); + this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8')); } catch (e) { logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`); this.fundingTxCache = {}; @@ -51,7 +54,7 @@ class FundingTxFetcher { elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer); if (elapsedSeconds > 60) { logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`); - writeFileSync(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)); + fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)); cacheTimer = new Date().getTime() / 1000; } } @@ -59,7 +62,7 @@ class FundingTxFetcher { if (this.channelNewlyProcessed > 0) { logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`); logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`); - writeFileSync(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)); + fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)); } this.running = false; @@ -76,13 +79,30 @@ class FundingTxFetcher { const outputIdx = parts[2]; let block = this.blocksCache[blockHeight]; + // Check if we have the block in the `blocks_summaries` table to avoid calling core + if (!block) { + const [rows] = await DB.query(` + SELECT UNIX_TIMESTAMP(blocks.blockTimestamp) AS time, blocks_summaries.transactions AS tx + FROM blocks_summaries + JOIN blocks ON blocks.hash = blocks_summaries.id + WHERE blocks_summaries.height = ${blockHeight} + `); + block = rows[0] ?? null; + if (block) { + block.tx = JSON.parse(block.tx); + if (block.tx.length === 0) { + block = null; + } + } + } + // Fetch it from core if (!block) { const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10)); block = await bitcoinClient.getBlock(blockHash, 2); - this.blocksCache[block.height] = block; } + this.blocksCache[block.height] = block; - const blocksCacheHashes = Object.keys(this.blocksCache).sort(); + const blocksCacheHashes = Object.keys(this.blocksCache).sort((a, b) => parseInt(b) - parseInt(a)).reverse(); if (blocksCacheHashes.length > BLOCKS_CACHE_MAX_SIZE) { for (let i = 0; i < 10; ++i) { delete this.blocksCache[blocksCacheHashes[i]]; @@ -92,7 +112,7 @@ class FundingTxFetcher { this.fundingTxCache[channelId] = { timestamp: block.time, txid: block.tx[txIdx].txid, - value: block.tx[txIdx].vout[outputIdx].value, + value: block.tx[txIdx].value / 100000000 ?? block.tx[txIdx].vout[outputIdx].value, }; ++this.channelNewlyProcessed; diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index f6d70df7d..8482b558c 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -1,10 +1,12 @@ import DB from '../../../database'; -import { readdirSync, readFileSync } from 'fs'; +import { promises } from 'fs'; import { XMLParser } from 'fast-xml-parser'; import logger from '../../../logger'; import fundingTxFetcher from './funding-tx-fetcher'; import config from '../../../config'; +const fsPromises = promises; + interface Node { id: string; timestamp: number; @@ -33,14 +35,12 @@ class LightningStatsImporter { topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER; parser = new XMLParser(); - latestNodeCount = 1; // Ignore gap in the data - async $run(): Promise { logger.info(`Importing historical lightning stats`); - // const [channels]: any[] = await DB.query('SELECT short_id from channels;'); - // logger.info('Caching funding txs for currently existing channels'); - // await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); + const [channels]: any[] = await DB.query('SELECT short_id from channels;'); + logger.info('Caching funding txs for currently existing channels'); + await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); await this.$importHistoricalLightningStats(); } @@ -148,6 +148,8 @@ class LightningStatsImporter { const capacities: number[] = []; const feeRates: number[] = []; const baseFees: number[] = []; + const alreadyCountedChannels = {}; + for (const channel of networkGraph.channels) { const tx = await fundingTxFetcher.$fetchChannelOpenTx(channel.scid.slice(0, -2)); if (!tx) { @@ -173,10 +175,14 @@ class LightningStatsImporter { nodeStats[channel.destination].capacity += Math.round(tx.value * 100000000); nodeStats[channel.destination].channels++; - capacity += Math.round(tx.value * 100000000); + if (!alreadyCountedChannels[channel.scid.slice(0, -2)]) { + capacity += Math.round(tx.value * 100000000); + capacities.push(Math.round(tx.value * 100000000)); + alreadyCountedChannels[channel.scid.slice(0, -2)] = true; + } + avgFeeRate += channel.fee_proportional_millionths; avgBaseFee += channel.fee_base_msat; - capacities.push(Math.round(tx.value * 100000000)); feeRates.push(channel.fee_proportional_millionths); baseFees.push(channel.fee_base_msat); } @@ -186,6 +192,7 @@ class LightningStatsImporter { const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; + const avgCapacity = Math.round(capacity / capacities.length); let query = `INSERT INTO lightning_stats( added, @@ -207,14 +214,14 @@ class LightningStatsImporter { await DB.query(query, [ timestamp, - networkGraph.channels.length, + capacities.length, networkGraph.nodes.length, capacity, torNodes, clearnetNodes, unannouncedNodes, clearnetTorNodes, - Math.round(capacity / networkGraph.channels.length), + avgCapacity, avgFeeRate, avgBaseFee, medCapacity, @@ -241,10 +248,10 @@ class LightningStatsImporter { } async $importHistoricalLightningStats(): Promise { - const fileList = readdirSync(this.topologiesFolder); + const fileList = await fsPromises.readdir(this.topologiesFolder); fileList.sort().reverse(); - const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) as added FROM lightning_stats'); + const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) AS added FROM lightning_stats'); const existingStatsTimestamps = {}; for (const row of rows) { existingStatsTimestamps[row.added] = true; @@ -252,26 +259,30 @@ class LightningStatsImporter { for (const filename of fileList) { const timestamp = parseInt(filename.split('_')[1], 10); - const fileContent = readFileSync(`${this.topologiesFolder}/${filename}`, 'utf8'); - - const graph = this.parseFile(fileContent); - if (!graph) { - continue; - } - - // Ignore drop of more than 90% of the node count as it's probably a missing data point - const diffRatio = graph.nodes.length / this.latestNodeCount; - if (diffRatio < 0.90) { - continue; - } - this.latestNodeCount = graph.nodes.length; // Stats exist already, don't calculate/insert them - if (existingStatsTimestamps[timestamp] === true) { + if (existingStatsTimestamps[timestamp] !== undefined) { continue; } logger.debug(`Processing ${this.topologiesFolder}/${filename}`); + const fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8'); + + let graph; + if (filename.indexOf('.json') !== -1) { + try { + graph = JSON.parse(fileContent); + } catch (e) { + logger.debug(`Invalid topology file, cannot parse the content`); + } + } else { + graph = this.parseFile(fileContent); + if (!graph) { + logger.debug(`Invalid topology file, cannot parse the content`); + continue; + } + await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph)); + } const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.channels.length} channels`); @@ -282,6 +293,8 @@ class LightningStatsImporter { logger.debug(`Generating LN network stats for ${datestr}`); await this.computeNetworkStats(timestamp, graph); + + existingStatsTimestamps[timestamp] = true; } logger.info(`Lightning network stats historical import completed`); From 7fdf95ad3403cca41dd29aa61f8bbadab5a189b5 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 13:03:32 +0200 Subject: [PATCH 21/53] Remove buggy tx vout value fetching and improve performances --- .../sync-tasks/funding-tx-fetcher.ts | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts index 4068de8f1..9da721876 100644 --- a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -1,4 +1,5 @@ import { existsSync, promises } from 'fs'; +import bitcoinApiFactory from '../../../api/bitcoin/bitcoin-api-factory'; import bitcoinClient from '../../../api/bitcoin/bitcoin-client'; import config from '../../../config'; import DB from '../../../database'; @@ -79,26 +80,10 @@ class FundingTxFetcher { const outputIdx = parts[2]; let block = this.blocksCache[blockHeight]; - // Check if we have the block in the `blocks_summaries` table to avoid calling core - if (!block) { - const [rows] = await DB.query(` - SELECT UNIX_TIMESTAMP(blocks.blockTimestamp) AS time, blocks_summaries.transactions AS tx - FROM blocks_summaries - JOIN blocks ON blocks.hash = blocks_summaries.id - WHERE blocks_summaries.height = ${blockHeight} - `); - block = rows[0] ?? null; - if (block) { - block.tx = JSON.parse(block.tx); - if (block.tx.length === 0) { - block = null; - } - } - } // Fetch it from core if (!block) { const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10)); - block = await bitcoinClient.getBlock(blockHash, 2); + block = await bitcoinClient.getBlock(blockHash, 1); } this.blocksCache[block.height] = block; @@ -109,10 +94,14 @@ class FundingTxFetcher { } } + const txid = block.tx[txIdx]; + const rawTx = await bitcoinClient.getRawTransaction(txid); + const tx = await bitcoinClient.decodeRawTransaction(rawTx); + this.fundingTxCache[channelId] = { timestamp: block.time, - txid: block.tx[txIdx].txid, - value: block.tx[txIdx].value / 100000000 ?? block.tx[txIdx].vout[outputIdx].value, + txid: txid, + value: tx.vout[outputIdx].value, }; ++this.channelNewlyProcessed; From 5d7e42195f4f666bb112b5f3deba02e90cafbf2e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 15:02:24 +0200 Subject: [PATCH 22/53] Reduce massive gaps in the imported historical LN data --- .../tasks/lightning/sync-tasks/stats-importer.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 8482b558c..4f7c5ca04 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -248,6 +248,8 @@ class LightningStatsImporter { } async $importHistoricalLightningStats(): Promise { + let latestNodeCount = 1; + const fileList = await fsPromises.readdir(this.topologiesFolder); fileList.sort().reverse(); @@ -284,6 +286,17 @@ class LightningStatsImporter { await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph)); } + if (timestamp > 1556316000) { + // "No, the reason most likely is just that I started collection in 2019, + // so what I had before that is just the survivors from before, which weren't that many" + const diffRatio = graph.nodes.length / latestNodeCount; + if (diffRatio < 0.9) { + // Ignore drop of more than 90% of the node count as it's probably a missing data point + continue; + } + } + latestNodeCount = graph.nodes.length; + const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.channels.length} channels`); From d7f2f4136c9a0fb13a19bbf7bd0ce407a18af19b Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 15:58:29 +0200 Subject: [PATCH 23/53] Small cleanup --- .../src/tasks/lightning/sync-tasks/stats-importer.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 4f7c5ca04..5c6a6c5a2 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -151,9 +151,11 @@ class LightningStatsImporter { const alreadyCountedChannels = {}; for (const channel of networkGraph.channels) { - const tx = await fundingTxFetcher.$fetchChannelOpenTx(channel.scid.slice(0, -2)); + const short_id = channel.scid.slice(0, -2); + + const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id); if (!tx) { - logger.err(`Unable to fetch funding tx for channel ${channel.scid}. Capacity and creation date will stay unknown.`); + logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`); continue; } @@ -175,10 +177,10 @@ class LightningStatsImporter { nodeStats[channel.destination].capacity += Math.round(tx.value * 100000000); nodeStats[channel.destination].channels++; - if (!alreadyCountedChannels[channel.scid.slice(0, -2)]) { + if (!alreadyCountedChannels[short_id]) { capacity += Math.round(tx.value * 100000000); capacities.push(Math.round(tx.value * 100000000)); - alreadyCountedChannels[channel.scid.slice(0, -2)] = true; + alreadyCountedChannels[short_id] = true; } avgFeeRate += channel.fee_proportional_millionths; From 5b521cfc7cc9160a1b722c8beb3fadccf3d31576 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 17:56:46 +0200 Subject: [PATCH 24/53] Don't insert gapped gossip data upon restart --- backend/src/tasks/lightning/sync-tasks/stats-importer.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 5c6a6c5a2..f99529e02 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -255,10 +255,10 @@ class LightningStatsImporter { const fileList = await fsPromises.readdir(this.topologiesFolder); fileList.sort().reverse(); - const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) AS added FROM lightning_stats'); + const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) as added, node_count FROM lightning_stats'); const existingStatsTimestamps = {}; for (const row of rows) { - existingStatsTimestamps[row.added] = true; + existingStatsTimestamps[row.added] = rows[0]; } for (const filename of fileList) { @@ -266,6 +266,7 @@ class LightningStatsImporter { // Stats exist already, don't calculate/insert them if (existingStatsTimestamps[timestamp] !== undefined) { + latestNodeCount = existingStatsTimestamps[timestamp].node_count; continue; } From b6ba3c57811693f2deaa86a785eb15ee4fb7b707 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 18:15:34 +0200 Subject: [PATCH 25/53] Ignore channels fee rate > 5000ppm or base fee > 5000 in stats --- .../lightning/sync-tasks/stats-importer.ts | 144 ++++++++++-------- 1 file changed, 77 insertions(+), 67 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index f99529e02..91e67f77d 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -45,70 +45,10 @@ class LightningStatsImporter { await this.$importHistoricalLightningStats(); } - /** - * Parse the file content into XML, and return a list of nodes and channels - */ - parseFile(fileContent): any { - const graph = this.parser.parse(fileContent); - if (Object.keys(graph).length === 0) { - return null; - } - - const nodes: Node[] = []; - const channels: Channel[] = []; - - // If there is only one entry, the parser does not return an array, so we override this - if (!Array.isArray(graph.graphml.graph.node)) { - graph.graphml.graph.node = [graph.graphml.graph.node]; - } - if (!Array.isArray(graph.graphml.graph.edge)) { - graph.graphml.graph.edge = [graph.graphml.graph.edge]; - } - - for (const node of graph.graphml.graph.node) { - if (!node.data) { - continue; - } - nodes.push({ - id: node.data[0], - timestamp: node.data[1], - features: node.data[2], - rgb_color: node.data[3], - alias: node.data[4], - addresses: node.data[5], - out_degree: node.data[6], - in_degree: node.data[7], - }); - } - - for (const channel of graph.graphml.graph.edge) { - if (!channel.data) { - continue; - } - channels.push({ - scid: channel.data[0], - source: channel.data[1], - destination: channel.data[2], - timestamp: channel.data[3], - features: channel.data[4], - fee_base_msat: channel.data[5], - fee_proportional_millionths: channel.data[6], - htlc_minimim_msat: channel.data[7], - cltv_expiry_delta: channel.data[8], - htlc_maximum_msat: channel.data[9], - }); - } - - return { - nodes: nodes, - channels: channels, - }; - } - /** * Generate LN network stats for one day */ - public async computeNetworkStats(timestamp: number, networkGraph): Promise { + public async computeNetworkStats(timestamp: number, networkGraph): Promise { // Node counts and network shares let clearnetNodes = 0; let torNodes = 0; @@ -183,10 +123,15 @@ class LightningStatsImporter { alreadyCountedChannels[short_id] = true; } - avgFeeRate += channel.fee_proportional_millionths; - avgBaseFee += channel.fee_base_msat; - feeRates.push(channel.fee_proportional_millionths); - baseFees.push(channel.fee_base_msat); + if (channel.fee_proportional_millionths < 5000) { + avgFeeRate += channel.fee_proportional_millionths; + feeRates.push(channel.fee_proportional_millionths); + } + + if (channel.fee_base_msat < 5000) { + avgBaseFee += channel.fee_base_msat; + baseFees.push(channel.fee_base_msat); + } } avgFeeRate /= networkGraph.channels.length; @@ -247,6 +192,11 @@ class LightningStatsImporter { nodeStats[public_key].channels, ]); } + + return { + added: timestamp, + node_count: networkGraph.nodes.length + }; } async $importHistoricalLightningStats(): Promise { @@ -308,13 +258,73 @@ class LightningStatsImporter { await fundingTxFetcher.$fetchChannelsFundingTxs(graph.channels.map(channel => channel.scid.slice(0, -2))); logger.debug(`Generating LN network stats for ${datestr}`); - await this.computeNetworkStats(timestamp, graph); + const stat = await this.computeNetworkStats(timestamp, graph); - existingStatsTimestamps[timestamp] = true; + existingStatsTimestamps[timestamp] = stat; } logger.info(`Lightning network stats historical import completed`); } + + /** + * Parse the file content into XML, and return a list of nodes and channels + */ + private parseFile(fileContent): any { + const graph = this.parser.parse(fileContent); + if (Object.keys(graph).length === 0) { + return null; + } + + const nodes: Node[] = []; + const channels: Channel[] = []; + + // If there is only one entry, the parser does not return an array, so we override this + if (!Array.isArray(graph.graphml.graph.node)) { + graph.graphml.graph.node = [graph.graphml.graph.node]; + } + if (!Array.isArray(graph.graphml.graph.edge)) { + graph.graphml.graph.edge = [graph.graphml.graph.edge]; + } + + for (const node of graph.graphml.graph.node) { + if (!node.data) { + continue; + } + nodes.push({ + id: node.data[0], + timestamp: node.data[1], + features: node.data[2], + rgb_color: node.data[3], + alias: node.data[4], + addresses: node.data[5], + out_degree: node.data[6], + in_degree: node.data[7], + }); + } + + for (const channel of graph.graphml.graph.edge) { + if (!channel.data) { + continue; + } + channels.push({ + scid: channel.data[0], + source: channel.data[1], + destination: channel.data[2], + timestamp: channel.data[3], + features: channel.data[4], + fee_base_msat: channel.data[5], + fee_proportional_millionths: channel.data[6], + htlc_minimim_msat: channel.data[7], + cltv_expiry_delta: channel.data[8], + htlc_maximum_msat: channel.data[9], + }); + } + + return { + nodes: nodes, + channels: channels, + }; + } } export default new LightningStatsImporter; \ No newline at end of file From 82cef095fc3ccb35e2858d9caf2d88cc070b7e40 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 18:26:07 +0200 Subject: [PATCH 26/53] Rewrite queries to get top nodes by channels and capacity --- backend/src/api/explorer/nodes.api.ts | 40 +++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 4c7028136..96da7d1d5 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -66,16 +66,19 @@ class NodesApi { public async $getTopCapacityNodes(): Promise { try { + let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); + const latestDate = rows[0].maxAdded; + const query = ` - SELECT IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, nodes.public_key, - CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, - CAST(COALESCE(node_stats.channels, 0) as INT) as channels - FROM nodes - LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key - ORDER BY node_stats.added DESC, node_stats.capacity DESC - LIMIT 10 + SELECT nodes.public_key, nodes.alias, node_stats.capacity, node_stats.channels + FROM node_stats + JOIN nodes ON nodes.public_key = node_stats.public_key + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY capacity DESC + LIMIT 10; `; - const [rows]: any = await DB.query(query); + [rows] = await DB.query(query); + return rows; } catch (e) { logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e)); @@ -85,16 +88,19 @@ class NodesApi { public async $getTopChannelsNodes(): Promise { try { + let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); + const latestDate = rows[0].maxAdded; + const query = ` - SELECT IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, nodes.public_key, - CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, - CAST(COALESCE(node_stats.channels, 0) as INT) as channels - FROM nodes - LEFT JOIN node_stats - ON node_stats.public_key = nodes.public_key - ORDER BY node_stats.added DESC, node_stats.channels DESC - LIMIT 10`; - const [rows]: any = await DB.query(query); + SELECT nodes.public_key, nodes.alias, node_stats.capacity, node_stats.channels + FROM node_stats + JOIN nodes ON nodes.public_key = node_stats.public_key + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY channels DESC + LIMIT 10; + `; + [rows] = await DB.query(query); + return rows; } catch (e) { logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e)); From 3f83e517f05828af08175cb417d8a2d4b87b8cf4 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 29 Jul 2022 08:08:22 +0200 Subject: [PATCH 27/53] Create CLightningClient class --- backend/src/rpc-api/core-lightning/jsonrpc.ts | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 backend/src/rpc-api/core-lightning/jsonrpc.ts diff --git a/backend/src/rpc-api/core-lightning/jsonrpc.ts b/backend/src/rpc-api/core-lightning/jsonrpc.ts new file mode 100644 index 000000000..037dfff75 --- /dev/null +++ b/backend/src/rpc-api/core-lightning/jsonrpc.ts @@ -0,0 +1,249 @@ +'use strict'; + +const methods = [ + 'addgossip', + 'autocleaninvoice', + 'check', + 'checkmessage', + 'close', + 'connect', + 'createinvoice', + 'createinvoicerequest', + 'createoffer', + 'createonion', + 'decode', + 'decodepay', + 'delexpiredinvoice', + 'delinvoice', + 'delpay', + 'dev-listaddrs', + 'dev-rescan-outputs', + 'disableoffer', + 'disconnect', + 'estimatefees', + 'feerates', + 'fetchinvoice', + 'fundchannel', + 'fundchannel_cancel', + 'fundchannel_complete', + 'fundchannel_start', + 'fundpsbt', + 'getchaininfo', + 'getinfo', + 'getlog', + 'getrawblockbyheight', + 'getroute', + 'getsharedsecret', + 'getutxout', + 'help', + 'invoice', + 'keysend', + 'legacypay', + 'listchannels', + 'listconfigs', + 'listforwards', + 'listfunds', + 'listinvoices', + 'listnodes', + 'listoffers', + 'listpays', + 'listpeers', + 'listsendpays', + 'listtransactions', + 'multifundchannel', + 'multiwithdraw', + 'newaddr', + 'notifications', + 'offer', + 'offerout', + 'openchannel_abort', + 'openchannel_bump', + 'openchannel_init', + 'openchannel_signed', + 'openchannel_update', + 'pay', + 'payersign', + 'paystatus', + 'ping', + 'plugin', + 'reserveinputs', + 'sendinvoice', + 'sendonion', + 'sendonionmessage', + 'sendpay', + 'sendpsbt', + 'sendrawtransaction', + 'setchannelfee', + 'signmessage', + 'signpsbt', + 'stop', + 'txdiscard', + 'txprepare', + 'txsend', + 'unreserveinputs', + 'utxopsbt', + 'waitanyinvoice', + 'waitblockheight', + 'waitinvoice', + 'waitsendpay', + 'withdraw' +]; + + +import EventEmitter from 'events'; +import { existsSync, statSync } from 'fs'; +import { createConnection, Socket } from 'net'; +import { homedir } from 'os'; +import path from 'path'; +import { createInterface, Interface } from 'readline'; +import logger from '../../logger'; + +class LightningError extends Error { + type: string = 'lightning'; + message: string = 'lightning-client error'; + + constructor(error) { + super(); + this.type = error.type; + this.message = error.message; + } +} + +const defaultRpcPath = path.join(homedir(), '.lightning') + , fStat = (...p) => statSync(path.join(...p)) + , fExists = (...p) => existsSync(path.join(...p)) + +class CLightningClient extends EventEmitter { + private rpcPath: string; + private reconnectWait: number; + private reconnectTimeout; + private reqcount: number; + private client: Socket; + private rl: Interface; + private clientConnectionPromise: Promise; + + constructor(rpcPath = defaultRpcPath) { + if (!path.isAbsolute(rpcPath)) { + throw new Error('The rpcPath must be an absolute path'); + } + + if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) { + // network directory provided, use the lightning-rpc within in + if (fExists(rpcPath, 'lightning-rpc')) { + rpcPath = path.join(rpcPath, 'lightning-rpc'); + } + + // main data directory provided, default to using the bitcoin mainnet subdirectory + // to be removed in v0.2.0 + else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) { + logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`) + logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`) + rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc') + } + } + + logger.debug(`[CLightningClient] Connecting to ${rpcPath}`); + + super(); + this.rpcPath = rpcPath; + this.reconnectWait = 0.5; + this.reconnectTimeout = null; + this.reqcount = 0; + + const _self = this; + + this.client = createConnection(rpcPath); + this.rl = createInterface({ input: this.client }) + + this.clientConnectionPromise = new Promise(resolve => { + _self.client.on('connect', () => { + logger.debug(`[CLightningClient] Lightning client connected`); + _self.reconnectWait = 1; + resolve(); + }); + + _self.client.on('end', () => { + logger.err('[CLightningClient] Lightning client connection closed, reconnecting'); + _self.increaseWaitTime(); + _self.reconnect(); + }); + + _self.client.on('error', error => { + logger.err(`[CLightningClient] Lightning client connection error: ${error}`); + _self.emit('error', error); + _self.increaseWaitTime(); + _self.reconnect(); + }); + }); + + this.rl.on('line', line => { + line = line.trim(); + if (!line) { + return; + } + const data = JSON.parse(line); + logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); + _self.emit('res:' + data.id, data); + }); + } + + increaseWaitTime(): void { + if (this.reconnectWait >= 16) { + this.reconnectWait = 16; + } else { + this.reconnectWait *= 2; + } + } + + reconnect(): void { + const _self = this; + + if (this.reconnectTimeout) { + return; + } + + this.reconnectTimeout = setTimeout(() => { + logger.debug('[CLightningClient] Trying to reconnect...'); + + _self.client.connect(_self.rpcPath); + _self.reconnectTimeout = null; + }, this.reconnectWait * 1000); + } + + call(method, args = []): Promise { + const _self = this; + + const callInt = ++this.reqcount; + const sendObj = { + jsonrpc: '2.0', + method, + params: args, + id: '' + callInt + }; + + logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`); + + // Wait for the client to connect + return this.clientConnectionPromise + .then(() => new Promise((resolve, reject) => { + // Wait for a response + this.once('res:' + callInt, res => res.error == null + ? resolve(res.result) + : reject(new LightningError(res.error)) + ); + + // Send the command + _self.client.write(JSON.stringify(sendObj)); + })); + } +} + +const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase()); + +methods.forEach(k => { + CLightningClient.prototype[protify(k)] = function (...args: any) { + return this.call(k, args); + }; +}); + +export default new CLightningClient(); From a94403b3a1bd3ec65435d1b4067ad8f90a8bb2f8 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 29 Jul 2022 16:33:07 +0200 Subject: [PATCH 28/53] Wrote some utility functions to convert clightning output to our db schema --- .../lightning/clightning/clightning-client.ts | 4 + .../clightning/clightning-convert.ts | 95 +++++++++++++++++++ .../lightning/clightning}/jsonrpc.ts | 12 +-- backend/src/config.ts | 8 ++ 4 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 backend/src/api/lightning/clightning/clightning-client.ts create mode 100644 backend/src/api/lightning/clightning/clightning-convert.ts rename backend/src/{rpc-api/core-lightning => api/lightning/clightning}/jsonrpc.ts (94%) diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts new file mode 100644 index 000000000..2b974bca0 --- /dev/null +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -0,0 +1,4 @@ +import config from '../../../config'; +import CLightningClient from './jsonrpc'; + +export default new CLightningClient(config.CLIGHTNING.SOCKET); diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts new file mode 100644 index 000000000..34ef6f942 --- /dev/null +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -0,0 +1,95 @@ +import logger from "../../../logger"; +import { ILightningApi } from "../lightning-api.interface"; + +export function convertNode(clNode: any): ILightningApi.Node { + return { + alias: clNode.alias ?? '', + color: `#${clNode.color ?? ''}`, + features: [], // TODO parse and return clNode.feature + public_key: clNode.nodeid, + sockets: clNode.addresses?.map(addr => `${addr.address}:${addr.port}`) ?? [], + updated_at: new Date((clNode?.last_timestamp ?? 0) * 1000).toUTCString(), + }; +} + +export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { + const consolidatedChannelList: ILightningApi.Channel[] = []; + const clChannelsDict = {}; + const clChannelsDictCount = {}; + + for (const clChannel of clChannels) { + if (!clChannelsDict[clChannel.short_channel_id]) { + clChannelsDict[clChannel.short_channel_id] = clChannel; + clChannelsDictCount[clChannel.short_channel_id] = 1; + } else { + consolidatedChannelList.push( + buildBidirectionalChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) + ); + delete clChannelsDict[clChannel.short_channel_id]; + clChannelsDictCount[clChannel.short_channel_id]++; + } + } + const bidirectionalChannelsCount = consolidatedChannelList.length; + + for (const short_channel_id of Object.keys(clChannelsDict)) { + consolidatedChannelList.push(buildUnidirectionalChannel(clChannelsDict[short_channel_id])); + } + const unidirectionalChannelsCount = consolidatedChannelList.length - bidirectionalChannelsCount; + + logger.debug(`clightning knows ${clChannels.length} channels. ` + + `We found ${bidirectionalChannelsCount} bidirectional channels ` + + `and ${unidirectionalChannelsCount} unidirectional channels.`); + + return consolidatedChannelList; +} + +function buildBidirectionalChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { + const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0); + + return { + id: clChannelA.short_channel_id, + capacity: clChannelA.satoshis, + transaction_id: '', // TODO + transaction_vout: 0, // TODO + updated_at: new Date(lastUpdate * 1000).toUTCString(), + policies: [ + convertPolicy(clChannelA), + convertPolicy(clChannelB) + ] + }; +} + +function buildUnidirectionalChannel(clChannel: any): ILightningApi.Channel { + return { + id: clChannel.short_channel_id, + capacity: clChannel.satoshis, + policies: [convertPolicy(clChannel), getEmptyPolicy()], + transaction_id: '', // TODO + transaction_vout: 0, // TODO + updated_at: new Date((clChannel.last_update ?? 0) * 1000).toUTCString(), + }; +} + +function convertPolicy(clChannel: any): ILightningApi.Policy { + return { + public_key: clChannel.source, + base_fee_mtokens: clChannel.base_fee_millisatoshi, + fee_rate: clChannel.fee_per_millionth, + is_disabled: !clChannel.active, + max_htlc_mtokens: clChannel.htlc_maximum_msat.slice(0, -4), + min_htlc_mtokens: clChannel.htlc_minimum_msat.slice(0, -4), + updated_at: new Date((clChannel.last_update ?? 0) * 1000).toUTCString(), + }; +} + +function getEmptyPolicy(): ILightningApi.Policy { + return { + public_key: 'null', + base_fee_mtokens: '0', + fee_rate: 0, + is_disabled: true, + max_htlc_mtokens: '0', + min_htlc_mtokens: '0', + updated_at: new Date(0).toUTCString(), + }; +} diff --git a/backend/src/rpc-api/core-lightning/jsonrpc.ts b/backend/src/api/lightning/clightning/jsonrpc.ts similarity index 94% rename from backend/src/rpc-api/core-lightning/jsonrpc.ts rename to backend/src/api/lightning/clightning/jsonrpc.ts index 037dfff75..d0b187a54 100644 --- a/backend/src/rpc-api/core-lightning/jsonrpc.ts +++ b/backend/src/api/lightning/clightning/jsonrpc.ts @@ -1,3 +1,5 @@ +// Imported from https://github.com/shesek/lightning-client-js + 'use strict'; const methods = [ @@ -96,7 +98,7 @@ import { createConnection, Socket } from 'net'; import { homedir } from 'os'; import path from 'path'; import { createInterface, Interface } from 'readline'; -import logger from '../../logger'; +import logger from '../../../logger'; class LightningError extends Error { type: string = 'lightning'; @@ -113,7 +115,7 @@ const defaultRpcPath = path.join(homedir(), '.lightning') , fStat = (...p) => statSync(path.join(...p)) , fExists = (...p) => existsSync(path.join(...p)) -class CLightningClient extends EventEmitter { +export default class CLightningClient extends EventEmitter { private rpcPath: string; private reconnectWait: number; private reconnectTimeout; @@ -182,7 +184,7 @@ class CLightningClient extends EventEmitter { return; } const data = JSON.parse(line); - logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); + // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); _self.emit('res:' + data.id, data); }); } @@ -210,7 +212,7 @@ class CLightningClient extends EventEmitter { }, this.reconnectWait * 1000); } - call(method, args = []): Promise { + call(method, args = []): Promise { const _self = this; const callInt = ++this.reqcount; @@ -245,5 +247,3 @@ methods.forEach(k => { return this.call(k, args); }; }); - -export default new CLightningClient(); diff --git a/backend/src/config.ts b/backend/src/config.ts index d480e6c51..b42a45ab2 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,6 +38,9 @@ interface IConfig { MACAROON_PATH: string; REST_API_URL: string; }; + CLIGHTNING: { + SOCKET: string; + }; ELECTRUM: { HOST: string; PORT: number; @@ -186,6 +189,9 @@ const defaults: IConfig = { 'MACAROON_PATH': '', 'REST_API_URL': 'https://localhost:8080', }, + 'CLIGHTNING': { + 'SOCKET': '', + }, 'SOCKS5PROXY': { 'ENABLED': false, 'USE_ONION': true, @@ -226,6 +232,7 @@ class Config implements IConfig { BISQ: IConfig['BISQ']; LIGHTNING: IConfig['LIGHTNING']; LND: IConfig['LND']; + CLIGHTNING: IConfig['CLIGHTNING']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; @@ -244,6 +251,7 @@ class Config implements IConfig { this.BISQ = configs.BISQ; this.LIGHTNING = configs.LIGHTNING; this.LND = configs.LND; + this.CLIGHTNING = configs.CLIGHTNING; this.SOCKS5PROXY = configs.SOCKS5PROXY; this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; From eb90434c28f7d19b1e172226438291195f8105f0 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 29 Jul 2022 17:41:09 +0200 Subject: [PATCH 29/53] Delete historical generation code --- .../lightning/clightning/clightning-client.ts | 265 +++++++++++++++++- .../clightning/clightning-convert.ts | 44 +-- .../src/api/lightning/clightning/jsonrpc.ts | 249 ---------------- .../lightning-api-abstract-factory.ts | 2 - .../api/lightning/lightning-api-factory.ts | 5 +- .../src/tasks/lightning/node-sync.service.ts | 2 +- 6 files changed, 295 insertions(+), 272 deletions(-) delete mode 100644 backend/src/api/lightning/clightning/jsonrpc.ts diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts index 2b974bca0..629092d03 100644 --- a/backend/src/api/lightning/clightning/clightning-client.ts +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -1,4 +1,263 @@ -import config from '../../../config'; -import CLightningClient from './jsonrpc'; +// Imported from https://github.com/shesek/lightning-client-js -export default new CLightningClient(config.CLIGHTNING.SOCKET); +'use strict'; + +const methods = [ + 'addgossip', + 'autocleaninvoice', + 'check', + 'checkmessage', + 'close', + 'connect', + 'createinvoice', + 'createinvoicerequest', + 'createoffer', + 'createonion', + 'decode', + 'decodepay', + 'delexpiredinvoice', + 'delinvoice', + 'delpay', + 'dev-listaddrs', + 'dev-rescan-outputs', + 'disableoffer', + 'disconnect', + 'estimatefees', + 'feerates', + 'fetchinvoice', + 'fundchannel', + 'fundchannel_cancel', + 'fundchannel_complete', + 'fundchannel_start', + 'fundpsbt', + 'getchaininfo', + 'getinfo', + 'getlog', + 'getrawblockbyheight', + 'getroute', + 'getsharedsecret', + 'getutxout', + 'help', + 'invoice', + 'keysend', + 'legacypay', + 'listchannels', + 'listconfigs', + 'listforwards', + 'listfunds', + 'listinvoices', + 'listnodes', + 'listoffers', + 'listpays', + 'listpeers', + 'listsendpays', + 'listtransactions', + 'multifundchannel', + 'multiwithdraw', + 'newaddr', + 'notifications', + 'offer', + 'offerout', + 'openchannel_abort', + 'openchannel_bump', + 'openchannel_init', + 'openchannel_signed', + 'openchannel_update', + 'pay', + 'payersign', + 'paystatus', + 'ping', + 'plugin', + 'reserveinputs', + 'sendinvoice', + 'sendonion', + 'sendonionmessage', + 'sendpay', + 'sendpsbt', + 'sendrawtransaction', + 'setchannelfee', + 'signmessage', + 'signpsbt', + 'stop', + 'txdiscard', + 'txprepare', + 'txsend', + 'unreserveinputs', + 'utxopsbt', + 'waitanyinvoice', + 'waitblockheight', + 'waitinvoice', + 'waitsendpay', + 'withdraw' +]; + + +import EventEmitter from 'events'; +import { existsSync, statSync } from 'fs'; +import { createConnection, Socket } from 'net'; +import { homedir } from 'os'; +import path from 'path'; +import { createInterface, Interface } from 'readline'; +import logger from '../../../logger'; +import { AbstractLightningApi } from '../lightning-api-abstract-factory'; +import { ILightningApi } from '../lightning-api.interface'; +import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert'; + +class LightningError extends Error { + type: string = 'lightning'; + message: string = 'lightning-client error'; + + constructor(error) { + super(); + this.type = error.type; + this.message = error.message; + } +} + +const defaultRpcPath = path.join(homedir(), '.lightning') + , fStat = (...p) => statSync(path.join(...p)) + , fExists = (...p) => existsSync(path.join(...p)) + +export default class CLightningClient extends EventEmitter implements AbstractLightningApi { + private rpcPath: string; + private reconnectWait: number; + private reconnectTimeout; + private reqcount: number; + private client: Socket; + private rl: Interface; + private clientConnectionPromise: Promise; + + constructor(rpcPath = defaultRpcPath) { + if (!path.isAbsolute(rpcPath)) { + throw new Error('The rpcPath must be an absolute path'); + } + + if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) { + // network directory provided, use the lightning-rpc within in + if (fExists(rpcPath, 'lightning-rpc')) { + rpcPath = path.join(rpcPath, 'lightning-rpc'); + } + + // main data directory provided, default to using the bitcoin mainnet subdirectory + // to be removed in v0.2.0 + else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) { + logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`) + logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`) + rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc') + } + } + + logger.debug(`[CLightningClient] Connecting to ${rpcPath}`); + + super(); + this.rpcPath = rpcPath; + this.reconnectWait = 0.5; + this.reconnectTimeout = null; + this.reqcount = 0; + + const _self = this; + + this.client = createConnection(rpcPath); + this.rl = createInterface({ input: this.client }) + + this.clientConnectionPromise = new Promise(resolve => { + _self.client.on('connect', () => { + logger.info(`[CLightningClient] Lightning client connected`); + _self.reconnectWait = 1; + resolve(); + }); + + _self.client.on('end', () => { + logger.err('[CLightningClient] Lightning client connection closed, reconnecting'); + _self.increaseWaitTime(); + _self.reconnect(); + }); + + _self.client.on('error', error => { + logger.err(`[CLightningClient] Lightning client connection error: ${error}`); + _self.emit('error', error); + _self.increaseWaitTime(); + _self.reconnect(); + }); + }); + + this.rl.on('line', line => { + line = line.trim(); + if (!line) { + return; + } + const data = JSON.parse(line); + // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); + _self.emit('res:' + data.id, data); + }); + } + + increaseWaitTime(): void { + if (this.reconnectWait >= 16) { + this.reconnectWait = 16; + } else { + this.reconnectWait *= 2; + } + } + + reconnect(): void { + const _self = this; + + if (this.reconnectTimeout) { + return; + } + + this.reconnectTimeout = setTimeout(() => { + logger.debug('[CLightningClient] Trying to reconnect...'); + + _self.client.connect(_self.rpcPath); + _self.reconnectTimeout = null; + }, this.reconnectWait * 1000); + } + + call(method, args = []): Promise { + const _self = this; + + const callInt = ++this.reqcount; + const sendObj = { + jsonrpc: '2.0', + method, + params: args, + id: '' + callInt + }; + + // logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`); + + // Wait for the client to connect + return this.clientConnectionPromise + .then(() => new Promise((resolve, reject) => { + // Wait for a response + this.once('res:' + callInt, res => res.error == null + ? resolve(res.result) + : reject(new LightningError(res.error)) + ); + + // Send the command + _self.client.write(JSON.stringify(sendObj)); + })); + } + + async $getNetworkGraph(): Promise { + const listnodes: any[] = await this.call('listnodes'); + const listchannels: any[] = await this.call('listchannels'); + const channelsList = convertAndmergeBidirectionalChannels(listchannels['channels']); + + return { + nodes: listnodes['nodes'].map(node => convertNode(node)), + channels: channelsList, + }; + } +} + +const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase()); + +methods.forEach(k => { + CLightningClient.prototype[protify(k)] = function (...args: any) { + return this.call(k, args); + }; +}); diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 34ef6f942..8ceec3b7e 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -1,6 +1,8 @@ -import logger from "../../../logger"; -import { ILightningApi } from "../lightning-api.interface"; +import { ILightningApi } from '../lightning-api.interface'; +/** + * Convert a clightning "listnode" entry to a lnd node entry + */ export function convertNode(clNode: any): ILightningApi.Node { return { alias: clNode.alias ?? '', @@ -12,7 +14,10 @@ export function convertNode(clNode: any): ILightningApi.Node { }; } -export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { +/** + * Convert clightning "listchannels" response to lnd "describegraph.channels" format + */ + export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { const consolidatedChannelList: ILightningApi.Channel[] = []; const clChannelsDict = {}; const clChannelsDictCount = {}; @@ -23,27 +28,24 @@ export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightn clChannelsDictCount[clChannel.short_channel_id] = 1; } else { consolidatedChannelList.push( - buildBidirectionalChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) + buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) ); delete clChannelsDict[clChannel.short_channel_id]; clChannelsDictCount[clChannel.short_channel_id]++; } } - const bidirectionalChannelsCount = consolidatedChannelList.length; - for (const short_channel_id of Object.keys(clChannelsDict)) { - consolidatedChannelList.push(buildUnidirectionalChannel(clChannelsDict[short_channel_id])); + consolidatedChannelList.push(buildIncompleteChannel(clChannelsDict[short_channel_id])); } - const unidirectionalChannelsCount = consolidatedChannelList.length - bidirectionalChannelsCount; - - logger.debug(`clightning knows ${clChannels.length} channels. ` + - `We found ${bidirectionalChannelsCount} bidirectional channels ` + - `and ${unidirectionalChannelsCount} unidirectional channels.`); return consolidatedChannelList; } -function buildBidirectionalChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { +/** + * Convert two clightning "getchannels" entries into a full a lnd "describegraph.channels" format + * In this case, clightning knows the channel policy for both nodes + */ +function buildFullChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0); return { @@ -59,7 +61,11 @@ function buildBidirectionalChannel(clChannelA: any, clChannelB: any): ILightning }; } -function buildUnidirectionalChannel(clChannel: any): ILightningApi.Channel { +/** + * Convert one clightning "getchannels" entry into a full a lnd "describegraph.channels" format + * In this case, clightning knows the channel policy of only one node + */ + function buildIncompleteChannel(clChannel: any): ILightningApi.Channel { return { id: clChannel.short_channel_id, capacity: clChannel.satoshis, @@ -70,7 +76,10 @@ function buildUnidirectionalChannel(clChannel: any): ILightningApi.Channel { }; } -function convertPolicy(clChannel: any): ILightningApi.Policy { +/** + * Convert a clightning "listnode" response to a lnd channel policy format + */ + function convertPolicy(clChannel: any): ILightningApi.Policy { return { public_key: clChannel.source, base_fee_mtokens: clChannel.base_fee_millisatoshi, @@ -82,7 +91,10 @@ function convertPolicy(clChannel: any): ILightningApi.Policy { }; } -function getEmptyPolicy(): ILightningApi.Policy { +/** + * Create an empty channel policy in lnd format + */ + function getEmptyPolicy(): ILightningApi.Policy { return { public_key: 'null', base_fee_mtokens: '0', diff --git a/backend/src/api/lightning/clightning/jsonrpc.ts b/backend/src/api/lightning/clightning/jsonrpc.ts deleted file mode 100644 index d0b187a54..000000000 --- a/backend/src/api/lightning/clightning/jsonrpc.ts +++ /dev/null @@ -1,249 +0,0 @@ -// Imported from https://github.com/shesek/lightning-client-js - -'use strict'; - -const methods = [ - 'addgossip', - 'autocleaninvoice', - 'check', - 'checkmessage', - 'close', - 'connect', - 'createinvoice', - 'createinvoicerequest', - 'createoffer', - 'createonion', - 'decode', - 'decodepay', - 'delexpiredinvoice', - 'delinvoice', - 'delpay', - 'dev-listaddrs', - 'dev-rescan-outputs', - 'disableoffer', - 'disconnect', - 'estimatefees', - 'feerates', - 'fetchinvoice', - 'fundchannel', - 'fundchannel_cancel', - 'fundchannel_complete', - 'fundchannel_start', - 'fundpsbt', - 'getchaininfo', - 'getinfo', - 'getlog', - 'getrawblockbyheight', - 'getroute', - 'getsharedsecret', - 'getutxout', - 'help', - 'invoice', - 'keysend', - 'legacypay', - 'listchannels', - 'listconfigs', - 'listforwards', - 'listfunds', - 'listinvoices', - 'listnodes', - 'listoffers', - 'listpays', - 'listpeers', - 'listsendpays', - 'listtransactions', - 'multifundchannel', - 'multiwithdraw', - 'newaddr', - 'notifications', - 'offer', - 'offerout', - 'openchannel_abort', - 'openchannel_bump', - 'openchannel_init', - 'openchannel_signed', - 'openchannel_update', - 'pay', - 'payersign', - 'paystatus', - 'ping', - 'plugin', - 'reserveinputs', - 'sendinvoice', - 'sendonion', - 'sendonionmessage', - 'sendpay', - 'sendpsbt', - 'sendrawtransaction', - 'setchannelfee', - 'signmessage', - 'signpsbt', - 'stop', - 'txdiscard', - 'txprepare', - 'txsend', - 'unreserveinputs', - 'utxopsbt', - 'waitanyinvoice', - 'waitblockheight', - 'waitinvoice', - 'waitsendpay', - 'withdraw' -]; - - -import EventEmitter from 'events'; -import { existsSync, statSync } from 'fs'; -import { createConnection, Socket } from 'net'; -import { homedir } from 'os'; -import path from 'path'; -import { createInterface, Interface } from 'readline'; -import logger from '../../../logger'; - -class LightningError extends Error { - type: string = 'lightning'; - message: string = 'lightning-client error'; - - constructor(error) { - super(); - this.type = error.type; - this.message = error.message; - } -} - -const defaultRpcPath = path.join(homedir(), '.lightning') - , fStat = (...p) => statSync(path.join(...p)) - , fExists = (...p) => existsSync(path.join(...p)) - -export default class CLightningClient extends EventEmitter { - private rpcPath: string; - private reconnectWait: number; - private reconnectTimeout; - private reqcount: number; - private client: Socket; - private rl: Interface; - private clientConnectionPromise: Promise; - - constructor(rpcPath = defaultRpcPath) { - if (!path.isAbsolute(rpcPath)) { - throw new Error('The rpcPath must be an absolute path'); - } - - if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) { - // network directory provided, use the lightning-rpc within in - if (fExists(rpcPath, 'lightning-rpc')) { - rpcPath = path.join(rpcPath, 'lightning-rpc'); - } - - // main data directory provided, default to using the bitcoin mainnet subdirectory - // to be removed in v0.2.0 - else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) { - logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`) - logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`) - rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc') - } - } - - logger.debug(`[CLightningClient] Connecting to ${rpcPath}`); - - super(); - this.rpcPath = rpcPath; - this.reconnectWait = 0.5; - this.reconnectTimeout = null; - this.reqcount = 0; - - const _self = this; - - this.client = createConnection(rpcPath); - this.rl = createInterface({ input: this.client }) - - this.clientConnectionPromise = new Promise(resolve => { - _self.client.on('connect', () => { - logger.debug(`[CLightningClient] Lightning client connected`); - _self.reconnectWait = 1; - resolve(); - }); - - _self.client.on('end', () => { - logger.err('[CLightningClient] Lightning client connection closed, reconnecting'); - _self.increaseWaitTime(); - _self.reconnect(); - }); - - _self.client.on('error', error => { - logger.err(`[CLightningClient] Lightning client connection error: ${error}`); - _self.emit('error', error); - _self.increaseWaitTime(); - _self.reconnect(); - }); - }); - - this.rl.on('line', line => { - line = line.trim(); - if (!line) { - return; - } - const data = JSON.parse(line); - // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); - _self.emit('res:' + data.id, data); - }); - } - - increaseWaitTime(): void { - if (this.reconnectWait >= 16) { - this.reconnectWait = 16; - } else { - this.reconnectWait *= 2; - } - } - - reconnect(): void { - const _self = this; - - if (this.reconnectTimeout) { - return; - } - - this.reconnectTimeout = setTimeout(() => { - logger.debug('[CLightningClient] Trying to reconnect...'); - - _self.client.connect(_self.rpcPath); - _self.reconnectTimeout = null; - }, this.reconnectWait * 1000); - } - - call(method, args = []): Promise { - const _self = this; - - const callInt = ++this.reqcount; - const sendObj = { - jsonrpc: '2.0', - method, - params: args, - id: '' + callInt - }; - - logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`); - - // Wait for the client to connect - return this.clientConnectionPromise - .then(() => new Promise((resolve, reject) => { - // Wait for a response - this.once('res:' + callInt, res => res.error == null - ? resolve(res.result) - : reject(new LightningError(res.error)) - ); - - // Send the command - _self.client.write(JSON.stringify(sendObj)); - })); - } -} - -const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase()); - -methods.forEach(k => { - CLightningClient.prototype[protify(k)] = function (...args: any) { - return this.call(k, args); - }; -}); diff --git a/backend/src/api/lightning/lightning-api-abstract-factory.ts b/backend/src/api/lightning/lightning-api-abstract-factory.ts index 026568c6d..e6691b0a4 100644 --- a/backend/src/api/lightning/lightning-api-abstract-factory.ts +++ b/backend/src/api/lightning/lightning-api-abstract-factory.ts @@ -1,7 +1,5 @@ import { ILightningApi } from './lightning-api.interface'; export interface AbstractLightningApi { - $getNetworkInfo(): Promise; $getNetworkGraph(): Promise; - $getInfo(): Promise; } diff --git a/backend/src/api/lightning/lightning-api-factory.ts b/backend/src/api/lightning/lightning-api-factory.ts index ab551095c..fdadd8230 100644 --- a/backend/src/api/lightning/lightning-api-factory.ts +++ b/backend/src/api/lightning/lightning-api-factory.ts @@ -1,9 +1,12 @@ import config from '../../config'; +import CLightningClient from './clightning/clightning-client'; import { AbstractLightningApi } from './lightning-api-abstract-factory'; import LndApi from './lnd/lnd-api'; function lightningApiFactory(): AbstractLightningApi { - switch (config.LIGHTNING.BACKEND) { + switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) { + case 'cln': + return new CLightningClient(config.CLIGHTNING.SOCKET); case 'lnd': default: return new LndApi(); diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index 10cd2d744..d3367d51c 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -5,9 +5,9 @@ import bitcoinClient from '../../api/bitcoin/bitcoin-client'; import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory'; import config from '../../config'; import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; -import lightningApi from '../../api/lightning/lightning-api-factory'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { $lookupNodeLocation } from './sync-tasks/node-locations'; +import lightningApi from '../../api/lightning/lightning-api-factory'; class NodeSyncService { constructor() {} From 80f1ee45b5b8a0198fa572a69effef09e4d4fc95 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 1 Aug 2022 19:42:33 +0200 Subject: [PATCH 30/53] Rebased using the update lightning interfaces --- .../lightning/clightning/clightning-client.ts | 2 +- .../clightning/clightning-convert.ts | 80 +++++++++---------- .../src/tasks/lightning/node-sync.service.ts | 24 ++++-- 3 files changed, 57 insertions(+), 49 deletions(-) diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts index 629092d03..f5643ed01 100644 --- a/backend/src/api/lightning/clightning/clightning-client.ts +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -249,7 +249,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi return { nodes: listnodes['nodes'].map(node => convertNode(node)), - channels: channelsList, + edges: channelsList, }; } } diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 8ceec3b7e..008094bf5 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -8,14 +8,19 @@ export function convertNode(clNode: any): ILightningApi.Node { alias: clNode.alias ?? '', color: `#${clNode.color ?? ''}`, features: [], // TODO parse and return clNode.feature - public_key: clNode.nodeid, - sockets: clNode.addresses?.map(addr => `${addr.address}:${addr.port}`) ?? [], - updated_at: new Date((clNode?.last_timestamp ?? 0) * 1000).toUTCString(), + pub_key: clNode.nodeid, + addresses: clNode.addresses?.map((addr) => { + return { + network: addr.type, + addr: `${addr.address}:${addr.port}` + }; + }), + last_update: clNode?.last_timestamp ?? 0, }; } /** - * Convert clightning "listchannels" response to lnd "describegraph.channels" format + * Convert clightning "listchannels" response to lnd "describegraph.edges" format */ export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { const consolidatedChannelList: ILightningApi.Channel[] = []; @@ -41,67 +46,58 @@ export function convertNode(clNode: any): ILightningApi.Node { return consolidatedChannelList; } +export function convertChannelId(channelId): string { + const s = channelId.split('x').map(part => parseInt(part)); + return BigInt((s[0] << 40) | (s[1] << 16) | s[2]).toString(); +} + /** - * Convert two clightning "getchannels" entries into a full a lnd "describegraph.channels" format + * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format * In this case, clightning knows the channel policy for both nodes */ function buildFullChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0); return { - id: clChannelA.short_channel_id, + channel_id: clChannelA.short_channel_id, capacity: clChannelA.satoshis, - transaction_id: '', // TODO - transaction_vout: 0, // TODO - updated_at: new Date(lastUpdate * 1000).toUTCString(), - policies: [ - convertPolicy(clChannelA), - convertPolicy(clChannelB) - ] + last_update: lastUpdate, + node1_policy: convertPolicy(clChannelA), + node2_policy: convertPolicy(clChannelB), + chan_point: ':0', // TODO + node1_pub: clChannelA.source, + node2_pub: clChannelB.source, }; } /** - * Convert one clightning "getchannels" entry into a full a lnd "describegraph.channels" format + * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format * In this case, clightning knows the channel policy of only one node */ function buildIncompleteChannel(clChannel: any): ILightningApi.Channel { return { - id: clChannel.short_channel_id, + channel_id: clChannel.short_channel_id, capacity: clChannel.satoshis, - policies: [convertPolicy(clChannel), getEmptyPolicy()], - transaction_id: '', // TODO - transaction_vout: 0, // TODO - updated_at: new Date((clChannel.last_update ?? 0) * 1000).toUTCString(), + last_update: clChannel.last_update ?? 0, + node1_policy: convertPolicy(clChannel), + node2_policy: null, + chan_point: ':0', // TODO + node1_pub: clChannel.source, + node2_pub: clChannel.destination, }; } /** * Convert a clightning "listnode" response to a lnd channel policy format */ - function convertPolicy(clChannel: any): ILightningApi.Policy { + function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy { return { - public_key: clChannel.source, - base_fee_mtokens: clChannel.base_fee_millisatoshi, - fee_rate: clChannel.fee_per_millionth, - is_disabled: !clChannel.active, - max_htlc_mtokens: clChannel.htlc_maximum_msat.slice(0, -4), - min_htlc_mtokens: clChannel.htlc_minimum_msat.slice(0, -4), - updated_at: new Date((clChannel.last_update ?? 0) * 1000).toUTCString(), - }; -} - -/** - * Create an empty channel policy in lnd format - */ - function getEmptyPolicy(): ILightningApi.Policy { - return { - public_key: 'null', - base_fee_mtokens: '0', - fee_rate: 0, - is_disabled: true, - max_htlc_mtokens: '0', - min_htlc_mtokens: '0', - updated_at: new Date(0).toUTCString(), + time_lock_delta: 0, // TODO + min_htlc: clChannel.htlc_minimum_msat.slice(0, -4), + max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4), + fee_base_msat: clChannel.base_fee_millisatoshi, + fee_rate_milli_msat: clChannel.fee_per_millionth, + disabled: !clChannel.active, + last_update: clChannel.last_update ?? 0, }; } diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index d3367d51c..863ee30da 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -8,6 +8,7 @@ import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { $lookupNodeLocation } from './sync-tasks/node-locations'; import lightningApi from '../../api/lightning/lightning-api-factory'; +import { convertChannelId } from '../../api/lightning/clightning/clightning-convert'; class NodeSyncService { constructor() {} @@ -320,7 +321,7 @@ class NodeSyncService { ;`; await DB.query(query, [ - channel.channel_id, + this.toIntegerId(channel.channel_id), this.toShortId(channel.channel_id), channel.capacity, txid, @@ -391,8 +392,7 @@ class NodeSyncService { private async $saveNode(node: ILightningApi.Node): Promise { try { - const updatedAt = this.utcDateToMysql(node.last_update); - const sockets = node.addresses.map(a => a.addr).join(','); + const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? ''; const query = `INSERT INTO nodes( public_key, first_seen, @@ -401,15 +401,16 @@ class NodeSyncService { color, sockets ) - VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`; + VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?) + ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?`; await DB.query(query, [ node.pub_key, - updatedAt, + node.last_update, node.alias, node.color, sockets, - updatedAt, + node.last_update, node.alias, node.color, sockets, @@ -419,8 +420,19 @@ class NodeSyncService { } } + private toIntegerId(id: string): string { + if (config.LIGHTNING.BACKEND === 'lnd') { + return id; + } + return convertChannelId(id); + } + /** Decodes a channel id returned by lnd as uint64 to a short channel id */ private toShortId(id: string): string { + if (config.LIGHTNING.BACKEND === 'cln') { + return id; + } + const n = BigInt(id); return [ n >> 40n, // nth block From 00cd3ee9bf6c74487b30ebb7ec092495059b4198 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 16:18:19 +0200 Subject: [PATCH 31/53] Don't run the ln network update if the graph is emtpy --- backend/src/index.ts | 6 +++--- ...node-sync.service.ts => network-sync.service.ts} | 13 +++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) rename backend/src/tasks/lightning/{node-sync.service.ts => network-sync.service.ts} (97%) diff --git a/backend/src/index.ts b/backend/src/index.ts index fa80fb2ad..0f7cc7aa7 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -28,7 +28,7 @@ import nodesRoutes from './api/explorer/nodes.routes'; import channelsRoutes from './api/explorer/channels.routes'; import generalLightningRoutes from './api/explorer/general.routes'; import lightningStatsUpdater from './tasks/lightning/stats-updater.service'; -import nodeSyncService from './tasks/lightning/node-sync.service'; +import networkSyncService from './tasks/lightning/network-sync.service'; import statisticsRoutes from './api/statistics/statistics.routes'; import miningRoutes from './api/mining/mining-routes'; import bisqRoutes from './api/bisq/bisq.routes'; @@ -136,8 +136,8 @@ class Server { } if (config.LIGHTNING.ENABLED) { - nodeSyncService.$startService() - .then(() => lightningStatsUpdater.$startService()); + networkSyncService.$startService() + .then(() => lightningStatsUpdater.$startService()); } this.server.listen(config.MEMPOOL.HTTP_PORT, () => { diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts similarity index 97% rename from backend/src/tasks/lightning/node-sync.service.ts rename to backend/src/tasks/lightning/network-sync.service.ts index 863ee30da..826664cf4 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -10,7 +10,7 @@ import { $lookupNodeLocation } from './sync-tasks/node-locations'; import lightningApi from '../../api/lightning/lightning-api-factory'; import { convertChannelId } from '../../api/lightning/clightning/clightning-convert'; -class NodeSyncService { +class NetworkSyncService { constructor() {} public async $startService() { @@ -28,6 +28,11 @@ class NodeSyncService { logger.info(`Updating nodes and channels...`); const networkGraph = await lightningApi.$getNetworkGraph(); + if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) { + logger.info(`LN Network graph is empty, retrying in 10 seconds`); + setTimeout(this.$runUpdater, 10000); + return; + } for (const node of networkGraph.nodes) { await this.$saveNode(node); @@ -376,6 +381,10 @@ class NodeSyncService { } private async $setChannelsInactive(graphChannelsIds: string[]): Promise { + if (graphChannelsIds.length === 0) { + return; + } + try { await DB.query(` UPDATE channels @@ -447,4 +456,4 @@ class NodeSyncService { } } -export default new NodeSyncService(); +export default new NetworkSyncService(); From a25af16f7c0557d1144d035344ee060c56dc04a7 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 16:39:34 +0200 Subject: [PATCH 32/53] Fetch funding tx for clightning channels --- .../lightning/clightning/clightning-client.ts | 2 +- .../clightning/clightning-convert.ts | 25 +++++++++++++------ backend/src/index.ts | 4 ++- .../sync-tasks/funding-tx-fetcher.ts | 16 ++++++------ 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts index f5643ed01..15f472f2e 100644 --- a/backend/src/api/lightning/clightning/clightning-client.ts +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -245,7 +245,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi async $getNetworkGraph(): Promise { const listnodes: any[] = await this.call('listnodes'); const listchannels: any[] = await this.call('listchannels'); - const channelsList = convertAndmergeBidirectionalChannels(listchannels['channels']); + const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']); return { nodes: listnodes['nodes'].map(node => convertNode(node)), diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 008094bf5..1a267bc65 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -1,4 +1,5 @@ import { ILightningApi } from '../lightning-api.interface'; +import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher'; /** * Convert a clightning "listnode" entry to a lnd node entry @@ -22,7 +23,7 @@ export function convertNode(clNode: any): ILightningApi.Node { /** * Convert clightning "listchannels" response to lnd "describegraph.edges" format */ - export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { + export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise { const consolidatedChannelList: ILightningApi.Channel[] = []; const clChannelsDict = {}; const clChannelsDictCount = {}; @@ -33,14 +34,14 @@ export function convertNode(clNode: any): ILightningApi.Node { clChannelsDictCount[clChannel.short_channel_id] = 1; } else { consolidatedChannelList.push( - buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) + await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) ); delete clChannelsDict[clChannel.short_channel_id]; clChannelsDictCount[clChannel.short_channel_id]++; } } for (const short_channel_id of Object.keys(clChannelsDict)) { - consolidatedChannelList.push(buildIncompleteChannel(clChannelsDict[short_channel_id])); + consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id])); } return consolidatedChannelList; @@ -55,16 +56,20 @@ export function convertChannelId(channelId): string { * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format * In this case, clightning knows the channel policy for both nodes */ -function buildFullChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { +async function buildFullChannel(clChannelA: any, clChannelB: any): Promise { const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0); - + + const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id); + const parts = clChannelA.short_channel_id.split('x'); + const outputIdx = parts[2]; + return { channel_id: clChannelA.short_channel_id, capacity: clChannelA.satoshis, last_update: lastUpdate, node1_policy: convertPolicy(clChannelA), node2_policy: convertPolicy(clChannelB), - chan_point: ':0', // TODO + chan_point: `${tx.txid}:${outputIdx}`, node1_pub: clChannelA.source, node2_pub: clChannelB.source, }; @@ -74,14 +79,18 @@ function buildFullChannel(clChannelA: any, clChannelB: any): ILightningApi.Chann * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format * In this case, clightning knows the channel policy of only one node */ - function buildIncompleteChannel(clChannel: any): ILightningApi.Channel { + async function buildIncompleteChannel(clChannel: any): Promise { + const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id); + const parts = clChannel.short_channel_id.split('x'); + const outputIdx = parts[2]; + return { channel_id: clChannel.short_channel_id, capacity: clChannel.satoshis, last_update: clChannel.last_update ?? 0, node1_policy: convertPolicy(clChannel), node2_policy: null, - chan_point: ':0', // TODO + chan_point: `${tx.txid}:${outputIdx}`, node1_pub: clChannel.source, node2_pub: clChannel.destination, }; diff --git a/backend/src/index.ts b/backend/src/index.ts index 0f7cc7aa7..976ec12df 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -34,6 +34,7 @@ import miningRoutes from './api/mining/mining-routes'; import bisqRoutes from './api/bisq/bisq.routes'; import liquidRoutes from './api/liquid/liquid.routes'; import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; +import fundingTxFetcher from "./tasks/lightning/sync-tasks/funding-tx-fetcher"; class Server { private wss: WebSocket.Server | undefined; @@ -136,7 +137,8 @@ class Server { } if (config.LIGHTNING.ENABLED) { - networkSyncService.$startService() + fundingTxFetcher.$init() + .then(() => networkSyncService.$startService()) .then(() => lightningStatsUpdater.$startService()); } diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts index 9da721876..926d20c91 100644 --- a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -1,8 +1,6 @@ import { existsSync, promises } from 'fs'; -import bitcoinApiFactory from '../../../api/bitcoin/bitcoin-api-factory'; import bitcoinClient from '../../../api/bitcoin/bitcoin-client'; import config from '../../../config'; -import DB from '../../../database'; import logger from '../../../logger'; const fsPromises = promises; @@ -16,12 +14,7 @@ class FundingTxFetcher { private channelNewlyProcessed = 0; public fundingTxCache = {}; - async $fetchChannelsFundingTxs(channelIds: string[]): Promise { - if (this.running) { - return; - } - this.running = true; - + async $init(): Promise { // Load funding tx disk cache if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) { try { @@ -32,6 +25,13 @@ class FundingTxFetcher { } logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`); } + } + + async $fetchChannelsFundingTxs(channelIds: string[]): Promise { + if (this.running) { + return; + } + this.running = true; const globalTimer = new Date().getTime() / 1000; let cacheTimer = new Date().getTime() / 1000; From 33f3b0006bd4d44c709b151552c6421645e393f8 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 2 Aug 2022 21:49:53 +0200 Subject: [PATCH 33/53] Move fast-xml-parser from devDeps to deps --- backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index 750380156..47694ecf8 100644 --- a/backend/package.json +++ b/backend/package.json @@ -37,6 +37,7 @@ "bitcoinjs-lib": "6.0.1", "crypto-js": "^4.0.0", "express": "^4.18.0", + "fast-xml-parser": "^4.0.9", "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", @@ -53,7 +54,6 @@ "@typescript-eslint/parser": "^5.30.5", "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", - "fast-xml-parser": "^4.0.9", "prettier": "^2.7.1" } } From 6be2985b407ef8cdb44ffbbcf59d227169564e08 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 3 Aug 2022 12:13:55 +0200 Subject: [PATCH 34/53] Fix daily LN stats crash --- .../clightning/clightning-convert.ts | 33 ++++- .../tasks/lightning/stats-updater.service.ts | 10 +- .../sync-tasks/funding-tx-fetcher.ts | 2 +- .../lightning/sync-tasks/stats-importer.ts | 132 ++++++++++++------ 4 files changed, 123 insertions(+), 54 deletions(-) diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 1a267bc65..75c8ec20c 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -1,5 +1,6 @@ import { ILightningApi } from '../lightning-api.interface'; import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher'; +import logger from '../../../logger'; /** * Convert a clightning "listnode" entry to a lnd node entry @@ -23,12 +24,17 @@ export function convertNode(clNode: any): ILightningApi.Node { /** * Convert clightning "listchannels" response to lnd "describegraph.edges" format */ - export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise { +export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise { + logger.info('Converting clightning nodes and channels to lnd graph format'); + + let loggerTimer = new Date().getTime() / 1000; + let channelProcessed = 0; + const consolidatedChannelList: ILightningApi.Channel[] = []; const clChannelsDict = {}; const clChannelsDictCount = {}; - for (const clChannel of clChannels) { + for (const clChannel of clChannels) { if (!clChannelsDict[clChannel.short_channel_id]) { clChannelsDict[clChannel.short_channel_id] = clChannel; clChannelsDictCount[clChannel.short_channel_id] = 1; @@ -39,9 +45,26 @@ export function convertNode(clNode: any): ILightningApi.Node { delete clChannelsDict[clChannel.short_channel_id]; clChannelsDictCount[clChannel.short_channel_id]++; } + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`); + loggerTimer = new Date().getTime() / 1000; + } + + ++channelProcessed; } - for (const short_channel_id of Object.keys(clChannelsDict)) { + + channelProcessed = 0; + const keys = Object.keys(clChannelsDict); + for (const short_channel_id of keys) { consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id])); + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`); + loggerTimer = new Date().getTime() / 1000; + } } return consolidatedChannelList; @@ -79,7 +102,7 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise { +async function buildIncompleteChannel(clChannel: any): Promise { const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id); const parts = clChannel.short_channel_id.split('x'); const outputIdx = parts[2]; @@ -99,7 +122,7 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise { this.$runTasks(); }, this.timeUntilMidnight()); @@ -42,9 +43,14 @@ class LightningStatsUpdater { this.setDateMidnight(date); date.setUTCHours(24); + const [rows] = await DB.query(`SELECT UNIX_TIMESTAMP(MAX(added)) as lastAdded from lightning_stats`); + if ((rows[0].lastAdded ?? 0) === date.getTime() / 1000) { + return; + } + logger.info(`Running lightning daily stats log...`); const networkGraph = await lightningApi.$getNetworkGraph(); - LightningStatsImporter.computeNetworkStats(date.getTime(), networkGraph); + LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph); } } diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts index 926d20c91..6ca72aef7 100644 --- a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -45,7 +45,7 @@ class FundingTxFetcher { let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer); if (elapsedSeconds > 10) { elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer); - logger.debug(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` + + logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` + `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` + `elapsed: ${elapsedSeconds} seconds` ); diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 91e67f77d..d9c441498 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -13,19 +13,19 @@ interface Node { features: string; rgb_color: string; alias: string; - addresses: string; + addresses: unknown[]; out_degree: number; in_degree: number; } interface Channel { - scid: string; - source: string; - destination: string; + channel_id: string; + node1_pub: string; + node2_pub: string; timestamp: number; features: string; fee_base_msat: number; - fee_proportional_millionths: number; + fee_rate_milli_msat: number; htlc_minimim_msat: number; cltv_expiry_delta: number; htlc_maximum_msat: number; @@ -60,10 +60,9 @@ class LightningStatsImporter { let hasClearnet = false; let isUnnanounced = true; - const sockets = node.addresses.split(','); - for (const socket of sockets) { - hasOnion = hasOnion || (socket.indexOf('torv3://') !== -1); - hasClearnet = hasClearnet || (socket.indexOf('ipv4://') !== -1 || socket.indexOf('ipv6://') !== -1); + for (const socket of (node.addresses ?? [])) { + hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network); + hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network); } if (hasOnion && hasClearnet) { clearnetTorNodes++; @@ -90,8 +89,11 @@ class LightningStatsImporter { const baseFees: number[] = []; const alreadyCountedChannels = {}; - for (const channel of networkGraph.channels) { - const short_id = channel.scid.slice(0, -2); + for (const channel of networkGraph.edges) { + let short_id = channel.channel_id; + if (short_id.indexOf('/') !== -1) { + short_id = short_id.slice(0, -2); + } const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id); if (!tx) { @@ -99,23 +101,23 @@ class LightningStatsImporter { continue; } - if (!nodeStats[channel.source]) { - nodeStats[channel.source] = { + if (!nodeStats[channel.node1_pub]) { + nodeStats[channel.node1_pub] = { capacity: 0, channels: 0, }; } - if (!nodeStats[channel.destination]) { - nodeStats[channel.destination] = { + if (!nodeStats[channel.node2_pub]) { + nodeStats[channel.node2_pub] = { capacity: 0, channels: 0, }; } - nodeStats[channel.source].capacity += Math.round(tx.value * 100000000); - nodeStats[channel.source].channels++; - nodeStats[channel.destination].capacity += Math.round(tx.value * 100000000); - nodeStats[channel.destination].channels++; + nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.node1_pub].channels++; + nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.node2_pub].channels++; if (!alreadyCountedChannels[short_id]) { capacity += Math.round(tx.value * 100000000); @@ -123,19 +125,31 @@ class LightningStatsImporter { alreadyCountedChannels[short_id] = true; } - if (channel.fee_proportional_millionths < 5000) { - avgFeeRate += channel.fee_proportional_millionths; - feeRates.push(channel.fee_proportional_millionths); - } - - if (channel.fee_base_msat < 5000) { - avgBaseFee += channel.fee_base_msat; - baseFees.push(channel.fee_base_msat); + if (channel.node1_policy !== undefined) { // Coming from the node + for (const policy of [channel.node1_policy, channel.node2_policy]) { + if (policy && policy.fee_rate_milli_msat < 5000) { + avgFeeRate += policy.fee_rate_milli_msat; + feeRates.push(policy.fee_rate_milli_msat); + } + if (policy && policy.fee_base_msat < 5000) { + avgBaseFee += policy.fee_base_msat; + baseFees.push(policy.fee_base_msat); + } + } + } else { // Coming from the historical import + if (channel.fee_rate_milli_msat < 5000) { + avgFeeRate += channel.fee_rate_milli_msat; + feeRates.push(channel.fee_rate_milli_msat); + } + if (channel.fee_base_msat < 5000) { + avgBaseFee += channel.fee_base_msat; + baseFees.push(channel.fee_base_msat); + } } } - avgFeeRate /= networkGraph.channels.length; - avgBaseFee /= networkGraph.channels.length; + avgFeeRate /= networkGraph.edges.length; + avgBaseFee /= networkGraph.edges.length; const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; @@ -203,15 +217,28 @@ class LightningStatsImporter { let latestNodeCount = 1; const fileList = await fsPromises.readdir(this.topologiesFolder); + // Insert history from the most recent to the oldest + // This also put the .json cached files first fileList.sort().reverse(); - const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) as added, node_count FROM lightning_stats'); + const [rows]: any[] = await DB.query(` + SELECT UNIX_TIMESTAMP(added) AS added, node_count + FROM lightning_stats + ORDER BY added DESC + `); const existingStatsTimestamps = {}; for (const row of rows) { - existingStatsTimestamps[row.added] = rows[0]; + existingStatsTimestamps[row.added] = row; } + // For logging purpose + let processed = 10; + let totalProcessed = -1; + for (const filename of fileList) { + processed++; + totalProcessed++; + const timestamp = parseInt(filename.split('_')[1], 10); // Stats exist already, don't calculate/insert them @@ -220,7 +247,7 @@ class LightningStatsImporter { continue; } - logger.debug(`Processing ${this.topologiesFolder}/${filename}`); + logger.debug(`Reading ${this.topologiesFolder}/${filename}`); const fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8'); let graph; @@ -228,12 +255,13 @@ class LightningStatsImporter { try { graph = JSON.parse(fileContent); } catch (e) { - logger.debug(`Invalid topology file, cannot parse the content`); + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); + continue; } } else { graph = this.parseFile(fileContent); if (!graph) { - logger.debug(`Invalid topology file, cannot parse the content`); + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); continue; } await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph)); @@ -245,19 +273,22 @@ class LightningStatsImporter { const diffRatio = graph.nodes.length / latestNodeCount; if (diffRatio < 0.9) { // Ignore drop of more than 90% of the node count as it's probably a missing data point + logger.debug(`Nodes count diff ratio threshold reached, ignore the data for this day ${graph.nodes.length} nodes vs ${latestNodeCount}`); continue; } } latestNodeCount = graph.nodes.length; const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; - logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.channels.length} channels`); + logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); - // Cache funding txs - logger.debug(`Caching funding txs for ${datestr}`); - await fundingTxFetcher.$fetchChannelsFundingTxs(graph.channels.map(channel => channel.scid.slice(0, -2))); - - logger.debug(`Generating LN network stats for ${datestr}`); + if (processed > 10) { + logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); + processed = 0; + } else { + logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); + } + await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2))); const stat = await this.computeNetworkStats(timestamp, graph); existingStatsTimestamps[timestamp] = stat; @@ -290,13 +321,22 @@ class LightningStatsImporter { if (!node.data) { continue; } + const addresses: unknown[] = []; + const sockets = node.data[5].split(','); + for (const socket of sockets) { + const parts = socket.split('://'); + addresses.push({ + network: parts[0], + addr: parts[1], + }); + } nodes.push({ id: node.data[0], timestamp: node.data[1], features: node.data[2], rgb_color: node.data[3], alias: node.data[4], - addresses: node.data[5], + addresses: addresses, out_degree: node.data[6], in_degree: node.data[7], }); @@ -307,13 +347,13 @@ class LightningStatsImporter { continue; } channels.push({ - scid: channel.data[0], - source: channel.data[1], - destination: channel.data[2], + channel_id: channel.data[0], + node1_pub: channel.data[1], + node2_pub: channel.data[2], timestamp: channel.data[3], features: channel.data[4], fee_base_msat: channel.data[5], - fee_proportional_millionths: channel.data[6], + fee_rate_milli_msat: channel.data[6], htlc_minimim_msat: channel.data[7], cltv_expiry_delta: channel.data[8], htlc_maximum_msat: channel.data[9], @@ -322,7 +362,7 @@ class LightningStatsImporter { return { nodes: nodes, - channels: channels, + edges: channels, }; } } From 99379d53bff509ee748d78512768e7f7d96a3d34 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 3 Aug 2022 12:43:41 +0200 Subject: [PATCH 35/53] When LN backend crashed, catch the error and restart after 1 minute --- backend/src/index.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 976ec12df..683f964f0 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -137,9 +137,7 @@ class Server { } if (config.LIGHTNING.ENABLED) { - fundingTxFetcher.$init() - .then(() => networkSyncService.$startService()) - .then(() => lightningStatsUpdater.$startService()); + this.$runLightningBackend(); } this.server.listen(config.MEMPOOL.HTTP_PORT, () => { @@ -185,6 +183,18 @@ class Server { } } + async $runLightningBackend() { + try { + await fundingTxFetcher.$init(); + await networkSyncService.$startService(); + await lightningStatsUpdater.$startService(); + } catch(e) { + logger.err(`Lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`); + await Common.sleep$(1000 * 60); + this.$runLightningBackend(); + }; +} + setUpWebsocketHandling() { if (this.wss) { websocketHandler.setWebsocketServer(this.wss); From 3c2e27f778dc0628ab88be1036a6b343a5bcf97a Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 4 Aug 2022 11:30:32 +0200 Subject: [PATCH 36/53] Fix node page and display real time data --- backend/src/api/explorer/channels.api.ts | 66 +++++++++++-- backend/src/api/explorer/channels.routes.ts | 6 +- backend/src/api/explorer/nodes.api.ts | 96 ++++++++++++++----- backend/src/api/explorer/nodes.routes.ts | 6 ++ .../channels-list.component.html | 38 +++++--- .../channels-list.component.scss | 8 +- .../channels-list/channels-list.component.ts | 42 +++++--- .../app/lightning/node/node.component.html | 23 ++--- .../src/app/lightning/node/node.component.ts | 10 +- 9 files changed, 218 insertions(+), 77 deletions(-) diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 79aeebb97..9928cc85b 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -1,5 +1,6 @@ import logger from '../../logger'; import DB from '../../database'; +import nodesApi from './nodes.api'; class ChannelsApi { public async $getAllChannels(): Promise { @@ -181,15 +182,57 @@ class ChannelsApi { public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise { try { - // Default active and inactive channels - let statusQuery = '< 2'; - // Closed channels only - if (status === 'closed') { - statusQuery = '= 2'; + let channelStatusFilter; + if (status === 'open') { + channelStatusFilter = '< 2'; + } else if (status === 'closed') { + channelStatusFilter = '= 2'; } - const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery} ORDER BY channels.capacity DESC LIMIT ?, ?`; - const [rows]: any = await DB.query(query, [public_key, public_key, index, length]); - const channels = rows.map((row) => this.convertChannel(row)); + + // Channels originating from node + let query = ` + SELECT node2.alias, node2.public_key, channels.status, channels.node1_fee_rate, + channels.capacity, channels.short_id, channels.id + FROM channels + JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key + WHERE node1_public_key = ? AND channels.status ${channelStatusFilter} + `; + const [channelsFromNode]: any = await DB.query(query, [public_key, index, length]); + + // Channels incoming to node + query = ` + SELECT node1.alias, node1.public_key, channels.status, channels.node2_fee_rate, + channels.capacity, channels.short_id, channels.id + FROM channels + JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key + WHERE node2_public_key = ? AND channels.status ${channelStatusFilter} + `; + const [channelsToNode]: any = await DB.query(query, [public_key, index, length]); + + let allChannels = channelsFromNode.concat(channelsToNode); + allChannels.sort((a, b) => { + return b.capacity - a.capacity; + }); + allChannels = allChannels.slice(index, index + length); + + const channels: any[] = [] + for (const row of allChannels) { + const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key); + channels.push({ + status: row.status, + capacity: row.capacity ?? 0, + short_id: row.short_id, + id: row.id, + fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0, + node: { + alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), + public_key: row.public_key, + channels: activeChannelsStats.active_channel_count ?? 0, + capacity: activeChannelsStats.capacity ?? 0, + } + }); + } + return channels; } catch (e) { logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e)); @@ -205,7 +248,12 @@ class ChannelsApi { if (status === 'closed') { statusQuery = '= 2'; } - const query = `SELECT COUNT(*) AS count FROM channels WHERE (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery}`; + const query = ` + SELECT COUNT(*) AS count + FROM channels + WHERE (node1_public_key = ? OR node2_public_key = ?) + AND status ${statusQuery} + `; const [rows]: any = await DB.query(query, [public_key, public_key]); return rows[0]['count']; } catch (e) { diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index 495eec789..bbb075aa6 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -46,9 +46,11 @@ class ChannelsRoutes { } const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0; const status: string = typeof req.query.status === 'string' ? req.query.status : ''; - const length = 25; - const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, length, status); + const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status); const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.header('X-Total-Count', channelsCount.toString()); res.json(channels); } catch (e) { diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 96da7d1d5..6fba07449 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -4,21 +4,13 @@ import DB from '../../database'; class NodesApi { public async $getNode(public_key: string): Promise { try { - const query = ` - SELECT nodes.*, geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city, - geo_names_country.names as country, geo_names_subdivision.names as subdivision, - (SELECT Count(*) - FROM channels - WHERE channels.status = 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_closed_count, - (SELECT Count(*) - FROM channels - WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_active_count, - (SELECT Sum(capacity) - FROM channels - WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS capacity, - (SELECT Avg(capacity) - FROM channels - WHERE status = 1 AND ( node1_public_key = ? OR node2_public_key = ? )) AS channels_capacity_avg + // General info + let query = ` + SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen, + UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets, + as_number, city_id, country_id, subdivision_id, longitude, latitude, + geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city, + geo_names_country.names as country, geo_names_subdivision.names as subdivision FROM nodes LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id @@ -27,21 +19,70 @@ class NodesApi { LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' WHERE public_key = ? `; - const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key]); - if (rows.length > 0) { - rows[0].as_organization = JSON.parse(rows[0].as_organization); - rows[0].subdivision = JSON.parse(rows[0].subdivision); - rows[0].city = JSON.parse(rows[0].city); - rows[0].country = JSON.parse(rows[0].country); - return rows[0]; + let [rows]: any[] = await DB.query(query, [public_key]); + if (rows.length === 0) { + throw new Error(`This node does not exist, or our node is not seeing it yet`); } - return null; + + const node = rows[0]; + node.as_organization = JSON.parse(node.as_organization); + node.subdivision = JSON.parse(node.subdivision); + node.city = JSON.parse(node.city); + node.country = JSON.parse(node.country); + + // Active channels and capacity + const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key); + node.active_channel_count = activeChannelsStats.active_channel_count ?? 0; + node.capacity = activeChannelsStats.capacity ?? 0; + + // Opened channels count + query = ` + SELECT count(short_id) as opened_channel_count + FROM channels + WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?) + `; + [rows] = await DB.query(query, [public_key, public_key]); + node.opened_channel_count = 0; + if (rows.length > 0) { + node.opened_channel_count = rows[0].opened_channel_count; + } + + // Closed channels count + query = ` + SELECT count(short_id) as closed_channel_count + FROM channels + WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?) + `; + [rows] = await DB.query(query, [public_key, public_key]); + node.closed_channel_count = 0; + if (rows.length > 0) { + node.closed_channel_count = rows[0].closed_channel_count; + } + + return node; } catch (e) { - logger.err('$getNode error: ' + (e instanceof Error ? e.message : e)); + logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`); throw e; } } + public async $getActiveChannelsStats(node_public_key: string): Promise { + const query = ` + SELECT count(short_id) as active_channel_count, sum(capacity) as capacity + FROM channels + WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?) + `; + const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]); + if (rows.length > 0) { + return { + active_channel_count: rows[0].active_channel_count, + capacity: rows[0].capacity + }; + } else { + return null; + } + } + public async $getAllNodes(): Promise { try { const query = `SELECT * FROM nodes`; @@ -55,7 +96,12 @@ class NodesApi { public async $getNodeStats(public_key: string): Promise { try { - const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`; + const query = ` + SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels + FROM node_stats + WHERE public_key = ? + ORDER BY added DESC + `; const [rows]: any = await DB.query(query, [public_key]); return rows; } catch (e) { diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index 83e3c393e..a850b6a09 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -35,6 +35,9 @@ class NodesRoutes { res.status(404).send('Node not found'); return; } + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.json(node); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); @@ -44,6 +47,9 @@ class NodesRoutes { private async $getHistoricalNodeStats(req: Request, res: Response) { try { const statistics = await nodesApi.$getNodeStats(req.params.public_key); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.json(statistics); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); diff --git a/frontend/src/app/lightning/channels-list/channels-list.component.html b/frontend/src/app/lightning/channels-list/channels-list.component.html index 82283f689..b95cddf8d 100644 --- a/frontend/src/app/lightning/channels-list/channels-list.component.html +++ b/frontend/src/app/lightning/channels-list/channels-list.component.html @@ -2,24 +2,24 @@
- +
- +
- +
No channels to display
@@ -30,7 +30,7 @@ - + @@ -42,31 +42,41 @@
{{ node.alias || '?' }}
- {{ node.public_key | shortenString : 10 }} + {{ node.public_key | shortenString : publicKeySize }}
+ + + {{ channel.capacity | amountShortener: 1 }} + sats + + diff --git a/frontend/src/app/lightning/channels-list/channels-list.component.scss b/frontend/src/app/lightning/channels-list/channels-list.component.scss index 35a6ce0bc..ba7b0a3b5 100644 --- a/frontend/src/app/lightning/channels-list/channels-list.component.scss +++ b/frontend/src/app/lightning/channels-list/channels-list.component.scss @@ -1,3 +1,9 @@ .second-line { font-size: 12px; -} \ No newline at end of file +} + +.sats { + color: #ffffff66; + font-size: 12px; + top: 0px; +} diff --git a/frontend/src/app/lightning/channels-list/channels-list.component.ts b/frontend/src/app/lightning/channels-list/channels-list.component.ts index 4060d36da..6172a4a99 100644 --- a/frontend/src/app/lightning/channels-list/channels-list.component.ts +++ b/frontend/src/app/lightning/channels-list/channels-list.component.ts @@ -1,7 +1,8 @@ import { ChangeDetectionStrategy, Component, EventEmitter, Input, OnChanges, OnInit, Output } from '@angular/core'; import { FormBuilder, FormGroup } from '@angular/forms'; -import { BehaviorSubject, combineLatest, merge, Observable, of } from 'rxjs'; -import { map, startWith, switchMap } from 'rxjs/operators'; +import { BehaviorSubject, merge, Observable } from 'rxjs'; +import { map, switchMap, tap } from 'rxjs/operators'; +import { isMobile } from 'src/app/shared/common.utils'; import { LightningApiService } from '../lightning-api.service'; @Component({ @@ -18,11 +19,13 @@ export class ChannelsListComponent implements OnInit, OnChanges { // @ts-ignore paginationSize: 'sm' | 'lg' = 'md'; paginationMaxSize = 10; - itemsPerPage = 25; + itemsPerPage = 10; page = 1; channelsPage$ = new BehaviorSubject(1); channelStatusForm: FormGroup; defaultStatus = 'open'; + status = 'open'; + publicKeySize = 25; constructor( private lightningApiService: LightningApiService, @@ -31,9 +34,12 @@ export class ChannelsListComponent implements OnInit, OnChanges { this.channelStatusForm = this.formBuilder.group({ status: [this.defaultStatus], }); + if (isMobile()) { + this.publicKeySize = 12; + } } - ngOnInit() { + ngOnInit(): void { if (document.body.clientWidth < 670) { this.paginationSize = 'sm'; this.paginationMaxSize = 3; @@ -41,28 +47,36 @@ export class ChannelsListComponent implements OnInit, OnChanges { } ngOnChanges(): void { - this.channelStatusForm.get('status').setValue(this.defaultStatus, { emitEvent: false }) - this.channelsStatusChangedEvent.emit(this.defaultStatus); + this.channelStatusForm.get('status').setValue(this.defaultStatus, { emitEvent: false }); + this.channelsPage$.next(1); - this.channels$ = combineLatest([ + this.channels$ = merge( this.channelsPage$, - this.channelStatusForm.get('status').valueChanges.pipe(startWith(this.defaultStatus)) - ]) + this.channelStatusForm.get('status').valueChanges, + ) .pipe( - switchMap(([page, status]) => { - this.channelsStatusChangedEvent.emit(status); - return this.lightningApiService.getChannelsByNodeId$(this.publicKey, (page -1) * this.itemsPerPage, status); + tap((val) => { + if (typeof val === 'string') { + this.status = val; + this.page = 1; + } else if (typeof val === 'number') { + this.page = val; + } + }), + switchMap(() => { + this.channelsStatusChangedEvent.emit(this.status); + return this.lightningApiService.getChannelsByNodeId$(this.publicKey, (this.page - 1) * this.itemsPerPage, this.status); }), map((response) => { return { channels: response.body, - totalItems: parseInt(response.headers.get('x-total-count'), 10) + totalItems: parseInt(response.headers.get('x-total-count'), 10) + 1 }; }), ); } - pageChange(page: number) { + pageChange(page: number): void { this.channelsPage$.next(page); } diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index cb0e5ed43..ac50ed51b 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -2,8 +2,9 @@

{{ node.alias }}

- {{ node.public_key | shortenString : 12 - }} + + {{ node.public_key | shortenString : publicKeySize }} +
@@ -22,23 +23,23 @@
Node Alias  StatusStatus Fee Rate Capacity Channel ID
{{ node.channels }} channels
-
+
+ + + {{ node.capacity | amountShortener: 1 }} + sats + +
- Inactive - Active + Inactive + Active - Closed + Closed - {{ node.fee_rate }} ppm ({{ node.fee_rate / 10000 | number }}%) + {{ channel.fee_rate }} ppm ({{ channel.fee_rate / 10000 | number }}%) - - {{ channel.short_id }}
- + - + - + @@ -71,13 +72,13 @@ @@ -139,7 +140,7 @@
-

Channels ({{ channelsListStatus === 'open' ? node.channel_active_count : node.channel_closed_count }})

+

All channels ({{ channelsListStatus === 'open' ? node.opened_channel_count : node.closed_channel_count }})

diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index a8d487938..6f9358090 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -5,6 +5,7 @@ import { catchError, map, switchMap } from 'rxjs/operators'; import { SeoService } from 'src/app/services/seo.service'; import { getFlagEmoji } from 'src/app/shared/graphs.utils'; import { LightningApiService } from '../lightning-api.service'; +import { isMobile } from '../../shared/common.utils'; @Component({ selector: 'app-node', @@ -23,11 +24,17 @@ export class NodeComponent implements OnInit { error: Error; publicKey: string; + publicKeySize = 99; + constructor( private lightningApiService: LightningApiService, private activatedRoute: ActivatedRoute, private seoService: SeoService, - ) { } + ) { + if (isMobile()) { + this.publicKeySize = 12; + } + } ngOnInit(): void { this.node$ = this.activatedRoute.paramMap @@ -59,6 +66,7 @@ export class NodeComponent implements OnInit { }); } node.socketsObject = socketsObject; + node.avgCapacity = node.capacity / Math.max(1, node.active_channel_count); return node; }), catchError(err => { From 3bf7bf556372d705d75b9fb50833714892da41c0 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 4 Aug 2022 12:49:07 +0200 Subject: [PATCH 37/53] Add missing file --- frontend/src/app/shared/common.utils.ts | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 frontend/src/app/shared/common.utils.ts diff --git a/frontend/src/app/shared/common.utils.ts b/frontend/src/app/shared/common.utils.ts new file mode 100644 index 000000000..419c1665d --- /dev/null +++ b/frontend/src/app/shared/common.utils.ts @@ -0,0 +1,3 @@ +export function isMobile() { + return (window.innerWidth <= 767.98); +} From f6d6ea5d31d435b69a6381bb6dadf33969156ab2 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 4 Aug 2022 13:05:15 +0200 Subject: [PATCH 38/53] Gracefully attempt to reconnect to cln upon error --- .../api/lightning/clightning/clightning-client.ts | 15 ++++++++++++--- .../src/tasks/lightning/network-sync.service.ts | 6 ++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts index 15f472f2e..0535e0881 100644 --- a/backend/src/api/lightning/clightning/clightning-client.ts +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -157,8 +157,18 @@ export default class CLightningClient extends EventEmitter implements AbstractLi const _self = this; - this.client = createConnection(rpcPath); - this.rl = createInterface({ input: this.client }) + this.client = createConnection(rpcPath).on( + 'error', () => { + _self.increaseWaitTime(); + _self.reconnect(); + } + ); + this.rl = createInterface({ input: this.client }).on( + 'error', () => { + _self.increaseWaitTime(); + _self.reconnect(); + } + ); this.clientConnectionPromise = new Promise(resolve => { _self.client.on('connect', () => { @@ -175,7 +185,6 @@ export default class CLightningClient extends EventEmitter implements AbstractLi _self.client.on('error', error => { logger.err(`[CLightningClient] Lightning client connection error: ${error}`); - _self.emit('error', error); _self.increaseWaitTime(); _self.reconnect(); }); diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 826664cf4..5af6aef25 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -9,6 +9,7 @@ import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { $lookupNodeLocation } from './sync-tasks/node-locations'; import lightningApi from '../../api/lightning/lightning-api-factory'; import { convertChannelId } from '../../api/lightning/clightning/clightning-convert'; +import { Common } from '../../api/common'; class NetworkSyncService { constructor() {} @@ -23,14 +24,15 @@ class NetworkSyncService { }, 1000 * 60 * 60); } - private async $runUpdater() { + private async $runUpdater(): Promise { try { logger.info(`Updating nodes and channels...`); const networkGraph = await lightningApi.$getNetworkGraph(); if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) { logger.info(`LN Network graph is empty, retrying in 10 seconds`); - setTimeout(this.$runUpdater, 10000); + await Common.sleep$(10000); + this.$runUpdater(); return; } From f60ef05223ae66629d12d8dd764f1520fb9b438f Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 4 Aug 2022 13:11:24 +0200 Subject: [PATCH 39/53] Re-applied missing fix from https://github.com/mempool/mempool/pull/2233 --- backend/src/api/explorer/nodes.api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 96da7d1d5..d6984da45 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -70,7 +70,7 @@ class NodesApi { const latestDate = rows[0].maxAdded; const query = ` - SELECT nodes.public_key, nodes.alias, node_stats.capacity, node_stats.channels + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels FROM node_stats JOIN nodes ON nodes.public_key = node_stats.public_key WHERE added = FROM_UNIXTIME(${latestDate}) @@ -92,7 +92,7 @@ class NodesApi { const latestDate = rows[0].maxAdded; const query = ` - SELECT nodes.public_key, nodes.alias, node_stats.capacity, node_stats.channels + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels FROM node_stats JOIN nodes ON nodes.public_key = node_stats.public_key WHERE added = FROM_UNIXTIME(${latestDate}) From d647edcae3184e704ea2018c611e10d84cfd481f Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 4 Aug 2022 13:11:24 +0200 Subject: [PATCH 40/53] Re-applied missing fix from https://github.com/mempool/mempool/pull/2233 --- backend/src/api/explorer/nodes.api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 96da7d1d5..d6984da45 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -70,7 +70,7 @@ class NodesApi { const latestDate = rows[0].maxAdded; const query = ` - SELECT nodes.public_key, nodes.alias, node_stats.capacity, node_stats.channels + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels FROM node_stats JOIN nodes ON nodes.public_key = node_stats.public_key WHERE added = FROM_UNIXTIME(${latestDate}) @@ -92,7 +92,7 @@ class NodesApi { const latestDate = rows[0].maxAdded; const query = ` - SELECT nodes.public_key, nodes.alias, node_stats.capacity, node_stats.channels + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels FROM node_stats JOIN nodes ON nodes.public_key = node_stats.public_key WHERE added = FROM_UNIXTIME(${latestDate}) From 54669281debda36fe00a6b72015a57ff6e907693 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 4 Aug 2022 18:27:36 +0200 Subject: [PATCH 41/53] Run node stats every 10 minutes, only keep the latest entry per day --- backend/src/config.ts | 2 + .../tasks/lightning/stats-updater.service.ts | 29 ++--- .../lightning/sync-tasks/stats-importer.ts | 102 ++++++++++++------ 3 files changed, 80 insertions(+), 53 deletions(-) diff --git a/backend/src/config.ts b/backend/src/config.ts index b42a45ab2..d4dfc9edd 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -32,6 +32,7 @@ interface IConfig { ENABLED: boolean; BACKEND: 'lnd' | 'cln' | 'ldk'; TOPOLOGY_FOLDER: string; + NODE_STATS_REFRESH_INTERVAL: number; }; LND: { TLS_CERT_PATH: string; @@ -183,6 +184,7 @@ const defaults: IConfig = { 'ENABLED': false, 'BACKEND': 'lnd', 'TOPOLOGY_FOLDER': '', + 'NODE_STATS_REFRESH_INTERVAL': 600, }, 'LND': { 'TLS_CERT_PATH': '', diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 237cacd72..0fd147eef 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -2,25 +2,14 @@ import DB from '../../database'; import logger from '../../logger'; import lightningApi from '../../api/lightning/lightning-api-factory'; import LightningStatsImporter from './sync-tasks/stats-importer'; +import config from '../../config'; class LightningStatsUpdater { - hardCodedStartTime = '2018-01-12'; - public async $startService(): Promise { logger.info('Starting Lightning Stats service'); - LightningStatsImporter.$run(); - - setTimeout(() => { - this.$runTasks(); - }, this.timeUntilMidnight()); - } - - private timeUntilMidnight(): number { - const date = new Date(); - this.setDateMidnight(date); - date.setUTCHours(24); - return date.getTime() - new Date().getTime(); + // LightningStatsImporter.$run(); + this.$runTasks(); } private setDateMidnight(date: Date): void { @@ -35,20 +24,18 @@ class LightningStatsUpdater { setTimeout(() => { this.$runTasks(); - }, this.timeUntilMidnight()); + }, 1000 * config.LIGHTNING.NODE_STATS_REFRESH_INTERVAL); } + /** + * Update the latest entry for each node every config.LIGHTNING.NODE_STATS_REFRESH_INTERVAL seconds + */ private async $logStatsDaily(): Promise { const date = new Date(); this.setDateMidnight(date); date.setUTCHours(24); - const [rows] = await DB.query(`SELECT UNIX_TIMESTAMP(MAX(added)) as lastAdded from lightning_stats`); - if ((rows[0].lastAdded ?? 0) === date.getTime() / 1000) { - return; - } - - logger.info(`Running lightning daily stats log...`); + logger.info(`Updating latest node stats`); const networkGraph = await lightningApi.$getNetworkGraph(); LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph); } diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index d9c441498..ba4adc71c 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -41,7 +41,7 @@ class LightningStatsImporter { const [channels]: any[] = await DB.query('SELECT short_id from channels;'); logger.info('Caching funding txs for currently existing channels'); await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); - + await this.$importHistoricalLightningStats(); } @@ -114,15 +114,15 @@ class LightningStatsImporter { }; } - nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000); - nodeStats[channel.node1_pub].channels++; - nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000); - nodeStats[channel.node2_pub].channels++; - if (!alreadyCountedChannels[short_id]) { capacity += Math.round(tx.value * 100000000); capacities.push(Math.round(tx.value * 100000000)); alreadyCountedChannels[short_id] = true; + + nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.node1_pub].channels++; + nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.node2_pub].channels++; } if (channel.node1_policy !== undefined) { // Coming from the node @@ -154,24 +154,40 @@ class LightningStatsImporter { const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; const avgCapacity = Math.round(capacity / capacities.length); - + let query = `INSERT INTO lightning_stats( - added, - channel_count, - node_count, - total_capacity, - tor_nodes, - clearnet_nodes, - unannounced_nodes, - clearnet_tor_nodes, - avg_capacity, - avg_fee_rate, - avg_base_fee_mtokens, - med_capacity, - med_fee_rate, - med_base_fee_mtokens - ) - VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + added, + channel_count, + node_count, + total_capacity, + tor_nodes, + clearnet_nodes, + unannounced_nodes, + clearnet_tor_nodes, + avg_capacity, + avg_fee_rate, + avg_base_fee_mtokens, + med_capacity, + med_fee_rate, + med_base_fee_mtokens + ) + VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + added = FROM_UNIXTIME(?), + channel_count = ?, + node_count = ?, + total_capacity = ?, + tor_nodes = ?, + clearnet_nodes = ?, + unannounced_nodes = ?, + clearnet_tor_nodes = ?, + avg_capacity = ?, + avg_fee_rate = ?, + avg_base_fee_mtokens = ?, + med_capacity = ?, + med_fee_rate = ?, + med_base_fee_mtokens = ? + `; await DB.query(query, [ timestamp, @@ -188,22 +204,44 @@ class LightningStatsImporter { medCapacity, medFeeRate, medBaseFee, + timestamp, + capacities.length, + networkGraph.nodes.length, + capacity, + torNodes, + clearnetNodes, + unannouncedNodes, + clearnetTorNodes, + avgCapacity, + avgFeeRate, + avgBaseFee, + medCapacity, + medFeeRate, + medBaseFee, ]); for (const public_key of Object.keys(nodeStats)) { query = `INSERT INTO node_stats( - public_key, - added, - capacity, - channels - ) - VALUES (?, FROM_UNIXTIME(?), ?, ?)`; - + public_key, + added, + capacity, + channels + ) + VALUES (?, FROM_UNIXTIME(?), ?, ?) + ON DUPLICATE KEY UPDATE + added = FROM_UNIXTIME(?), + capacity = ?, + channels = ? + `; + await DB.query(query, [ public_key, timestamp, nodeStats[public_key].capacity, nodeStats[public_key].channels, + timestamp, + nodeStats[public_key].capacity, + nodeStats[public_key].channels, ]); } @@ -278,7 +316,7 @@ class LightningStatsImporter { } } latestNodeCount = graph.nodes.length; - + const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); @@ -367,4 +405,4 @@ class LightningStatsImporter { } } -export default new LightningStatsImporter; \ No newline at end of file +export default new LightningStatsImporter; From 5d81a13a80e26b417740afd6599237525693066c Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 5 Aug 2022 10:11:29 +0200 Subject: [PATCH 42/53] Always show channels map in node page - auto zoom on the node --- .../app/lightning/node/node.component.html | 20 +++------ .../src/app/lightning/node/node.component.ts | 9 ---- .../nodes-channels-map.component.html | 3 -- .../nodes-channels-map.component.scss | 3 +- .../nodes-channels-map.component.ts | 41 +++++++++---------- 5 files changed, 27 insertions(+), 49 deletions(-) diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index ac50ed51b..de6c816f0 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -102,9 +102,7 @@
-
- -
+
-
- + -
-
-

All channels ({{ channelsListStatus === 'open' ? node.opened_channel_count : node.closed_channel_count }})

-
- -
+

Channels ({{ channelsListStatus === 'open' ? node.opened_channel_count : node.closed_channel_count }})

- - -
-
\ No newline at end of file +
diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 6f9358090..a81849388 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -19,7 +19,6 @@ export class NodeComponent implements OnInit { publicKey$: Observable; selectedSocketIndex = 0; qrCodeVisible = false; - channelsListMode = 'list'; channelsListStatus: string; error: Error; publicKey: string; @@ -83,14 +82,6 @@ export class NodeComponent implements OnInit { this.selectedSocketIndex = index; } - channelsListModeChange(toggle) { - if (toggle === true) { - this.channelsListMode = 'map'; - } else { - this.channelsListMode = 'list'; - } - } - onChannelsListStatusChanged(e) { this.channelsListStatus = e; } diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html index 1208a906a..5ccb9f3bc 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html @@ -3,9 +3,6 @@
Lightning nodes channels world map -
(Tor nodes excluded)
diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss index 7914a5364..7e6b9f050 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss @@ -20,7 +20,8 @@ } .full-container.nodepage { - margin-top: 50px; + margin-top: 25px; + margin-bottom: 25px; } .full-container.widget { diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index c71ff88ad..e0063858b 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -3,7 +3,6 @@ import { SeoService } from 'src/app/services/seo.service'; import { ApiService } from 'src/app/services/api.service'; import { Observable, switchMap, tap, zip } from 'rxjs'; import { AssetsService } from 'src/app/services/assets.service'; -import { download } from 'src/app/shared/graphs.utils'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe'; import { StateService } from 'src/app/services/state.service'; @@ -21,6 +20,7 @@ export class NodesChannelsMap implements OnInit, OnDestroy { @Input() publicKey: string | undefined; observable$: Observable; + center: number[] | undefined = undefined; chartInstance = undefined; chartOptions: EChartsOption = {}; @@ -42,6 +42,8 @@ export class NodesChannelsMap implements OnInit, OnDestroy { ngOnDestroy(): void {} ngOnInit(): void { + this.center = this.style === 'widget' ? [0, 0, -10] : undefined; + if (this.style === 'graph') { this.seoService.setTitle($localize`Lightning nodes channels world map`); } @@ -52,13 +54,21 @@ export class NodesChannelsMap implements OnInit, OnDestroy { return zip( this.assetsService.getWorldMapJson$, this.apiService.getChannelsGeo$(params.get('public_key') ?? undefined), + [params.get('public_key') ?? undefined] ).pipe(tap((data) => { registerMap('world', data[0]); const channelsLoc = []; const nodes = []; const nodesPubkeys = {}; + let thisNodeGPS: number[] | undefined = undefined; for (const channel of data[1]) { + if (!thisNodeGPS && data[2] === channel[0]) { + thisNodeGPS = [channel[2], channel[3]]; + } else if (!thisNodeGPS && data[2] === channel[4]) { + thisNodeGPS = [channel[6], channel[7]]; + } + channelsLoc.push([[channel[2], channel[3]], [channel[6], channel[7]]]); if (!nodesPubkeys[channel[0]]) { nodes.push({ @@ -77,6 +87,13 @@ export class NodesChannelsMap implements OnInit, OnDestroy { nodesPubkeys[channel[4]] = true; } } + if (this.style === 'nodepage' && thisNodeGPS) { + // 1ML 0217890e3aad8d35bc054f43acc00084b25229ecff0ab68debd82883ad65ee8266 + // New York GPS [-74.0068, 40.7123] + // Map center [-20.55, 0, -9.85] + this.center = [thisNodeGPS[0] * -20.55 / -74.0068, 0, thisNodeGPS[1] * -9.85 / 40.7123]; + } + this.prepareChartOptions(nodes, channelsLoc); })); }) @@ -111,10 +128,10 @@ export class NodesChannelsMap implements OnInit, OnDestroy { } }, viewControl: { - center: this.style === 'widget' ? [0, 0, -10] : undefined, + center: this.center, minDistance: 1, maxDistance: 60, - distance: this.style === 'widget' ? 22 : 60, + distance: this.style === 'widget' ? 22 : this.style === 'nodepage' ? 22 : 60, alpha: 90, rotateSensitivity: 0, panSensitivity: this.style === 'widget' ? 0 : 1, @@ -204,22 +221,4 @@ export class NodesChannelsMap implements OnInit, OnDestroy { } }); } - - onSaveChart() { - // @ts-ignore - const prevBottom = this.chartOptions.grid.bottom; - const now = new Date(); - // @ts-ignore - this.chartOptions.grid.bottom = 30; - this.chartOptions.backgroundColor = '#11131f'; - this.chartInstance.setOption(this.chartOptions); - download(this.chartInstance.getDataURL({ - pixelRatio: 2, - excludeComponents: ['dataZoom'], - }), `lightning-nodes-heatmap-clearnet-${Math.round(now.getTime() / 1000)}.svg`); - // @ts-ignore - this.chartOptions.grid.bottom = prevBottom; - this.chartOptions.backgroundColor = 'none'; - this.chartInstance.setOption(this.chartOptions); - } } From 5a50a0d973141c170461498981c83ed2336f2d39 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 5 Aug 2022 12:32:20 +0200 Subject: [PATCH 43/53] Make sure lightning stats are not duplicated in db --- backend/src/api/database-migration.ts | 7 ++++++- backend/src/tasks/lightning/stats-updater.service.ts | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 816efc7cc..19f523eb3 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 34; + private static currentVersion = 35; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -315,6 +315,11 @@ class DatabaseMigration { if (databaseSchemaVersion < 34 && isBitcoin == true) { await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"'); } + + if (databaseSchemaVersion < 35 && isBitcoin == true) { + await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);'); + } } /** diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 0fd147eef..fbbef4021 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -8,8 +8,8 @@ class LightningStatsUpdater { public async $startService(): Promise { logger.info('Starting Lightning Stats service'); - // LightningStatsImporter.$run(); - this.$runTasks(); + await this.$runTasks(); + LightningStatsImporter.$run(); } private setDateMidnight(date: Date): void { @@ -35,7 +35,7 @@ class LightningStatsUpdater { this.setDateMidnight(date); date.setUTCHours(24); - logger.info(`Updating latest node stats`); + logger.info(`Updating latest networks stats`); const networkGraph = await lightningApi.$getNetworkGraph(); LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph); } From c01c610bb383918bbd08e19f8ad7d997f0706ab6 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 5 Aug 2022 12:43:26 +0200 Subject: [PATCH 44/53] Don't insert stats in the future --- backend/src/tasks/lightning/stats-updater.service.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 0fd147eef..8fea9eb30 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -33,7 +33,6 @@ class LightningStatsUpdater { private async $logStatsDaily(): Promise { const date = new Date(); this.setDateMidnight(date); - date.setUTCHours(24); logger.info(`Updating latest node stats`); const networkGraph = await lightningApi.$getNetworkGraph(); From 5612a033d56f297151c09ddebce1092c2bb5b9ad Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 6 Aug 2022 04:25:21 +0400 Subject: [PATCH 45/53] Right align mempool logo on mobile with dual logos --- .../app/components/master-page/master-page.component.html | 2 +- .../app/components/master-page/master-page.component.scss | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index f152cb7b3..39acf122d 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -1,7 +1,7 @@
Total capacityActive capacity
Total channelsActive channels - {{ node.channel_active_count }} + {{ node.active_channel_count }}
Average channel sizeAverage channel size - - + +
First seen - +
Last update - +