highperfocused/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2025-04-07 19:38:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into ops/add-core-lightning

Browse Source
This commit is contained in:
wiz 2022-08-10 22:38:36 +09:00 committed by GitHub
commit e0d677b01c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
49 changed files with 2338 additions and 1821 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/mempool-config.sample.json
View File

@ -79,8 +79,8 @@
},
"LND": {
"TLS_CERT_PATH": "tls.cert",
"MACAROON_PATH": "admin.macaroon",
"SOCKET": "localhost:10009"
"MACAROON_PATH": "readonly.macaroon",
"REST_API_URL": "https://localhost:8080"
},
"SOCKS5PROXY": {
"ENABLED": false,

930
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

6
backend/package.json
View File

@ -16,7 +16,8 @@
"mempool",
"blockchain",
"explorer",
"liquid"
"liquid",
"lightning"
],
"main": "index.ts",
"scripts": {
@ -34,10 +35,9 @@
"@types/node": "^16.11.41",
"axios": "~0.27.2",
"bitcoinjs-lib": "6.0.1",
"bolt07": "^1.8.1",
"crypto-js": "^4.0.0",
"express": "^4.18.0",
"lightning": "^5.16.3",
"fast-xml-parser": "^4.0.9",
"maxmind": "^4.3.6",
"mysql2": "2.3.3",
"node-worker-threads-pool": "^1.5.1",

15
backend/src/api/blocks.ts
View File

@ -22,6 +22,8 @@ import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
class Blocks {
private blocks: BlockExtended[] = [];
@ -457,6 +459,19 @@ class Blocks {
}
await blocksRepository.$saveBlockInDatabase(blockExtended);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
await blocksRepository.$saveBlockPrices([{
height: blockExtended.height,
priceId: lastestPriceId,
}]);
} else {
logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
setTimeout(() => {
indexer.runSingleTask('blocksPrices');
}, 10000);
}
// Save blocks summary for visualization if it's enabled
if (Common.blocksSummariesIndexingEnabled() === true) {
await this.$getStrippedBlockTransactions(blockExtended.id, true);

34
backend/src/api/common.ts
View File

@ -1,5 +1,6 @@
import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
import config from '../config';
import { convertChannelId } from './lightning/clightning/clightning-convert';
export class Common {
static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
'144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
@ -184,4 +185,37 @@ export class Common {
config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
);
}
static setDateMidnight(date: Date): void {
date.setUTCHours(0);
date.setUTCMinutes(0);
date.setUTCSeconds(0);
date.setUTCMilliseconds(0);
}
static channelShortIdToIntegerId(id: string): string {
if (config.LIGHTNING.BACKEND === 'lnd') {
return id;
}
return convertChannelId(id);
}
/** Decodes a channel id returned by lnd as uint64 to a short channel id */
static channelIntegerIdToShortId(id: string): string {
if (config.LIGHTNING.BACKEND === 'cln') {
return id;
}
const n = BigInt(id);
return [
n >> 40n, // nth block
(n >> 16n) & 0xffffffn, // nth tx of the block
n & 0xffffn // nth output of the tx
].join('x');
}
static utcDateToMysql(date?: number): string {
const d = new Date((date || 0) * 1000);
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
}

15
backend/src/api/database-migration.ts
View File

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 33;
private static currentVersion = 36;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -311,6 +311,19 @@ class DatabaseMigration {
if (databaseSchemaVersion < 33 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
}
if (databaseSchemaVersion < 34 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 35 && isBitcoin == true) {
await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
}
if (databaseSchemaVersion < 36 && isBitcoin == true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
}
}
/**

198
backend/src/api/explorer/channels.api.ts
View File

@ -1,5 +1,9 @@
import logger from '../../logger';
import DB from '../../database';
import nodesApi from './nodes.api';
import { ResultSetHeader } from 'mysql2';
import { ILightningApi } from '../lightning/lightning-api.interface';
import { Common } from '../common';
class ChannelsApi {
public async $getAllChannels(): Promise<any[]> {
@ -181,15 +185,57 @@ class ChannelsApi {
public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
try {
// Default active and inactive channels
let statusQuery = '< 2';
// Closed channels only
if (status === 'closed') {
statusQuery = '= 2';
let channelStatusFilter;
if (status === 'open') {
channelStatusFilter = '< 2';
} else if (status === 'closed') {
channelStatusFilter = '= 2';
}
const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery} ORDER BY channels.capacity DESC LIMIT ?, ?`;
const [rows]: any = await DB.query(query, [public_key, public_key, index, length]);
const channels = rows.map((row) => this.convertChannel(row));
// Channels originating from node
let query = `
SELECT node2.alias, node2.public_key, channels.status, channels.node1_fee_rate,
channels.capacity, channels.short_id, channels.id
FROM channels
JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
`;
const [channelsFromNode]: any = await DB.query(query, [public_key, index, length]);
// Channels incoming to node
query = `
SELECT node1.alias, node1.public_key, channels.status, channels.node2_fee_rate,
channels.capacity, channels.short_id, channels.id
FROM channels
JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
`;
const [channelsToNode]: any = await DB.query(query, [public_key, index, length]);
let allChannels = channelsFromNode.concat(channelsToNode);
allChannels.sort((a, b) => {
return b.capacity - a.capacity;
});
allChannels = allChannels.slice(index, index + length);
const channels: any[] = []
for (const row of allChannels) {
const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
channels.push({
status: row.status,
capacity: row.capacity ?? 0,
short_id: row.short_id,
id: row.id,
fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
node: {
alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
public_key: row.public_key,
channels: activeChannelsStats.active_channel_count ?? 0,
capacity: activeChannelsStats.capacity ?? 0,
}
});
}
return channels;
} catch (e) {
logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
@ -205,7 +251,12 @@ class ChannelsApi {
if (status === 'closed') {
statusQuery = '= 2';
}
const query = `SELECT COUNT(*) AS count FROM channels WHERE (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery}`;
const query = `
SELECT COUNT(*) AS count
FROM channels
WHERE (node1_public_key = ? OR node2_public_key = ?)
AND status ${statusQuery}
`;
const [rows]: any = await DB.query(query, [public_key, public_key]);
return rows[0]['count'];
} catch (e) {
@ -254,6 +305,135 @@ class ChannelsApi {
},
};
}
/**
* Save or update a channel present in the graph
*/
public async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
const [ txid, vout ] = channel.chan_point.split(':');
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
const query = `INSERT INTO channels
(
id,
short_id,
capacity,
transaction_id,
transaction_vout,
updated_at,
status,
node1_public_key,
node1_base_fee_mtokens,
node1_cltv_delta,
node1_fee_rate,
node1_is_disabled,
node1_max_htlc_mtokens,
node1_min_htlc_mtokens,
node1_updated_at,
node2_public_key,
node2_base_fee_mtokens,
node2_cltv_delta,
node2_fee_rate,
node2_is_disabled,
node2_max_htlc_mtokens,
node2_min_htlc_mtokens,
node2_updated_at
)
VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
capacity = ?,
updated_at = ?,
status = 1,
node1_public_key = ?,
node1_base_fee_mtokens = ?,
node1_cltv_delta = ?,
node1_fee_rate = ?,
node1_is_disabled = ?,
node1_max_htlc_mtokens = ?,
node1_min_htlc_mtokens = ?,
node1_updated_at = ?,
node2_public_key = ?,
node2_base_fee_mtokens = ?,
node2_cltv_delta = ?,
node2_fee_rate = ?,
node2_is_disabled = ?,
node2_max_htlc_mtokens = ?,
node2_min_htlc_mtokens = ?,
node2_updated_at = ?
;`;
await DB.query(query, [
Common.channelShortIdToIntegerId(channel.channel_id),
Common.channelIntegerIdToShortId(channel.channel_id),
channel.capacity,
txid,
vout,
Common.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
Common.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
Common.utcDateToMysql(policy2.last_update),
channel.capacity,
Common.utcDateToMysql(channel.last_update),
channel.node1_pub,
policy1.fee_base_msat,
policy1.time_lock_delta,
policy1.fee_rate_milli_msat,
policy1.disabled,
policy1.max_htlc_msat,
policy1.min_htlc,
Common.utcDateToMysql(policy1.last_update),
channel.node2_pub,
policy2.fee_base_msat,
policy2.time_lock_delta,
policy2.fee_rate_milli_msat,
policy2.disabled,
policy2.max_htlc_msat,
policy2.min_htlc,
Common.utcDateToMysql(policy2.last_update)
]);
}
/**
* Set all channels not in `graphChannelsIds` as inactive (status = 0)
*/
public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
if (graphChannelsIds.length === 0) {
return;
}
try {
const result = await DB.query<ResultSetHeader>(`
UPDATE channels
SET status = 0
WHERE short_id NOT IN (
${graphChannelsIds.map(id => `"${id}"`).join(',')}
)
AND status != 2
`);
if (result[0].changedRows ?? 0 > 0) {
logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
} else {
logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
}
} catch (e) {
logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
}
export default new ChannelsApi();

6
backend/src/api/explorer/channels.routes.ts
View File

@ -46,9 +46,11 @@ class ChannelsRoutes {
}
const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
const status: string = typeof req.query.status === 'string' ? req.query.status : '';
const length = 25;
const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, length, status);
const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.header('X-Total-Count', channelsCount.toString());
res.json(channels);
} catch (e) {

198
backend/src/api/explorer/nodes.api.ts
View File

@ -1,24 +1,18 @@
import logger from '../../logger';
import DB from '../../database';
import { ResultSetHeader } from 'mysql2';
import { ILightningApi } from '../lightning/lightning-api.interface';
class NodesApi {
public async $getNode(public_key: string): Promise<any> {
try {
const query = `
SELECT nodes.*, geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
geo_names_country.names as country, geo_names_subdivision.names as subdivision,
(SELECT Count(*)
FROM channels
WHERE channels.status = 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_closed_count,
(SELECT Count(*)
FROM channels
WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_active_count,
(SELECT Sum(capacity)
FROM channels
WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS capacity,
(SELECT Avg(capacity)
FROM channels
WHERE status = 1 AND ( node1_public_key = ? OR node2_public_key = ? )) AS channels_capacity_avg
// General info
let query = `
SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen,
UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
as_number, city_id, country_id, subdivision_id, longitude, latitude,
geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
geo_names_country.names as country, geo_names_subdivision.names as subdivision
FROM nodes
LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
@ -27,21 +21,70 @@ class NodesApi {
LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
WHERE public_key = ?
`;
const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key]);
if (rows.length > 0) {
rows[0].as_organization = JSON.parse(rows[0].as_organization);
rows[0].subdivision = JSON.parse(rows[0].subdivision);
rows[0].city = JSON.parse(rows[0].city);
rows[0].country = JSON.parse(rows[0].country);
return rows[0];
let [rows]: any[] = await DB.query(query, [public_key]);
if (rows.length === 0) {
throw new Error(`This node does not exist, or our node is not seeing it yet`);
}
return null;
const node = rows[0];
node.as_organization = JSON.parse(node.as_organization);
node.subdivision = JSON.parse(node.subdivision);
node.city = JSON.parse(node.city);
node.country = JSON.parse(node.country);
// Active channels and capacity
const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
node.capacity = activeChannelsStats.capacity ?? 0;
// Opened channels count
query = `
SELECT count(short_id) as opened_channel_count
FROM channels
WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
`;
[rows] = await DB.query(query, [public_key, public_key]);
node.opened_channel_count = 0;
if (rows.length > 0) {
node.opened_channel_count = rows[0].opened_channel_count;
}
// Closed channels count
query = `
SELECT count(short_id) as closed_channel_count
FROM channels
WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
`;
[rows] = await DB.query(query, [public_key, public_key]);
node.closed_channel_count = 0;
if (rows.length > 0) {
node.closed_channel_count = rows[0].closed_channel_count;
}
return node;
} catch (e) {
logger.err('$getNode error: ' + (e instanceof Error ? e.message : e));
logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
throw e;
}
}
public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
const query = `
SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
FROM channels
WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
`;
const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]);
if (rows.length > 0) {
return {
active_channel_count: rows[0].active_channel_count,
capacity: rows[0].capacity
};
} else {
return null;
}
}
public async $getAllNodes(): Promise<any> {
try {
const query = `SELECT * FROM nodes`;
@ -55,7 +98,12 @@ class NodesApi {
public async $getNodeStats(public_key: string): Promise<any> {
try {
const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`;
const query = `
SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels
FROM node_stats
WHERE public_key = ?
ORDER BY added DESC
`;
const [rows]: any = await DB.query(query, [public_key]);
return rows;
} catch (e) {
@ -66,8 +114,19 @@ class NodesApi {
public async $getTopCapacityNodes(): Promise<any> {
try {
const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.capacity DESC LIMIT 10`;
const [rows]: any = await DB.query(query);
let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
const latestDate = rows[0].maxAdded;
const query = `
SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels
FROM node_stats
JOIN nodes ON nodes.public_key = node_stats.public_key
WHERE added = FROM_UNIXTIME(${latestDate})
ORDER BY capacity DESC
LIMIT 10;
`;
[rows] = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
@ -77,8 +136,19 @@ class NodesApi {
public async $getTopChannelsNodes(): Promise<any> {
try {
const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.channels DESC LIMIT 10`;
const [rows]: any = await DB.query(query);
let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
const latestDate = rows[0].maxAdded;
const query = `
SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels
FROM node_stats
JOIN nodes ON nodes.public_key = node_stats.public_key
WHERE added = FROM_UNIXTIME(${latestDate})
ORDER BY channels DESC
LIMIT 10;
`;
[rows] = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
@ -163,8 +233,8 @@ class NodesApi {
public async $getNodesPerCountry(countryId: string) {
try {
const query = `
SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city
FROM node_stats
JOIN (
@ -172,7 +242,7 @@ class NodesApi {
FROM node_stats
GROUP BY public_key
) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
JOIN nodes ON nodes.public_key = node_stats.public_key
RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
WHERE geo_names_country.id = ?
@ -193,8 +263,8 @@ class NodesApi {
public async $getNodesPerISP(ISPId: string) {
try {
const query = `
SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
geo_names_city.names as city, geo_names_country.names as country
FROM node_stats
JOIN (
@ -253,6 +323,66 @@ class NodesApi {
throw e;
}
}
/**
* Save or update a node present in the graph
*/
public async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
const query = `INSERT INTO nodes(
public_key,
first_seen,
updated_at,
alias,
color,
sockets,
status
)
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?, status = 1`;
await DB.query(query, [
node.pub_key,
node.last_update,
node.alias,
node.color,
sockets,
node.last_update,
node.alias,
node.color,
sockets,
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
}
}
/**
* Set all nodes not in `nodesPubkeys` as inactive (status = 0)
*/
public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
if (graphNodesPubkeys.length === 0) {
return;
}
try {
const result = await DB.query<ResultSetHeader>(`
UPDATE nodes
SET status = 0
WHERE public_key NOT IN (
${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
)
`);
if (result[0].changedRows ?? 0 > 0) {
logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
} else {
logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
}
} catch (e) {
logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
}
export default new NodesApi();

6
backend/src/api/explorer/nodes.routes.ts
View File

@ -35,6 +35,9 @@ class NodesRoutes {
res.status(404).send('Node not found');
return;
}
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(node);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
@ -44,6 +47,9 @@ class NodesRoutes {
private async $getHistoricalNodeStats(req: Request, res: Response) {
try {
const statistics = await nodesApi.$getNodeStats(req.params.public_key);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(statistics);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);

272
backend/src/api/lightning/clightning/clightning-client.ts Normal file
View File

@ -0,0 +1,272 @@
// Imported from https://github.com/shesek/lightning-client-js
'use strict';
const methods = [
'addgossip',
'autocleaninvoice',
'check',
'checkmessage',
'close',
'connect',
'createinvoice',
'createinvoicerequest',
'createoffer',
'createonion',
'decode',
'decodepay',
'delexpiredinvoice',
'delinvoice',
'delpay',
'dev-listaddrs',
'dev-rescan-outputs',
'disableoffer',
'disconnect',
'estimatefees',
'feerates',
'fetchinvoice',
'fundchannel',
'fundchannel_cancel',
'fundchannel_complete',
'fundchannel_start',
'fundpsbt',
'getchaininfo',
'getinfo',
'getlog',
'getrawblockbyheight',
'getroute',
'getsharedsecret',
'getutxout',
'help',
'invoice',
'keysend',
'legacypay',
'listchannels',
'listconfigs',
'listforwards',
'listfunds',
'listinvoices',
'listnodes',
'listoffers',
'listpays',
'listpeers',
'listsendpays',
'listtransactions',
'multifundchannel',
'multiwithdraw',
'newaddr',
'notifications',
'offer',
'offerout',
'openchannel_abort',
'openchannel_bump',
'openchannel_init',
'openchannel_signed',
'openchannel_update',
'pay',
'payersign',
'paystatus',
'ping',
'plugin',
'reserveinputs',
'sendinvoice',
'sendonion',
'sendonionmessage',
'sendpay',
'sendpsbt',
'sendrawtransaction',
'setchannelfee',
'signmessage',
'signpsbt',
'stop',
'txdiscard',
'txprepare',
'txsend',
'unreserveinputs',
'utxopsbt',
'waitanyinvoice',
'waitblockheight',
'waitinvoice',
'waitsendpay',
'withdraw'
];
import EventEmitter from 'events';
import { existsSync, statSync } from 'fs';
import { createConnection, Socket } from 'net';
import { homedir } from 'os';
import path from 'path';
import { createInterface, Interface } from 'readline';
import logger from '../../../logger';
import { AbstractLightningApi } from '../lightning-api-abstract-factory';
import { ILightningApi } from '../lightning-api.interface';
import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert';
class LightningError extends Error {
type: string = 'lightning';
message: string = 'lightning-client error';
constructor(error) {
super();
this.type = error.type;
this.message = error.message;
}
}
const defaultRpcPath = path.join(homedir(), '.lightning')
, fStat = (...p) => statSync(path.join(...p))
, fExists = (...p) => existsSync(path.join(...p))
export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
private rpcPath: string;
private reconnectWait: number;
private reconnectTimeout;
private reqcount: number;
private client: Socket;
private rl: Interface;
private clientConnectionPromise: Promise<unknown>;
constructor(rpcPath = defaultRpcPath) {
if (!path.isAbsolute(rpcPath)) {
throw new Error('The rpcPath must be an absolute path');
}
if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) {
// network directory provided, use the lightning-rpc within in
if (fExists(rpcPath, 'lightning-rpc')) {
rpcPath = path.join(rpcPath, 'lightning-rpc');
}
// main data directory provided, default to using the bitcoin mainnet subdirectory
// to be removed in v0.2.0
else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`)
logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`)
rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
}
}
logger.debug(`[CLightningClient] Connecting to ${rpcPath}`);
super();
this.rpcPath = rpcPath;
this.reconnectWait = 0.5;
this.reconnectTimeout = null;
this.reqcount = 0;
const _self = this;
this.client = createConnection(rpcPath).on(
'error', () => {
_self.increaseWaitTime();
_self.reconnect();
}
);
this.rl = createInterface({ input: this.client }).on(
'error', () => {
_self.increaseWaitTime();
_self.reconnect();
}
);
this.clientConnectionPromise = new Promise<void>(resolve => {
_self.client.on('connect', () => {
logger.info(`[CLightningClient] Lightning client connected`);
_self.reconnectWait = 1;
resolve();
});
_self.client.on('end', () => {
logger.err('[CLightningClient] Lightning client connection closed, reconnecting');
_self.increaseWaitTime();
_self.reconnect();
});
_self.client.on('error', error => {
logger.err(`[CLightningClient] Lightning client connection error: ${error}`);
_self.increaseWaitTime();
_self.reconnect();
});
});
this.rl.on('line', line => {
line = line.trim();
if (!line) {
return;
}
const data = JSON.parse(line);
// logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`);
_self.emit('res:' + data.id, data);
});
}
increaseWaitTime(): void {
if (this.reconnectWait >= 16) {
this.reconnectWait = 16;
} else {
this.reconnectWait *= 2;
}
}
reconnect(): void {
const _self = this;
if (this.reconnectTimeout) {
return;
}
this.reconnectTimeout = setTimeout(() => {
logger.debug('[CLightningClient] Trying to reconnect...');
_self.client.connect(_self.rpcPath);
_self.reconnectTimeout = null;
}, this.reconnectWait * 1000);
}
call(method, args = []): Promise<any> {
const _self = this;
const callInt = ++this.reqcount;
const sendObj = {
jsonrpc: '2.0',
method,
params: args,
id: '' + callInt
};
// logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`);
// Wait for the client to connect
return this.clientConnectionPromise
.then(() => new Promise((resolve, reject) => {
// Wait for a response
this.once('res:' + callInt, res => res.error == null
? resolve(res.result)
: reject(new LightningError(res.error))
);
// Send the command
_self.client.write(JSON.stringify(sendObj));
}));
}
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
const listnodes: any[] = await this.call('listnodes');
const listchannels: any[] = await this.call('listchannels');
const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']);
return {
nodes: listnodes['nodes'].map(node => convertNode(node)),
edges: channelsList,
};
}
}
const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase());
methods.forEach(k => {
CLightningClient.prototype[protify(k)] = function (...args: any) {
return this.call(k, args);
};
});

138
backend/src/api/lightning/clightning/clightning-convert.ts Normal file
View File

@ -0,0 +1,138 @@
import { ILightningApi } from '../lightning-api.interface';
import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
import logger from '../../../logger';
/**
* Convert a clightning "listnode" entry to a lnd node entry
*/
export function convertNode(clNode: any): ILightningApi.Node {
return {
alias: clNode.alias ?? '',
color: `#${clNode.color ?? ''}`,
features: [], // TODO parse and return clNode.feature
pub_key: clNode.nodeid,
addresses: clNode.addresses?.map((addr) => {
return {
network: addr.type,
addr: `${addr.address}:${addr.port}`
};
}),
last_update: clNode?.last_timestamp ?? 0,
};
}
/**
* Convert clightning "listchannels" response to lnd "describegraph.edges" format
*/
export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
logger.info('Converting clightning nodes and channels to lnd graph format');
let loggerTimer = new Date().getTime() / 1000;
let channelProcessed = 0;
const consolidatedChannelList: ILightningApi.Channel[] = [];
const clChannelsDict = {};
const clChannelsDictCount = {};
for (const clChannel of clChannels) {
if (!clChannelsDict[clChannel.short_channel_id]) {
clChannelsDict[clChannel.short_channel_id] = clChannel;
clChannelsDictCount[clChannel.short_channel_id] = 1;
} else {
consolidatedChannelList.push(
await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id])
);
delete clChannelsDict[clChannel.short_channel_id];
clChannelsDictCount[clChannel.short_channel_id]++;
}
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`);
loggerTimer = new Date().getTime() / 1000;
}
++channelProcessed;
}
channelProcessed = 0;
const keys = Object.keys(clChannelsDict);
for (const short_channel_id of keys) {
consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id]));
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
loggerTimer = new Date().getTime() / 1000;
}
}
return consolidatedChannelList;
}
export function convertChannelId(channelId): string {
if (channelId.indexOf('/') !== -1) {
channelId = channelId.slice(0, -2);
}
const s = channelId.split('x').map(part => BigInt(part));
return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
}
/**
* Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
* In this case, clightning knows the channel policy for both nodes
*/
async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel> {
const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
const parts = clChannelA.short_channel_id.split('x');
const outputIdx = parts[2];
return {
channel_id: clChannelA.short_channel_id,
capacity: clChannelA.satoshis,
last_update: lastUpdate,
node1_policy: convertPolicy(clChannelA),
node2_policy: convertPolicy(clChannelB),
chan_point: `${tx.txid}:${outputIdx}`,
node1_pub: clChannelA.source,
node2_pub: clChannelB.source,
};
}
/**
* Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
* In this case, clightning knows the channel policy of only one node
*/
async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel> {
const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
const parts = clChannel.short_channel_id.split('x');
const outputIdx = parts[2];
return {
channel_id: clChannel.short_channel_id,
capacity: clChannel.satoshis,
last_update: clChannel.last_update ?? 0,
node1_policy: convertPolicy(clChannel),
node2_policy: null,
chan_point: `${tx.txid}:${outputIdx}`,
node1_pub: clChannel.source,
node2_pub: clChannel.destination,
};
}
/**
* Convert a clightning "listnode" response to a lnd channel policy format
*/
function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
return {
time_lock_delta: 0, // TODO
min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
fee_base_msat: clChannel.base_fee_millisatoshi,
fee_rate_milli_msat: clChannel.fee_per_millionth,
disabled: !clChannel.active,
last_update: clChannel.last_update ?? 0,
};
}

2
backend/src/api/lightning/lightning-api-abstract-factory.ts
View File

@ -1,7 +1,5 @@
import { ILightningApi } from './lightning-api.interface';
export interface AbstractLightningApi {
$getNetworkInfo(): Promise<ILightningApi.NetworkInfo>;
$getNetworkGraph(): Promise<ILightningApi.NetworkGraph>;
$getInfo(): Promise<ILightningApi.Info>;
}

5
backend/src/api/lightning/lightning-api-factory.ts
View File

@ -1,9 +1,12 @@
import config from '../../config';
import CLightningClient from './clightning/clightning-client';
import { AbstractLightningApi } from './lightning-api-abstract-factory';
import LndApi from './lnd/lnd-api';
function lightningApiFactory(): AbstractLightningApi {
switch (config.LIGHTNING.BACKEND) {
switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) {
case 'cln':
return new CLightningClient(config.CLIGHTNING.SOCKET);
case 'lnd':
default:
return new LndApi();

104
backend/src/api/lightning/lightning-api.interface.ts
View File

@ -1,71 +1,85 @@
export namespace ILightningApi {
export interface NetworkInfo {
average_channel_size: number;
channel_count: number;
max_channel_size: number;
median_channel_size: number;
min_channel_size: number;
node_count: number;
not_recently_updated_policy_count: number;
total_capacity: number;
graph_diameter: number;
avg_out_degree: number;
max_out_degree: number;
num_nodes: number;
num_channels: number;
total_network_capacity: string;
avg_channel_size: number;
min_channel_size: string;
max_channel_size: string;
median_channel_size_sat: string;
num_zombie_chans: string;
}
export interface NetworkGraph {
channels: Channel[];
nodes: Node[];
edges: Channel[];
}
export interface Channel {
id: string;
capacity: number;
policies: Policy[];
transaction_id: string;
transaction_vout: number;
updated_at?: string;
channel_id: string;
chan_point: string;
last_update: number;
node1_pub: string;
node2_pub: string;
capacity: string;
node1_policy: RoutingPolicy | null;
node2_policy: RoutingPolicy | null;
}
interface Policy {
public_key: string;
base_fee_mtokens?: string;
cltv_delta?: number;
fee_rate?: number;
is_disabled?: boolean;
max_htlc_mtokens?: string;
min_htlc_mtokens?: string;
updated_at?: string;
export interface RoutingPolicy {
time_lock_delta: number;
min_htlc: string;
fee_base_msat: string;
fee_rate_milli_msat: string;
disabled: boolean;
max_htlc_msat: string;
last_update: number;
}
export interface Node {
last_update: number;
pub_key: string;
alias: string;
addresses: {
network: string;
addr: string;
}[];
color: string;
features: Feature[];
public_key: string;
sockets: string[];
updated_at?: string;
features: { [key: number]: Feature };
}
export interface Info {
chains: string[];
color: string;
active_channels_count: number;
identity_pubkey: string;
alias: string;
current_block_hash: string;
current_block_height: number;
features: Feature[];
is_synced_to_chain: boolean;
is_synced_to_graph: boolean;
latest_block_at: string;
peers_count: number;
pending_channels_count: number;
public_key: string;
uris: any[];
num_pending_channels: number;
num_active_channels: number;
num_peers: number;
block_height: number;
block_hash: string;
synced_to_chain: boolean;
testnet: boolean;
uris: string[];
best_header_timestamp: string;
version: string;
num_inactive_channels: number;
chains: {
chain: string;
network: string;
}[];
color: string;
synced_to_graph: boolean;
features: { [key: number]: Feature };
commit_hash: string;
/** Available on LND since v0.15.0-beta */
require_htlc_interceptor?: boolean;
}
export interface Feature {
bit: number;
is_known: boolean;
name: string;
is_required: boolean;
type?: string;
is_known: boolean;
}
}

46
backend/src/api/lightning/lnd/lnd-api.ts
View File

@ -1,44 +1,40 @@
import axios, { AxiosRequestConfig } from 'axios';
import { Agent } from 'https';
import * as fs from 'fs';
import { AbstractLightningApi } from '../lightning-api-abstract-factory';
import { ILightningApi } from '../lightning-api.interface';
import * as fs from 'fs';
import { authenticatedLndGrpc, getWalletInfo, getNetworkGraph, getNetworkInfo } from 'lightning';
import config from '../../../config';
import logger from '../../../logger';
class LndApi implements AbstractLightningApi {
private lnd: any;
axiosConfig: AxiosRequestConfig = {};
constructor() {
if (!config.LIGHTNING.ENABLED) {
return;
}
try {
const tls = fs.readFileSync(config.LND.TLS_CERT_PATH).toString('base64');
const macaroon = fs.readFileSync(config.LND.MACAROON_PATH).toString('base64');
const { lnd } = authenticatedLndGrpc({
cert: tls,
macaroon: macaroon,
socket: config.LND.SOCKET,
});
this.lnd = lnd;
} catch (e) {
logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e));
process.exit(1);
if (config.LIGHTNING.ENABLED) {
this.axiosConfig = {
headers: {
'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
},
httpsAgent: new Agent({
ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
}),
timeout: 10000
};
}
}
async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
return await getNetworkInfo({ lnd: this.lnd });
return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
.then((response) => response.data);
}
async $getInfo(): Promise<ILightningApi.Info> {
// @ts-ignore
return await getWalletInfo({ lnd: this.lnd });
return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
.then((response) => response.data);
}
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
return await getNetworkGraph({ lnd: this.lnd });
return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
.then((response) => response.data);
}
}

14
backend/src/api/mining/mining.ts
View File

@ -473,7 +473,7 @@ class Mining {
for (const block of blocksWithoutPrices) {
// Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
if (block.height < 68951) {
if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
blocksPrices.push({
height: block.height,
priceId: prices[0].id,
@ -492,11 +492,11 @@ class Mining {
if (blocksPrices.length >= 100000) {
totalInserted += blocksPrices.length;
let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices);
blocksPrices.length = 0;
}
@ -504,11 +504,11 @@ class Mining {
if (blocksPrices.length > 0) {
totalInserted += blocksPrices.length;
let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
}
logger.debug(logStr);
await BlocksRepository.$saveBlockPrices(blocksPrices);
}
} catch (e) {

18
backend/src/config.ts
View File

@ -31,10 +31,16 @@ interface IConfig {
LIGHTNING: {
ENABLED: boolean;
BACKEND: 'lnd' | 'cln' | 'ldk';
TOPOLOGY_FOLDER: string;
STATS_REFRESH_INTERVAL: number;
GRAPH_REFRESH_INTERVAL: number;
};
LND: {
TLS_CERT_PATH: string;
MACAROON_PATH: string;
REST_API_URL: string;
};
CLIGHTNING: {
SOCKET: string;
};
ELECTRUM: {
@ -177,12 +183,18 @@ const defaults: IConfig = {
},
'LIGHTNING': {
'ENABLED': false,
'BACKEND': 'lnd'
'BACKEND': 'lnd',
'TOPOLOGY_FOLDER': '',
'STATS_REFRESH_INTERVAL': 600,
'GRAPH_REFRESH_INTERVAL': 600,
},
'LND': {
'TLS_CERT_PATH': '',
'MACAROON_PATH': '',
'SOCKET': 'localhost:10009',
'REST_API_URL': 'https://localhost:8080',
},
'CLIGHTNING': {
'SOCKET': '',
},
'SOCKS5PROXY': {
'ENABLED': false,
@ -224,6 +236,7 @@ class Config implements IConfig {
BISQ: IConfig['BISQ'];
LIGHTNING: IConfig['LIGHTNING'];
LND: IConfig['LND'];
CLIGHTNING: IConfig['CLIGHTNING'];
SOCKS5PROXY: IConfig['SOCKS5PROXY'];
PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
@ -242,6 +255,7 @@ class Config implements IConfig {
this.BISQ = configs.BISQ;
this.LIGHTNING = configs.LIGHTNING;
this.LND = configs.LND;
this.CLIGHTNING = configs.CLIGHTNING;
this.SOCKS5PROXY = configs.SOCKS5PROXY;
this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;

6
backend/src/database.ts
View File

@ -1,7 +1,7 @@
import config from './config';
import { createPool, Pool, PoolConnection } from 'mysql2/promise';
import logger from './logger';
import { PoolOptions } from 'mysql2/typings/mysql';
import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
class DB {
constructor() {
@ -28,7 +28,9 @@ import { PoolOptions } from 'mysql2/typings/mysql';
}
}
public async query(query, params?) {
public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
{
this.checkDBFlag();
const pool = await this.getPool();
return pool.query(query, params);

28
backend/src/index.ts
View File

@ -28,12 +28,13 @@ import nodesRoutes from './api/explorer/nodes.routes';
import channelsRoutes from './api/explorer/channels.routes';
import generalLightningRoutes from './api/explorer/general.routes';
import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
import nodeSyncService from './tasks/lightning/node-sync.service';
import statisticsRoutes from "./api/statistics/statistics.routes";
import miningRoutes from "./api/mining/mining-routes";
import bisqRoutes from "./api/bisq/bisq.routes";
import liquidRoutes from "./api/liquid/liquid.routes";
import bitcoinRoutes from "./api/bitcoin/bitcoin.routes";
import networkSyncService from './tasks/lightning/network-sync.service';
import statisticsRoutes from './api/statistics/statistics.routes';
import miningRoutes from './api/mining/mining-routes';
import bisqRoutes from './api/bisq/bisq.routes';
import liquidRoutes from './api/liquid/liquid.routes';
import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
import fundingTxFetcher from "./tasks/lightning/sync-tasks/funding-tx-fetcher";
class Server {
private wss: WebSocket.Server | undefined;
@ -136,8 +137,7 @@ class Server {
}
if (config.LIGHTNING.ENABLED) {
nodeSyncService.$startService()
.then(() => lightningStatsUpdater.$startService());
this.$runLightningBackend();
}
this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
@ -183,6 +183,18 @@ class Server {
}
}
async $runLightningBackend() {
try {
await fundingTxFetcher.$init();
await networkSyncService.$startService();
await lightningStatsUpdater.$startService();
} catch(e) {
logger.err(`Lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
await Common.sleep$(1000 * 60);
this.$runLightningBackend();
};
}
setUpWebsocketHandling() {
if (this.wss) {
websocketHandler.setWebsocketServer(this.wss);

29
backend/src/indexer.ts
View File

@ -6,13 +6,12 @@ import logger from './logger';
import HashratesRepository from './repositories/HashratesRepository';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
class Indexer {
runIndexer = true;
indexerRunning = false;
constructor() {
}
tasksRunning: string[] = [];
public reindex() {
if (Common.indexingEnabled()) {
@ -20,6 +19,28 @@ class Indexer {
}
}
public async runSingleTask(task: 'blocksPrices') {
if (!Common.indexingEnabled()) {
return;
}
if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
this.tasksRunning.push(task);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === false || lastestPriceId === null) {
logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
setTimeout(() => {
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
this.runSingleTask('blocksPrices');
}, 10000);
} else {
logger.debug(`Blocks prices indexer will run now`)
await mining.$indexBlockPrices();
this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
}
}
}
public async $run() {
if (!Common.indexingEnabled() || this.runIndexer === false ||
this.indexerRunning === true || mempool.hasPriority()
@ -50,7 +71,7 @@ class Indexer {
return;
}
await mining.$indexBlockPrices();
this.runSingleTask('blocksPrices');
await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
await mining.$generateNetworkHashrateHistory();

5
backend/src/repositories/PricesRepository.ts
View File

@ -27,6 +27,11 @@ class PricesRepository {
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getLatestPriceId(): Promise<number | null> {
const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].id : null;
}
public async $getLatestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;

389
backend/src/tasks/lightning/node-sync.service.ts → backend/src/tasks/lightning/network-sync.service.ts
View File

@ -1,53 +1,43 @@
import { chanNumber } from 'bolt07';
import DB from '../../database';
import logger from '../../logger';
import channelsApi from '../../api/explorer/channels.api';
import bitcoinClient from '../../api/bitcoin/bitcoin-client';
import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
import config from '../../config';
import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import lightningApi from '../../api/lightning/lightning-api-factory';
import { ILightningApi } from '../../api/lightning/lightning-api.interface';
import { $lookupNodeLocation } from './sync-tasks/node-locations';
import lightningApi from '../../api/lightning/lightning-api-factory';
import nodesApi from '../../api/explorer/nodes.api';
import { ResultSetHeader } from 'mysql2';
import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
class NetworkSyncService {
loggerTimer = 0;
class NodeSyncService {
constructor() {}
public async $startService() {
logger.info('Starting node sync service');
public async $startService(): Promise<void> {
logger.info('Starting lightning network sync service');
await this.$runUpdater();
this.loggerTimer = new Date().getTime() / 1000;
setInterval(async () => {
await this.$runUpdater();
}, 1000 * 60 * 60);
await this.$runTasks();
}
private async $runUpdater() {
private async $runTasks(): Promise<void> {
try {
logger.info(`Updating nodes and channels...`);
logger.info(`Updating nodes and channels`);
const networkGraph = await lightningApi.$getNetworkGraph();
for (const node of networkGraph.nodes) {
await this.$saveNode(node);
}
logger.info(`Nodes updated.`);
if (config.MAXMIND.ENABLED) {
await $lookupNodeLocation();
if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
logger.info(`LN Network graph is empty, retrying in 10 seconds`);
setTimeout(() => { this.$runTasks(); }, 10000);
return;
}
const graphChannelsIds: string[] = [];
for (const channel of networkGraph.channels) {
await this.$saveChannel(channel);
graphChannelsIds.push(channel.id);
}
await this.$setChannelsInactive(graphChannelsIds);
logger.info(`Channels updated.`);
await this.$findInactiveNodesAndChannels();
await this.$updateNodesList(networkGraph.nodes);
await this.$updateChannelsList(networkGraph.edges);
await this.$deactivateChannelsWithoutActiveNodes();
await this.$lookUpCreationDateFromChain();
await this.$updateNodeFirstSeen();
await this.$scanForClosedChannels();
@ -56,85 +46,183 @@ class NodeSyncService {
}
} catch (e) {
logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e));
logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
}
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL);
}
/**
* Update the `nodes` table to reflect the current network graph state
*/
private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
let progress = 0;
const graphNodesPubkeys: string[] = [];
for (const node of nodes) {
await nodesApi.$saveNode(node);
graphNodesPubkeys.push(node.pub_key);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating node ${progress}/${nodes.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`${progress} nodes updated`);
// If a channel if not present in the graph, mark it as inactive
nodesApi.$setNodesInactive(graphNodesPubkeys);
if (config.MAXMIND.ENABLED) {
$lookupNodeLocation();
}
}
/**
* Update the `channels` table to reflect the current network graph state
*/
private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise<void> {
try {
let progress = 0;
const graphChannelsIds: string[] = [];
for (const channel of channels) {
await channelsApi.$saveChannel(channel);
graphChannelsIds.push(channel.channel_id);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`${progress} channels updated`);
// If a channel if not present in the graph, mark it as inactive
channelsApi.$setChannelsInactive(graphChannelsIds);
} catch (e) {
logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
}
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
}
// This method look up the creation date of the earliest channel of the node
// and update the node to that date in order to get the earliest first seen date
private async $updateNodeFirstSeen() {
private async $updateNodeFirstSeen(): Promise<void> {
let progress = 0;
let updated = 0;
try {
const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`);
const [nodes]: any[] = await DB.query(`
SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen,
(
SELECT MIN(UNIX_TIMESTAMP(created))
FROM channels
WHERE channels.node1_public_key = nodes.public_key
) AS created1,
(
SELECT MIN(UNIX_TIMESTAMP(created))
FROM channels
WHERE channels.node2_public_key = nodes.public_key
) AS created2
FROM nodes
`);
for (const node of nodes) {
let lowest = 0;
if (node.created1) {
if (node.created2 && node.created2 < node.created1) {
lowest = node.created2;
} else {
lowest = node.created1;
}
} else if (node.created2) {
lowest = node.created2;
}
if (lowest && lowest < node.first_seen) {
const lowest = Math.min(
node.created1 ?? Number.MAX_SAFE_INTEGER,
node.created2 ?? Number.MAX_SAFE_INTEGER,
node.first_seen ?? Number.MAX_SAFE_INTEGER
);
if (lowest < node.first_seen) {
const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
const params = [lowest, node.public_key];
await DB.query(query, params);
}
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating node first seen date ${progress}/${nodes.length}`);
this.loggerTimer = new Date().getTime() / 1000;
++updated;
}
}
logger.info(`Node first seen dates scan complete.`);
logger.info(`Updated ${updated} node first seen dates`);
} catch (e) {
logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $lookUpCreationDateFromChain() {
logger.info(`Running channel creation date lookup...`);
private async $lookUpCreationDateFromChain(): Promise<void> {
let progress = 0;
logger.info(`Running channel creation date lookup`);
try {
const channels = await channelsApi.$getChannelsWithoutCreatedDate();
for (const channel of channels) {
const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1);
await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]);
const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
await DB.query(`
UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
[transaction.timestamp, channel.id]
);
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel creation date ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`Channel creation dates scan complete.`);
logger.info(`Updated ${channels.length} channels' creation date`);
} catch (e) {
logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
}
}
// Looking for channels whos nodes are inactive
private async $findInactiveNodesAndChannels(): Promise<void> {
logger.info(`Running inactive channels scan...`);
/**
* If a channel does not have any active node linked to it, then also
* mark that channel as inactive
*/
private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
logger.info(`Find channels which nodes are offline`);
try {
// @ts-ignore
const [channels]: [ILightningApi.Channel[]] = await DB.query(`
SELECT channels.id
FROM channels
const result = await DB.query<ResultSetHeader>(`
UPDATE channels
SET status = 0
WHERE channels.status = 1
AND (
(
SELECT COUNT(*)
FROM nodes
WHERE nodes.public_key = channels.node1_public_key
AND nodes.status = 1
) = 0
OR (
SELECT COUNT(*)
FROM nodes
WHERE nodes.public_key = channels.node2_public_key
AND nodes.status = 1
) = 0)
`);
for (const channel of channels) {
await this.$updateChannelStatus(channel.id, 0);
if (result[0].changedRows ?? 0 > 0) {
logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
} else {
logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
}
logger.info(`Inactive channels scan complete.`);
} catch (e) {
logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e));
logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $scanForClosedChannels(): Promise<void> {
let progress = 0;
try {
logger.info(`Starting closed channels scan...`);
const channels = await channelsApi.$getChannelsByStatus(0);
@ -148,6 +236,13 @@ class NodeSyncService {
await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
}
}
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Checking if channel has been closed ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`Closed channels scan complete.`);
} catch (e) {
@ -165,6 +260,9 @@ class NodeSyncService {
if (!config.ESPLORA.REST_API_URL) {
return;
}
let progress = 0;
try {
logger.info(`Started running closed channel forensics...`);
const channels = await channelsApi.$getClosedChannelsWithoutReason();
@ -210,6 +308,13 @@ class NodeSyncService {
logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
}
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
this.loggerTimer = new Date().getTime() / 1000;
}
}
logger.info(`Closed channels forensics scan complete.`);
} catch (e) {
@ -264,164 +369,6 @@ class NodeSyncService {
}
return 1;
}
private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
const fromChannel = chanNumber({ channel: channel.id }).number;
try {
const query = `INSERT INTO channels
(
id,
short_id,
capacity,
transaction_id,
transaction_vout,
updated_at,
status,
node1_public_key,
node1_base_fee_mtokens,
node1_cltv_delta,
node1_fee_rate,
node1_is_disabled,
node1_max_htlc_mtokens,
node1_min_htlc_mtokens,
node1_updated_at,
node2_public_key,
node2_base_fee_mtokens,
node2_cltv_delta,
node2_fee_rate,
node2_is_disabled,
node2_max_htlc_mtokens,
node2_min_htlc_mtokens,
node2_updated_at
)
VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
capacity = ?,
updated_at = ?,
status = 1,
node1_public_key = ?,
node1_base_fee_mtokens = ?,
node1_cltv_delta = ?,
node1_fee_rate = ?,
node1_is_disabled = ?,
node1_max_htlc_mtokens = ?,
node1_min_htlc_mtokens = ?,
node1_updated_at = ?,
node2_public_key = ?,
node2_base_fee_mtokens = ?,
node2_cltv_delta = ?,
node2_fee_rate = ?,
node2_is_disabled = ?,
node2_max_htlc_mtokens = ?,
node2_min_htlc_mtokens = ?,
node2_updated_at = ?
;`;
await DB.query(query, [
fromChannel,
channel.id,
channel.capacity,
channel.transaction_id,
channel.transaction_vout,
channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
channel.policies[0].public_key,
channel.policies[0].base_fee_mtokens,
channel.policies[0].cltv_delta,
channel.policies[0].fee_rate,
channel.policies[0].is_disabled,
channel.policies[0].max_htlc_mtokens,
channel.policies[0].min_htlc_mtokens,
channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
channel.policies[1].public_key,
channel.policies[1].base_fee_mtokens,
channel.policies[1].cltv_delta,
channel.policies[1].fee_rate,
channel.policies[1].is_disabled,
channel.policies[1].max_htlc_mtokens,
channel.policies[1].min_htlc_mtokens,
channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
channel.capacity,
channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
channel.policies[0].public_key,
channel.policies[0].base_fee_mtokens,
channel.policies[0].cltv_delta,
channel.policies[0].fee_rate,
channel.policies[0].is_disabled,
channel.policies[0].max_htlc_mtokens,
channel.policies[0].min_htlc_mtokens,
channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
channel.policies[1].public_key,
channel.policies[1].base_fee_mtokens,
channel.policies[1].cltv_delta,
channel.policies[1].fee_rate,
channel.policies[1].is_disabled,
channel.policies[1].max_htlc_mtokens,
channel.policies[1].min_htlc_mtokens,
channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
]);
} catch (e) {
logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $updateChannelStatus(channelShortId: string, status: number): Promise<void> {
try {
await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]);
} catch (e) {
logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
try {
await DB.query(`
UPDATE channels
SET status = 0
WHERE short_id NOT IN (
${graphChannelsIds.map(id => `"${id}"`).join(',')}
)
AND status != 2
`);
} catch (e) {
logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $saveNode(node: ILightningApi.Node): Promise<void> {
try {
const updatedAt = node.updated_at ? this.utcDateToMysql(node.updated_at) : '0000-00-00 00:00:00';
const sockets = node.sockets.join(',');
const query = `INSERT INTO nodes(
public_key,
first_seen,
updated_at,
alias,
color,
sockets
)
VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
await DB.query(query, [
node.public_key,
updatedAt,
node.alias,
node.color,
sockets,
updatedAt,
node.alias,
node.color,
sockets,
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
}
}
private utcDateToMysql(dateString: string): string {
const d = new Date(Date.parse(dateString));
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
}
}
export default new NodeSyncService();
export default new NetworkSyncService();

353
backend/src/tasks/lightning/stats-updater.service.ts
View File

@ -1,350 +1,33 @@
import DB from '../../database';
import logger from '../../logger';
import lightningApi from '../../api/lightning/lightning-api-factory';
import channelsApi from '../../api/explorer/channels.api';
import * as net from 'net';
import LightningStatsImporter from './sync-tasks/stats-importer';
import config from '../../config';
import { Common } from '../../api/common';
class LightningStatsUpdater {
hardCodedStartTime = '2018-01-12';
public async $startService() {
public async $startService(): Promise<void> {
logger.info('Starting Lightning Stats service');
let isInSync = false;
let error: any;
try {
error = null;
isInSync = await this.$lightningIsSynced();
} catch (e) {
error = e;
}
if (!isInSync) {
if (error) {
logger.warn('Was not able to fetch Lightning Node status: ' + (error instanceof Error ? error.message : error) + '. Retrying in 1 minute...');
} else {
logger.notice('The Lightning graph is not yet in sync. Retrying in 1 minute...');
}
setTimeout(() => this.$startService(), 60 * 1000);
return;
}
await this.$populateHistoricalStatistics();
await this.$populateHistoricalNodeStatistics();
setTimeout(() => {
this.$runTasks();
}, this.timeUntilMidnight());
}
private timeUntilMidnight(): number {
const date = new Date();
this.setDateMidnight(date);
date.setUTCHours(24);
return date.getTime() - new Date().getTime();
}
private setDateMidnight(date: Date): void {
date.setUTCHours(0);
date.setUTCMinutes(0);
date.setUTCSeconds(0);
date.setUTCMilliseconds(0);
}
private async $lightningIsSynced(): Promise<boolean> {
const nodeInfo = await lightningApi.$getInfo();
return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph;
await this.$runTasks();
LightningStatsImporter.$run();
}
private async $runTasks(): Promise<void> {
await this.$logLightningStatsDaily();
await this.$logNodeStatsDaily();
await this.$logStatsDaily();
setTimeout(() => {
this.$runTasks();
}, this.timeUntilMidnight());
setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
}
private async $logLightningStatsDaily() {
try {
logger.info(`Running lightning daily stats log...`);
const networkGraph = await lightningApi.$getNetworkGraph();
let total_capacity = 0;
for (const channel of networkGraph.channels) {
if (channel.capacity) {
total_capacity += channel.capacity;
}
}
let clearnetNodes = 0;
let torNodes = 0;
let unannouncedNodes = 0;
for (const node of networkGraph.nodes) {
let isUnnanounced = true;
for (const socket of node.sockets) {
const hasOnion = socket.indexOf('.onion') !== -1;
if (hasOnion) {
torNodes++;
isUnnanounced = false;
}
const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0]));
if (hasClearnet) {
clearnetNodes++;
isUnnanounced = false;
}
}
if (isUnnanounced) {
unannouncedNodes++;
}
}
const channelStats = await channelsApi.$getChannelsStats();
const query = `INSERT INTO lightning_stats(
added,
channel_count,
node_count,
total_capacity,
tor_nodes,
clearnet_nodes,
unannounced_nodes,
avg_capacity,
avg_fee_rate,
avg_base_fee_mtokens,
med_capacity,
med_fee_rate,
med_base_fee_mtokens
)
VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
await DB.query(query, [
networkGraph.channels.length,
networkGraph.nodes.length,
total_capacity,
torNodes,
clearnetNodes,
unannouncedNodes,
channelStats.avgCapacity,
channelStats.avgFeeRate,
channelStats.avgBaseFee,
channelStats.medianCapacity,
channelStats.medianFeeRate,
channelStats.medianBaseFee,
]);
logger.info(`Lightning daily stats done.`);
} catch (e) {
logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $logNodeStatsDaily() {
try {
logger.info(`Running daily node stats update...`);
const query = `
SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left,
c2.channels_capacity_right
FROM nodes
LEFT JOIN (
SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left
FROM channels
WHERE channels.status = 1
GROUP BY node1_public_key
) c1 ON c1.node1_public_key = nodes.public_key
LEFT JOIN (
SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right
FROM channels WHERE channels.status = 1 GROUP BY node2_public_key
) c2 ON c2.node2_public_key = nodes.public_key
`;
const [nodes]: any = await DB.query(query);
for (const node of nodes) {
await DB.query(
`INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW() - INTERVAL 1 DAY, ?, ?)`,
[node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)),
node.channels_count_left + node.channels_count_right]);
}
logger.info('Daily node stats has updated.');
} catch (e) {
logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e));
}
}
// We only run this on first launch
private async $populateHistoricalStatistics() {
try {
const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`);
// Only run if table is empty
if (rows[0]['COUNT(*)'] > 0) {
return;
}
logger.info(`Running historical stats population...`);
const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`);
const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`);
const date: Date = new Date(this.hardCodedStartTime);
const currentDate = new Date();
this.setDateMidnight(currentDate);
while (date < currentDate) {
let totalCapacity = 0;
let channelsCount = 0;
for (const channel of channels) {
if (new Date(channel.created) > date) {
break;
}
if (channel.closing_date === null || new Date(channel.closing_date) > date) {
totalCapacity += channel.capacity;
channelsCount++;
}
}
let nodeCount = 0;
let clearnetNodes = 0;
let torNodes = 0;
let unannouncedNodes = 0;
for (const node of nodes) {
if (new Date(node.first_seen) > date) {
break;
}
nodeCount++;
const sockets = node.sockets.split(',');
let isUnnanounced = true;
for (const socket of sockets) {
const hasOnion = socket.indexOf('.onion') !== -1;
if (hasOnion) {
torNodes++;
isUnnanounced = false;
}
const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':'))));
if (hasClearnet) {
clearnetNodes++;
isUnnanounced = false;
}
}
if (isUnnanounced) {
unannouncedNodes++;
}
}
const query = `INSERT INTO lightning_stats(
added,
channel_count,
node_count,
total_capacity,
tor_nodes,
clearnet_nodes,
unannounced_nodes,
avg_capacity,
avg_fee_rate,
avg_base_fee_mtokens,
med_capacity,
med_fee_rate,
med_base_fee_mtokens
)
VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
const rowTimestamp = date.getTime() / 1000; // Save timestamp for the row insertion down below
date.setUTCDate(date.getUTCDate() + 1);
// Last iteration, save channels stats
const channelStats = (date >= currentDate ? await channelsApi.$getChannelsStats() : undefined);
await DB.query(query, [
rowTimestamp,
channelsCount,
nodeCount,
totalCapacity,
torNodes,
clearnetNodes,
unannouncedNodes,
channelStats?.avgCapacity ?? 0,
channelStats?.avgFeeRate ?? 0,
channelStats?.avgBaseFee ?? 0,
channelStats?.medianCapacity ?? 0,
channelStats?.medianFeeRate ?? 0,
channelStats?.medianBaseFee ?? 0,
]);
}
logger.info('Historical stats populated.');
} catch (e) {
logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e));
}
}
private async $populateHistoricalNodeStatistics() {
try {
const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`);
// Only run if table is empty
if (rows[0]['COUNT(*)'] > 0) {
return;
}
logger.info(`Running historical node stats population...`);
const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`);
for (const node of nodes) {
const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]);
const date: Date = new Date(this.hardCodedStartTime);
const currentDate = new Date();
this.setDateMidnight(currentDate);
let lastTotalCapacity = 0;
let lastChannelsCount = 0;
while (date < currentDate) {
let totalCapacity = 0;
let channelsCount = 0;
for (const channel of channels) {
if (new Date(channel.created) > date) {
break;
}
if (channel.closing_date !== null && new Date(channel.closing_date) < date) {
date.setUTCDate(date.getUTCDate() + 1);
continue;
}
totalCapacity += channel.capacity;
channelsCount++;
}
if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) {
date.setUTCDate(date.getUTCDate() + 1);
continue;
}
lastTotalCapacity = totalCapacity;
lastChannelsCount = channelsCount;
const query = `INSERT INTO node_stats(
public_key,
added,
capacity,
channels
)
VALUES (?, FROM_UNIXTIME(?), ?, ?)`;
await DB.query(query, [
node.public_key,
date.getTime() / 1000,
totalCapacity,
channelsCount,
]);
date.setUTCDate(date.getUTCDate() + 1);
}
logger.debug('Updated node_stats for: ' + node.alias);
}
logger.info('Historical stats populated.');
} catch (e) {
logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e));
}
/**
* Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
*/
private async $logStatsDaily(): Promise<void> {
const date = new Date();
Common.setDateMidnight(date);
const networkGraph = await lightningApi.$getNetworkGraph();
LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
logger.info(`Updated latest network stats`);
}
}

118
backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts Normal file
View File

@ -0,0 +1,118 @@
import { existsSync, promises } from 'fs';
import bitcoinClient from '../../../api/bitcoin/bitcoin-client';
import { Common } from '../../../api/common';
import config from '../../../config';
import logger from '../../../logger';
const fsPromises = promises;
const BLOCKS_CACHE_MAX_SIZE = 100;
const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json';
class FundingTxFetcher {
private running = false;
private blocksCache = {};
private channelNewlyProcessed = 0;
public fundingTxCache = {};
async $init(): Promise<void> {
// Load funding tx disk cache
if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) {
try {
this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8'));
} catch (e) {
logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`);
this.fundingTxCache = {};
}
logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`);
}
}
async $fetchChannelsFundingTxs(channelIds: string[]): Promise<void> {
if (this.running) {
return;
}
this.running = true;
const globalTimer = new Date().getTime() / 1000;
let cacheTimer = new Date().getTime() / 1000;
let loggerTimer = new Date().getTime() / 1000;
let channelProcessed = 0;
this.channelNewlyProcessed = 0;
for (const channelId of channelIds) {
await this.$fetchChannelOpenTx(channelId);
++channelProcessed;
let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
if (elapsedSeconds > 10) {
elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer);
logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` +
`(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` +
`elapsed: ${elapsedSeconds} seconds`
);
loggerTimer = new Date().getTime() / 1000;
}
elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer);
if (elapsedSeconds > 60) {
logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
cacheTimer = new Date().getTime() / 1000;
}
}
if (this.channelNewlyProcessed > 0) {
logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`);
logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
}
this.running = false;
}
public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> {
if (channelId.indexOf('x') === -1) {
channelId = Common.channelIntegerIdToShortId(channelId);
}
if (this.fundingTxCache[channelId]) {
return this.fundingTxCache[channelId];
}
const parts = channelId.split('x');
const blockHeight = parts[0];
const txIdx = parts[1];
const outputIdx = parts[2];
let block = this.blocksCache[blockHeight];
// Fetch it from core
if (!block) {
const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10));
block = await bitcoinClient.getBlock(blockHash, 1);
}
this.blocksCache[block.height] = block;
const blocksCacheHashes = Object.keys(this.blocksCache).sort((a, b) => parseInt(b) - parseInt(a)).reverse();
if (blocksCacheHashes.length > BLOCKS_CACHE_MAX_SIZE) {
for (let i = 0; i < 10; ++i) {
delete this.blocksCache[blocksCacheHashes[i]];
}
}
const txid = block.tx[txIdx];
const rawTx = await bitcoinClient.getRawTransaction(txid);
const tx = await bitcoinClient.decodeRawTransaction(rawTx);
this.fundingTxCache[channelId] = {
timestamp: block.time,
txid: txid,
value: tx.vout[outputIdx].value,
};
++this.channelNewlyProcessed;
return this.fundingTxCache[channelId];
}
}
export default new FundingTxFetcher;

55
backend/src/tasks/lightning/sync-tasks/node-locations.ts
View File

@ -6,7 +6,10 @@ import DB from '../../../database';
import logger from '../../../logger';
export async function $lookupNodeLocation(): Promise<void> {
logger.info(`Running node location updater using Maxmind...`);
let loggerTimer = new Date().getTime() / 1000;
let progress = 0;
logger.info(`Running node location updater using Maxmind`);
try {
const nodes = await nodesApi.$getAllNodes();
const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
@ -18,21 +21,24 @@ export async function $lookupNodeLocation(): Promise<void> {
for (const socket of sockets) {
const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
const hasClearnet = [4, 6].includes(net.isIP(ip));
if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
const city = lookupCity.get(ip);
const asn = lookupAsn.get(ip);
const isp = lookupIsp.get(ip);
if (city && (asn || isp)) {
const query = `UPDATE nodes SET
as_number = ?,
city_id = ?,
country_id = ?,
subdivision_id = ?,
longitude = ?,
latitude = ?,
accuracy_radius = ?
WHERE public_key = ?`;
const query = `
UPDATE nodes SET
as_number = ?,
city_id = ?,
country_id = ?,
subdivision_id = ?,
longitude = ?,
latitude = ?,
accuracy_radius = ?
WHERE public_key = ?
`;
const params = [
isp?.autonomous_system_number ?? asn?.autonomous_system_number,
@ -46,25 +52,25 @@ export async function $lookupNodeLocation(): Promise<void> {
];
await DB.query(query, params);
// Store Continent
if (city.continent?.geoname_id) {
await DB.query(
// Store Continent
if (city.continent?.geoname_id) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
[city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
}
}
// Store Country
if (city.country?.geoname_id) {
await DB.query(
// Store Country
if (city.country?.geoname_id) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
[city.country?.geoname_id, JSON.stringify(city.country?.names)]);
}
}
// Store Country ISO code
if (city.country?.iso_code) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
[city.country?.geoname_id, city.country?.iso_code]);
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`,
[city.country?.geoname_id, city.country?.iso_code]);
}
// Store Division
@ -88,10 +94,17 @@ export async function $lookupNodeLocation(): Promise<void> {
[isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
}
}
++progress;
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
if (elapsedSeconds > 10) {
logger.info(`Updating node location data ${progress}/${nodes.length}`);
loggerTimer = new Date().getTime() / 1000;
}
}
}
}
logger.info(`Node location data updated.`);
logger.info(`${progress} nodes location data updated`);
} catch (e) {
logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
}

411
backend/src/tasks/lightning/sync-tasks/stats-importer.ts Normal file
View File

@ -0,0 +1,411 @@
import DB from '../../../database';
import { promises } from 'fs';
import { XMLParser } from 'fast-xml-parser';
import logger from '../../../logger';
import fundingTxFetcher from './funding-tx-fetcher';
import config from '../../../config';
const fsPromises = promises;
interface Node {
id: string;
timestamp: number;
features: string;
rgb_color: string;
alias: string;
addresses: unknown[];
out_degree: number;
in_degree: number;
}
interface Channel {
channel_id: string;
node1_pub: string;
node2_pub: string;
timestamp: number;
features: string;
fee_base_msat: number;
fee_rate_milli_msat: number;
htlc_minimim_msat: number;
cltv_expiry_delta: number;
htlc_maximum_msat: number;
}
class LightningStatsImporter {
topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
parser = new XMLParser();
async $run(): Promise<void> {
logger.info(`Importing historical lightning stats`);
const [channels]: any[] = await DB.query('SELECT short_id from channels;');
logger.info('Caching funding txs for currently existing channels');
await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
await this.$importHistoricalLightningStats();
}
/**
* Generate LN network stats for one day
*/
public async computeNetworkStats(timestamp: number, networkGraph): Promise<unknown> {
// Node counts and network shares
let clearnetNodes = 0;
let torNodes = 0;
let clearnetTorNodes = 0;
let unannouncedNodes = 0;
for (const node of networkGraph.nodes) {
let hasOnion = false;
let hasClearnet = false;
let isUnnanounced = true;
for (const socket of (node.addresses ?? [])) {
hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network);
hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network);
}
if (hasOnion && hasClearnet) {
clearnetTorNodes++;
isUnnanounced = false;
} else if (hasOnion) {
torNodes++;
isUnnanounced = false;
} else if (hasClearnet) {
clearnetNodes++;
isUnnanounced = false;
}
if (isUnnanounced) {
unannouncedNodes++;
}
}
// Channels and node historical stats
const nodeStats = {};
let capacity = 0;
let avgFeeRate = 0;
let avgBaseFee = 0;
const capacities: number[] = [];
const feeRates: number[] = [];
const baseFees: number[] = [];
const alreadyCountedChannels = {};
for (const channel of networkGraph.edges) {
let short_id = channel.channel_id;
if (short_id.indexOf('/') !== -1) {
short_id = short_id.slice(0, -2);
}
const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id);
if (!tx) {
logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`);
continue;
}
if (!nodeStats[channel.node1_pub]) {
nodeStats[channel.node1_pub] = {
capacity: 0,
channels: 0,
};
}
if (!nodeStats[channel.node2_pub]) {
nodeStats[channel.node2_pub] = {
capacity: 0,
channels: 0,
};
}
if (!alreadyCountedChannels[short_id]) {
capacity += Math.round(tx.value * 100000000);
capacities.push(Math.round(tx.value * 100000000));
alreadyCountedChannels[short_id] = true;
nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000);
nodeStats[channel.node1_pub].channels++;
nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000);
nodeStats[channel.node2_pub].channels++;
}
if (channel.node1_policy !== undefined) { // Coming from the node
for (const policy of [channel.node1_policy, channel.node2_policy]) {
if (policy && policy.fee_rate_milli_msat < 5000) {
avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
}
if (policy && policy.fee_base_msat < 5000) {
avgBaseFee += parseInt(policy.fee_base_msat, 10);
baseFees.push(parseInt(policy.fee_base_msat, 10));
}
}
} else { // Coming from the historical import
if (channel.fee_rate_milli_msat < 5000) {
avgFeeRate += parseInt(channel.fee_rate_milli_msat, 10);
feeRates.push(parseInt(channel.fee_rate_milli_msat), 10);
}
if (channel.fee_base_msat < 5000) {
avgBaseFee += parseInt(channel.fee_base_msat, 10);
baseFees.push(parseInt(channel.fee_base_msat), 10);
}
}
}
avgFeeRate /= Math.max(networkGraph.edges.length, 1);
avgBaseFee /= Math.max(networkGraph.edges.length, 1);
const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)];
const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)];
const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)];
const avgCapacity = Math.round(capacity / Math.max(capacities.length, 1));
let query = `INSERT INTO lightning_stats(
added,
channel_count,
node_count,
total_capacity,
tor_nodes,
clearnet_nodes,
unannounced_nodes,
clearnet_tor_nodes,
avg_capacity,
avg_fee_rate,
avg_base_fee_mtokens,
med_capacity,
med_fee_rate,
med_base_fee_mtokens
)
VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
added = FROM_UNIXTIME(?),
channel_count = ?,
node_count = ?,
total_capacity = ?,
tor_nodes = ?,
clearnet_nodes = ?,
unannounced_nodes = ?,
clearnet_tor_nodes = ?,
avg_capacity = ?,
avg_fee_rate = ?,
avg_base_fee_mtokens = ?,
med_capacity = ?,
med_fee_rate = ?,
med_base_fee_mtokens = ?
`;
await DB.query(query, [
timestamp,
capacities.length,
networkGraph.nodes.length,
capacity,
torNodes,
clearnetNodes,
unannouncedNodes,
clearnetTorNodes,
avgCapacity,
avgFeeRate,
avgBaseFee,
medCapacity,
medFeeRate,
medBaseFee,
timestamp,
capacities.length,
networkGraph.nodes.length,
capacity,
torNodes,
clearnetNodes,
unannouncedNodes,
clearnetTorNodes,
avgCapacity,
avgFeeRate,
avgBaseFee,
medCapacity,
medFeeRate,
medBaseFee,
]);
for (const public_key of Object.keys(nodeStats)) {
query = `INSERT INTO node_stats(
public_key,
added,
capacity,
channels
)
VALUES (?, FROM_UNIXTIME(?), ?, ?)
ON DUPLICATE KEY UPDATE
added = FROM_UNIXTIME(?),
capacity = ?,
channels = ?
`;
await DB.query(query, [
public_key,
timestamp,
nodeStats[public_key].capacity,
nodeStats[public_key].channels,
timestamp,
nodeStats[public_key].capacity,
nodeStats[public_key].channels,
]);
}
return {
added: timestamp,
node_count: networkGraph.nodes.length
};
}
/**
* Import topology files LN historical data into the database
*/
async $importHistoricalLightningStats(): Promise<void> {
let latestNodeCount = 1;
const fileList = await fsPromises.readdir(this.topologiesFolder);
// Insert history from the most recent to the oldest
// This also put the .json cached files first
fileList.sort().reverse();
const [rows]: any[] = await DB.query(`
SELECT UNIX_TIMESTAMP(added) AS added, node_count
FROM lightning_stats
ORDER BY added DESC
`);
const existingStatsTimestamps = {};
for (const row of rows) {
existingStatsTimestamps[row.added] = row;
}
// For logging purpose
let processed = 10;
let totalProcessed = -1;
for (const filename of fileList) {
processed++;
totalProcessed++;
const timestamp = parseInt(filename.split('_')[1], 10);
// Stats exist already, don't calculate/insert them
if (existingStatsTimestamps[timestamp] !== undefined) {
latestNodeCount = existingStatsTimestamps[timestamp].node_count;
continue;
}
logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
const fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
let graph;
if (filename.indexOf('.json') !== -1) {
try {
graph = JSON.parse(fileContent);
} catch (e) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
continue;
}
} else {
graph = this.parseFile(fileContent);
if (!graph) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
continue;
}
await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph));
}
if (timestamp > 1556316000) {
// "No, the reason most likely is just that I started collection in 2019,
// so what I had before that is just the survivors from before, which weren't that many"
const diffRatio = graph.nodes.length / latestNodeCount;
if (diffRatio < 0.9) {
// Ignore drop of more than 90% of the node count as it's probably a missing data point
logger.debug(`Nodes count diff ratio threshold reached, ignore the data for this day ${graph.nodes.length} nodes vs ${latestNodeCount}`);
continue;
}
}
latestNodeCount = graph.nodes.length;
const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
if (processed > 10) {
logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
processed = 0;
} else {
logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
}
await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2)));
const stat = await this.computeNetworkStats(timestamp, graph);
existingStatsTimestamps[timestamp] = stat;
}
logger.info(`Lightning network stats historical import completed`);
}
/**
* Parse the file content into XML, and return a list of nodes and channels
*/
private parseFile(fileContent): any {
const graph = this.parser.parse(fileContent);
if (Object.keys(graph).length === 0) {
return null;
}
const nodes: Node[] = [];
const channels: Channel[] = [];
// If there is only one entry, the parser does not return an array, so we override this
if (!Array.isArray(graph.graphml.graph.node)) {
graph.graphml.graph.node = [graph.graphml.graph.node];
}
if (!Array.isArray(graph.graphml.graph.edge)) {
graph.graphml.graph.edge = [graph.graphml.graph.edge];
}
for (const node of graph.graphml.graph.node) {
if (!node.data) {
continue;
}
const addresses: unknown[] = [];
const sockets = node.data[5].split(',');
for (const socket of sockets) {
const parts = socket.split('://');
addresses.push({
network: parts[0],
addr: parts[1],
});
}
nodes.push({
id: node.data[0],
timestamp: node.data[1],
features: node.data[2],
rgb_color: node.data[3],
alias: node.data[4],
addresses: addresses,
out_degree: node.data[6],
in_degree: node.data[7],
});
}
for (const channel of graph.graphml.graph.edge) {
if (!channel.data) {
continue;
}
channels.push({
channel_id: channel.data[0],
node1_pub: channel.data[1],
node2_pub: channel.data[2],
timestamp: channel.data[3],
features: channel.data[4],
fee_base_msat: channel.data[5],
fee_rate_milli_msat: channel.data[6],
htlc_minimim_msat: channel.data[7],
cltv_expiry_delta: channel.data[8],
htlc_maximum_msat: channel.data[9],
});
}
return {
nodes: nodes,
edges: channels,
};
}
}
export default new LightningStatsImporter;

9
backend/src/tasks/price-updater.ts
View File

@ -1,4 +1,5 @@
import * as fs from 'fs';
import { Common } from '../api/common';
import config from '../config';
import logger from '../logger';
import PricesRepository from '../repositories/PricesRepository';
@ -34,10 +35,10 @@ export interface Prices {
}
class PriceUpdater {
historyInserted: boolean = false;
lastRun: number = 0;
lastHistoricalRun: number = 0;
running: boolean = false;
public historyInserted = false;
lastRun = 0;
lastHistoricalRun = 0;
running = false;
feeds: PriceFeed[] = [];
currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
latestPrices: Prices;

2
frontend/src/app/components/master-page/master-page.component.html
View File

@ -1,7 +1,7 @@
<ng-container *ngIf="{ val: network$ | async } as network">
<header>
<nav class="navbar navbar-expand-md navbar-dark bg-dark">
<a class="navbar-brand" [routerLink]="['/' | relativeUrl]">
<a class="navbar-brand" [ngClass]="{'dual-logos': subdomain}" [routerLink]="['/' | relativeUrl]">
<ng-template [ngIf]="subdomain">
<div class="subdomain_container">
<img [src]="'/api/v1/enterprise/images/' + subdomain + '/logo'" class="subdomain_logo">

8
frontend/src/app/components/master-page/master-page.component.scss
View File

@ -9,6 +9,7 @@ fa-icon {
.navbar {
z-index: 100;
min-height: 64px;
width: 100%;
}
li.nav-item {
@ -86,6 +87,13 @@ li.nav-item {
height: 65px;
}
.navbar-brand.dual-logos {
justify-content: space-between;
@media (max-width: 767.98px) {
width: 100%;
}
}
nav {
box-shadow: 0px 0px 15px 0px #000;
}

38
frontend/src/app/lightning/channels-list/channels-list.component.html
View File

@ -2,24 +2,24 @@
<form [formGroup]="channelStatusForm" class="formRadioGroup float-right">
<div class="btn-group btn-group-toggle" ngbRadioGroup name="radioBasic" formControlName="status">
<label ngbButtonLabel class="btn-primary btn-sm">
<input ngbButton type="radio" [value]="'open'" fragment="open"> Open
<input ngbButton type="radio" [value]="'open'" fragment="open" i18n="open">Open
</label>
<label ngbButtonLabel class="btn-primary btn-sm">
<input ngbButton type="radio" [value]="'closed'" fragment="closed"> Closed
<input ngbButton type="radio" [value]="'closed'" fragment="closed" i18n="closed">Closed
</label>
</div>
</form>
<table class="table table-borderless" *ngIf="response.channels.length > 1">
<table class="table table-borderless" *ngIf="response.channels.length > 0">
<ng-container *ngTemplateOutlet="tableHeader"></ng-container>
<tbody>
<tr *ngFor="let channel of response.channels; let i = index;">
<ng-container *ngTemplateOutlet="tableTemplate; context: { $implicit: channel, node: channel.node_left.public_key === publicKey ? channel.node_right : channel.node_left }"></ng-container>
<ng-container *ngTemplateOutlet="tableTemplate; context: { $implicit: channel, node: channel.node }"></ng-container>
</tr>
</tbody>
</table>
<ngb-pagination *ngIf="response.channels.length > 1" class="pagination-container float-right" [size]="paginationSize" [collectionSize]="response.totalItems" [rotate]="true" [pageSize]="itemsPerPage" [(page)]="page" (pageChange)="pageChange(page)" [maxSize]="paginationMaxSize" [boundaryLinks]="true" [ellipses]="false"></ngb-pagination>
<ngb-pagination *ngIf="response.channels.length > 0" class="pagination-container float-right" [size]="paginationSize" [collectionSize]="response.totalItems" [rotate]="true" [pageSize]="itemsPerPage" [(page)]="page" (pageChange)="pageChange(page)" [maxSize]="paginationMaxSize" [boundaryLinks]="true" [ellipses]="false"></ngb-pagination>
<table class="table table-borderless" *ngIf="response.channels.length === 0">
<div class="d-flex justify-content-center" i18n="lightning.empty-channels-list">No channels to display</div>
@ -30,7 +30,7 @@
<thead>
<th class="alias text-left" i18n="nodes.alias">Node Alias</th>
<th class="alias text-left d-none d-md-table-cell" i18n="channels.transaction">&nbsp;</th>
<th class="alias text-left d-none d-md-table-cell" i18n="nodes.alias">Status</th>
<th class="alias text-left d-none d-md-table-cell" i18n="status">Status</th>
<th class="channels text-left d-none d-md-table-cell" i18n="channels.rate">Fee Rate</th>
<th class="capacity text-right d-none d-md-table-cell" i18n="nodes.capacity">Capacity</th>
<th class="capacity text-right" i18n="channels.id">Channel ID</th>
@ -42,31 +42,41 @@
<div>{{ node.alias || '?' }}</div>
<div class="second-line">
<a [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">
<span>{{ node.public_key | shortenString : 10 }}</span>
<span>{{ node.public_key | shortenString : publicKeySize }}</span>
</a>
<app-clipboard [text]="node.public_key" size="small"></app-clipboard>
</div>
</td>
<td class="alias text-left d-none d-md-table-cell">
<div class="second-line">{{ node.channels }} channels</div>
<div class="second-line"><app-amount [satoshis]="node.capacity" digitsInfo="1.2-2"></app-amount></div>
<div class="second-line">
<app-amount *ngIf="node.capacity > 100000000; else smallnode" [satoshis]="node.capacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
<ng-template #smallnode>
{{ node.capacity | amountShortener: 1 }}
<span class="sats" i18n="shared.sats">sats</span>
</ng-template>
</div>
</td>
<td class="d-none d-md-table-cell">
<span class="badge rounded-pill badge-secondary" *ngIf="channel.status === 0">Inactive</span>
<span class="badge rounded-pill badge-success" *ngIf="channel.status === 1">Active</span>
<span class="badge rounded-pill badge-secondary" *ngIf="channel.status === 0" i18n="lightning.inactive">Inactive</span>
<span class="badge rounded-pill badge-success" *ngIf="channel.status === 1" i18n="lightning.active">Active</span>
<ng-template [ngIf]="channel.status === 2">
<span class="badge rounded-pill badge-secondary" *ngIf="!channel.closing_reason; else closingReason">Closed</span>
<span class="badge rounded-pill badge-secondary" *ngIf="!channel.closing_reason; else closingReason" i18n="lightning.closed">Closed</span>
<ng-template #closingReason>
<app-closing-type [type]="channel.closing_reason"></app-closing-type>
</ng-template>
</ng-template>
</td>
<td class="capacity text-left d-none d-md-table-cell">
{{ node.fee_rate }} <span class="symbol">ppm ({{ node.fee_rate / 10000 | number }}%)</span>
{{ channel.fee_rate }} <span class="symbol">ppm ({{ channel.fee_rate / 10000 | number }}%)</span>
</td>
<td class="capacity text-right d-none d-md-table-cell">
<app-amount [satoshis]="channel.capacity" digitsInfo="1.2-2"></app-amount>
</td>
<app-amount *ngIf="channel.capacity > 100000000; else smallchannel" [satoshis]="channel.capacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
<ng-template #smallchannel>
{{ channel.capacity | amountShortener: 1 }}
<span class="sats" i18n="shared.sats">sats</span>
</ng-template>
</td>
<td class="capacity text-right">
<a [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">{{ channel.short_id }}</a>
</td>

8
frontend/src/app/lightning/channels-list/channels-list.component.scss
View File

@ -1,3 +1,9 @@
.second-line {
font-size: 12px;
}
}
.sats {
color: #ffffff66;
font-size: 12px;
top: 0px;
}

42
frontend/src/app/lightning/channels-list/channels-list.component.ts
View File

@ -1,7 +1,8 @@
import { ChangeDetectionStrategy, Component, EventEmitter, Input, OnChanges, OnInit, Output } from '@angular/core';
import { FormBuilder, FormGroup } from '@angular/forms';
import { BehaviorSubject, combineLatest, merge, Observable, of } from 'rxjs';
import { map, startWith, switchMap } from 'rxjs/operators';
import { BehaviorSubject, merge, Observable } from 'rxjs';
import { map, switchMap, tap } from 'rxjs/operators';
import { isMobile } from 'src/app/shared/common.utils';
import { LightningApiService } from '../lightning-api.service';
@Component({
@ -18,11 +19,13 @@ export class ChannelsListComponent implements OnInit, OnChanges {
// @ts-ignore
paginationSize: 'sm' | 'lg' = 'md';
paginationMaxSize = 10;
itemsPerPage = 25;
itemsPerPage = 10;
page = 1;
channelsPage$ = new BehaviorSubject<number>(1);
channelStatusForm: FormGroup;
defaultStatus = 'open';
status = 'open';
publicKeySize = 25;
constructor(
private lightningApiService: LightningApiService,
@ -31,9 +34,12 @@ export class ChannelsListComponent implements OnInit, OnChanges {
this.channelStatusForm = this.formBuilder.group({
status: [this.defaultStatus],
});
if (isMobile()) {
this.publicKeySize = 12;
}
}
ngOnInit() {
ngOnInit(): void {
if (document.body.clientWidth < 670) {
this.paginationSize = 'sm';
this.paginationMaxSize = 3;
@ -41,28 +47,36 @@ export class ChannelsListComponent implements OnInit, OnChanges {
}
ngOnChanges(): void {
this.channelStatusForm.get('status').setValue(this.defaultStatus, { emitEvent: false })
this.channelsStatusChangedEvent.emit(this.defaultStatus);
this.channelStatusForm.get('status').setValue(this.defaultStatus, { emitEvent: false });
this.channelsPage$.next(1);
this.channels$ = combineLatest([
this.channels$ = merge(
this.channelsPage$,
this.channelStatusForm.get('status').valueChanges.pipe(startWith(this.defaultStatus))
])
this.channelStatusForm.get('status').valueChanges,
)
.pipe(
switchMap(([page, status]) => {
this.channelsStatusChangedEvent.emit(status);
return this.lightningApiService.getChannelsByNodeId$(this.publicKey, (page -1) * this.itemsPerPage, status);
tap((val) => {
if (typeof val === 'string') {
this.status = val;
this.page = 1;
} else if (typeof val === 'number') {
this.page = val;
}
}),
switchMap(() => {
this.channelsStatusChangedEvent.emit(this.status);
return this.lightningApiService.getChannelsByNodeId$(this.publicKey, (this.page - 1) * this.itemsPerPage, this.status);
}),
map((response) => {
return {
channels: response.body,
totalItems: parseInt(response.headers.get('x-total-count'), 10)
totalItems: parseInt(response.headers.get('x-total-count'), 10) + 1
};
}),
);
}
pageChange(page: number) {
pageChange(page: number): void {
this.channelsPage$.next(page);
}

41
frontend/src/app/lightning/node/node.component.html
View File

@ -2,8 +2,9 @@
<div class="title-container mb-2" *ngIf="!error">
<h1 class="mb-0">{{ node.alias }}</h1>
<span class="tx-link">
<a [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">{{ node.public_key | shortenString : 12
}}</a>
<a [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">
{{ node.public_key | shortenString : publicKeySize }}
</a>
<app-clipboard [text]="node.public_key"></app-clipboard>
</span>
</div>
@ -22,23 +23,23 @@
<table class="table table-borderless table-striped">
<tbody>
<tr>
<td i18n="address.total-received">Total capacity</td>
<td i18n="lightning.active-capacity">Active capacity</td>
<td>
<app-sats [satoshis]="node.capacity"></app-sats>
<app-fiat [value]="node.capacity" digitsInfo="1.0-0"></app-fiat>
</td>
</tr>
<tr>
<td i18n="address.total-sent">Total channels</td>
<td i18n="lightning.active-channels">Active channels</td>
<td>
{{ node.channel_active_count }}
{{ node.active_channel_count }}
</td>
</tr>
<tr>
<td i18n="address.total-received">Average channel size</td>
<td i18n="lightning.active-channels-avg">Average channel size</td>
<td>
<app-sats [satoshis]="node.channels_capacity_avg"></app-sats>
<app-fiat [value]="node.channels_capacity_avg" digitsInfo="1.0-0"></app-fiat>
<app-sats [satoshis]="node.avgCapacity"></app-sats>
<app-fiat [value]="node.avgCapacity" digitsInfo="1.0-0"></app-fiat>
</td>
</tr>
<tr *ngIf="node.country && node.city && node.subdivision">
@ -71,13 +72,13 @@
<tr>
<td i18n="address.total-received">First seen</td>
<td>
<app-timestamp [dateString]="node.first_seen"></app-timestamp>
<app-timestamp [unixTime]="node.first_seen"></app-timestamp>
</td>
</tr>
<tr>
<td i18n="address.total-sent">Last update</td>
<td>
<app-timestamp [dateString]="node.updated_at"></app-timestamp>
<app-timestamp [unixTime]="node.updated_at"></app-timestamp>
</td>
</tr>
<tr>
@ -101,9 +102,7 @@
</div>
<br>
<div class="input-group mb-3" *ngIf="!error && node.socketsObject.length">
<div class="input-group mt-3" *ngIf="!error && node.socketsObject.length">
<div class="d-inline-block" ngbDropdown #myDrop="ngbDropdown"
*ngIf="node.socketsObject.length > 1; else noDropdown">
<button class="btn btn-secondary dropdown-toggle" type="button" aria-expanded="false" ngbDropdownAnchor
@ -132,24 +131,16 @@
</button>
</div>
<br>
<app-nodes-channels-map *ngIf="!error" [style]="'nodepage'" [publicKey]="node.public_key"></app-nodes-channels-map>
<app-node-statistics-chart [publicKey]="node.public_key" *ngIf="!error"></app-node-statistics-chart>
<br>
<div class="d-flex justify-content-between" *ngIf="!error">
<h2>Channels ({{ channelsListStatus === 'open' ? node.channel_active_count : node.channel_closed_count }})</h2>
<div class="d-flex justify-content-end">
<app-toggle [textLeft]="'List'" [textRight]="'Map'" (toggleStatusChanged)="channelsListModeChange($event)"></app-toggle>
</div>
<h2>Channels ({{ channelsListStatus === 'open' ? node.opened_channel_count : node.closed_channel_count }})</h2>
</div>
<app-nodes-channels-map *ngIf="channelsListMode === 'map' && !error" [style]="'nodepage'" [publicKey]="node.public_key">
</app-nodes-channels-map>
<app-channels-list *ngIf="channelsListMode === 'list' && !error" [publicKey]="node.public_key"
<app-channels-list *ngIf="!error" [publicKey]="node.public_key"
(channelsStatusChangedEvent)="onChannelsListStatusChanged($event)"></app-channels-list>
</div>
<br>
<br>

19
frontend/src/app/lightning/node/node.component.ts
View File

@ -5,6 +5,7 @@ import { catchError, map, switchMap } from 'rxjs/operators';
import { SeoService } from 'src/app/services/seo.service';
import { getFlagEmoji } from 'src/app/shared/graphs.utils';
import { LightningApiService } from '../lightning-api.service';
import { isMobile } from '../../shared/common.utils';
@Component({
selector: 'app-node',
@ -18,16 +19,21 @@ export class NodeComponent implements OnInit {
publicKey$: Observable<string>;
selectedSocketIndex = 0;
qrCodeVisible = false;
channelsListMode = 'list';
channelsListStatus: string;
error: Error;
publicKey: string;
publicKeySize = 99;
constructor(
private lightningApiService: LightningApiService,
private activatedRoute: ActivatedRoute,
private seoService: SeoService,
) { }
) {
if (isMobile()) {
this.publicKeySize = 12;
}
}
ngOnInit(): void {
this.node$ = this.activatedRoute.paramMap
@ -59,6 +65,7 @@ export class NodeComponent implements OnInit {
});
}
node.socketsObject = socketsObject;
node.avgCapacity = node.capacity / Math.max(1, node.active_channel_count);
return node;
}),
catchError(err => {
@ -75,14 +82,6 @@ export class NodeComponent implements OnInit {
this.selectedSocketIndex = index;
}
channelsListModeChange(toggle) {
if (toggle === true) {
this.channelsListMode = 'map';
} else {
this.channelsListMode = 'list';
}
}
onChannelsListStatusChanged(e) {
this.channelsListStatus = e;
}

3
frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html
View File

@ -3,9 +3,6 @@
<div *ngIf="style === 'graph'" class="card-header">
<div class="d-flex d-md-block align-items-baseline" style="margin-bottom: -5px">
<span i18n="lightning.nodes-channels-world-map">Lightning nodes channels world map</span>
<button class="btn p-0 pl-2" style="margin: 0 0 4px 0px">
<fa-icon [icon]="['fas', 'download']" [fixedWidth]="true" (click)="onSaveChart()"></fa-icon>
</button>
</div>
<small style="color: #ffffff66" i18n="lightning.tor-nodes-excluded">(Tor nodes excluded)</small>
</div>

3
frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss
View File

@ -20,7 +20,8 @@
}
.full-container.nodepage {
margin-top: 50px;
margin-top: 25px;
margin-bottom: 25px;
}
.full-container.widget {

232
frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts
View File

@ -3,7 +3,6 @@ import { SeoService } from 'src/app/services/seo.service';
import { ApiService } from 'src/app/services/api.service';
import { Observable, switchMap, tap, zip } from 'rxjs';
import { AssetsService } from 'src/app/services/assets.service';
import { download } from 'src/app/shared/graphs.utils';
import { ActivatedRoute, ParamMap, Router } from '@angular/router';
import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe';
import { StateService } from 'src/app/services/state.service';
@ -21,6 +20,11 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
@Input() publicKey: string | undefined;
observable$: Observable<any>;
center: number[] | undefined;
zoom: number | undefined;
channelWidth = 0.6;
channelOpacity = 0.1;
chartInstance = undefined;
chartOptions: EChartsOption = {};
@ -42,6 +46,9 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
ngOnDestroy(): void {}
ngOnInit(): void {
this.center = this.style === 'widget' ? [0, 40] : [0, 5];
this.zoom = this.style === 'widget' ? 3.5 : 1.3;
if (this.style === 'graph') {
this.seoService.setTitle($localize`Lightning nodes channels world map`);
}
@ -52,31 +59,63 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
return zip(
this.assetsService.getWorldMapJson$,
this.apiService.getChannelsGeo$(params.get('public_key') ?? undefined),
[params.get('public_key') ?? undefined]
).pipe(tap((data) => {
registerMap('world', data[0]);
const channelsLoc = [];
const nodes = [];
const nodesPubkeys = {};
let thisNodeGPS: number[] | undefined = undefined;
for (const channel of data[1]) {
channelsLoc.push([[channel[2], channel[3]], [channel[6], channel[7]]]);
if (!thisNodeGPS && data[2] === channel[0]) {
thisNodeGPS = [channel[2], channel[3]];
} else if (!thisNodeGPS && data[2] === channel[4]) {
thisNodeGPS = [channel[6], channel[7]];
}
// We add a bit of noise so nodes at the same location are not all
// on top of each other
let random = Math.random() * 2 * Math.PI;
let random2 = Math.random() * 0.01;
if (!nodesPubkeys[channel[0]]) {
nodes.push({
publicKey: channel[0],
name: channel[1],
value: [channel[2], channel[3]],
});
nodesPubkeys[channel[0]] = true;
nodes.push([
channel[2] + random2 * Math.cos(random),
channel[3] + random2 * Math.sin(random),
1,
channel[0],
channel[1]
]);
nodesPubkeys[channel[0]] = nodes[nodes.length - 1];
}
random = Math.random() * 2 * Math.PI;
random2 = Math.random() * 0.01;
if (!nodesPubkeys[channel[4]]) {
nodes.push({
publicKey: channel[4],
name: channel[5],
value: [channel[6], channel[7]],
});
nodesPubkeys[channel[4]] = true;
nodes.push([
channel[6] + random2 * Math.cos(random),
channel[7] + random2 * Math.sin(random),
1,
channel[4],
channel[5]
]);
nodesPubkeys[channel[4]] = nodes[nodes.length - 1];
}
const channelLoc = [];
channelLoc.push(nodesPubkeys[channel[0]].slice(0, 2));
channelLoc.push(nodesPubkeys[channel[4]].slice(0, 2));
channelsLoc.push(channelLoc);
}
if (this.style === 'nodepage' && thisNodeGPS) {
this.center = [thisNodeGPS[0], thisNodeGPS[1]];
this.zoom = 10;
this.channelWidth = 1;
this.channelOpacity = 1;
}
this.prepareChartOptions(nodes, channelsLoc);
}));
})
@ -98,85 +137,84 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
}
this.chartOptions = {
silent: true,
silent: this.style === 'widget',
title: title ?? undefined,
geo3D: {
map: 'world',
shading: 'color',
tooltip: {},
geo: {
animation: false,
silent: true,
postEffect: {
enable: true,
bloom: {
intensity: 0.1,
}
},
viewControl: {
center: this.style === 'widget' ? [0, 0, -10] : undefined,
minDistance: this.style === 'widget' ? 22 : 0.1,
maxDistance: this.style === 'widget' ? 22 : 60,
distance: this.style === 'widget' ? 22 : 60,
alpha: 90,
panMouseButton: 'left',
rotateMouseButton: undefined,
zoomSensivity: 0.5,
center: this.center,
zoom: this.zoom,
tooltip: {
show: true
},
map: 'world',
roam: this.style === 'widget' ? false : true,
itemStyle: {
color: 'white',
opacity: 0.02,
borderWidth: 1,
borderColor: 'black',
color: '#ffffff44'
},
regionHeight: 0.01,
scaleLimit: {
min: 1.3,
max: 100000,
}
},
series: [
{
// @ts-ignore
type: 'lines3D',
coordinateSystem: 'geo3D',
blendMode: 'lighter',
lineStyle: {
width: 1,
opacity: ['widget', 'graph'].includes(this.style) ? 0.025 : 1,
large: true,
progressive: 200,
type: 'scatter',
data: nodes,
coordinateSystem: 'geo',
geoIndex: 0,
symbolSize: 4,
tooltip: {
backgroundColor: 'rgba(17, 19, 31, 1)',
borderRadius: 4,
shadowColor: 'rgba(0, 0, 0, 0.5)',
textStyle: {
color: '#b1b1b1',
align: 'left',
},
borderColor: '#000',
formatter: (value) => {
const data = value.data;
const alias = data[4].length > 0 ? data[4] : data[3].slice(0, 20);
return `<b style="color: white">${alias}</b>`;
}
},
data: channels
itemStyle: {
color: 'white',
borderColor: 'black',
borderWidth: 2,
opacity: 1,
},
blendMode: 'lighter',
zlevel: 1,
},
{
// @ts-ignore
type: 'scatter3D',
symbol: 'circle',
blendMode: 'lighter',
coordinateSystem: 'geo3D',
symbolSize: 3,
itemStyle: {
color: '#BBFFFF',
opacity: 1,
borderColor: '#FFFFFF00',
large: true,
progressive: 200,
silent: true,
type: 'lines',
coordinateSystem: 'geo',
data: channels,
lineStyle: {
opacity: this.channelOpacity,
width: this.channelWidth,
curveness: 0,
color: '#466d9d',
},
data: nodes,
emphasis: {
label: {
position: 'top',
color: 'white',
fontSize: 16,
formatter: function(value) {
return value.name;
},
show: true,
}
}
},
blendMode: 'lighter',
tooltip: {
show: false,
},
zlevel: 2,
}
]
};
}
@HostListener('window:wheel', ['$event'])
onWindowScroll(e): void {
// Not very smooth when using the mouse
if (this.style === 'widget' && e.target.tagName === 'CANVAS') {
window.scrollBy({left: 0, top: e.deltaY, behavior: 'auto'});
}
}
onChartInit(ec) {
if (this.chartInstance !== undefined) {
return;
@ -192,32 +230,34 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
});
});
}
this.chartInstance.on('click', (e) => {
if (e.data && e.data.publicKey) {
if (e.data) {
this.zone.run(() => {
const url = new RelativeUrlPipe(this.stateService).transform(`/lightning/node/${e.data.publicKey}`);
const url = new RelativeUrlPipe(this.stateService).transform(`/lightning/node/${e.data[3]}`);
this.router.navigate([url]);
});
}
});
}
onSaveChart() {
// @ts-ignore
const prevBottom = this.chartOptions.grid.bottom;
const now = new Date();
// @ts-ignore
this.chartOptions.grid.bottom = 30;
this.chartOptions.backgroundColor = '#11131f';
this.chartInstance.setOption(this.chartOptions);
download(this.chartInstance.getDataURL({
pixelRatio: 2,
excludeComponents: ['dataZoom'],
}), `lightning-nodes-heatmap-clearnet-${Math.round(now.getTime() / 1000)}.svg`);
// @ts-ignore
this.chartOptions.grid.bottom = prevBottom;
this.chartOptions.backgroundColor = 'none';
this.chartInstance.setOption(this.chartOptions);
this.chartInstance.on('georoam', (e) => {
if (!e.zoom || this.style === 'nodepage') {
return;
}
const speed = 0.005;
const chartOptions = {
series: this.chartOptions.series
};
chartOptions.series[1].lineStyle.opacity += e.zoom > 1 ? speed : -speed;
chartOptions.series[1].lineStyle.width += e.zoom > 1 ? speed : -speed;
chartOptions.series[0].symbolSize += e.zoom > 1 ? speed * 10 : -speed * 10;
chartOptions.series[1].lineStyle.opacity = Math.max(0.05, Math.min(0.5, chartOptions.series[1].lineStyle.opacity));
chartOptions.series[1].lineStyle.width = Math.max(0.5, Math.min(1, chartOptions.series[1].lineStyle.width));
chartOptions.series[0].symbolSize = Math.max(4, Math.min(5.5, chartOptions.series[0].symbolSize));
this.chartInstance.setOption(chartOptions);
});
}
}

4
frontend/src/app/services/enterprise.service.ts
View File

@ -24,7 +24,7 @@ export class EnterpriseService {
this.subdomain = subdomain;
this.fetchSubdomainInfo();
this.disableSubnetworks();
} else {
} else if (document.location.hostname === 'mempool.space') {
this.insertMatomo();
}
}
@ -49,7 +49,7 @@ export class EnterpriseService {
},
(error) => {
if (error.status === 404) {
window.location.href = 'https://mempool.space';
window.location.href = 'https://mempool.space' + window.location.pathname;
}
});
}

3
frontend/src/app/shared/common.utils.ts Normal file
View File

@ -0,0 +1,3 @@
export function isMobile() {
return (window.innerWidth <= 767.98);
}

8
production/elements.crontab
View File

@ -1,6 +1,10 @@
# start elements on reboot
@reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidv1 >/dev/null 2>&1
@reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidtestnet >/dev/null 2>&1
# start electrs on reboot
@reboot sleep 90 ; screen -dmS liquidv1 /elements/electrs/electrs-start-liquid
@reboot sleep 90 ; screen -dmS liquidtestnet /elements/electrs/electrs-start-liquidtestnet
6 * * * * cd $HOME/asset_registry_db && git pull origin master >/dev/null 2>&1
6 * * * * cd $HOME/asset_registry_testnet_db && git pull origin master >/dev/null 2>&1
# hourly asset update and electrs restart
6 * * * * cd $HOME/asset_registry_db && git pull --quiet origin master && cd $HOME/asset_registry_testnet_db && git pull --quiet origin master && killall electrs

228
production/install
View File

@ -40,6 +40,9 @@ BISQ_INSTALL=ON
ELEMENTS_INSTALL=ON
CLN_INSTALL=ON
# install UNFURL
UNFURL_INSTALL=ON
# configure 4 network instances
BITCOIN_MAINNET_ENABLE=ON
BITCOIN_MAINNET_MINFEE_ENABLE=ON
@ -50,8 +53,10 @@ ELEMENTS_LIQUID_ENABLE=ON
ELEMENTS_LIQUIDTESTNET_ENABLE=ON
# enable lightmode and disable compaction to fit on 1TB SSD drive
BITCOIN_ELECTRS_INSTALL=ON
BITCOIN_ELECTRS_LIGHT_MODE=ON
BITCOIN_ELECTRS_COMPACTION=OFF
ELEMENTS_ELECTRS_INSTALL=ON
ELEMENTS_ELECTRS_LIGHT_MODE=ON
ELEMENTS_ELECTRS_COMPACTION=OFF
@ -178,7 +183,6 @@ case $OS in
ROOT_USER=root
ROOT_GROUP=wheel
ROOT_HOME=/root
TOR_HOME=/var/db/tor
TOR_CONFIGURATION=/usr/local/etc/tor/torrc
TOR_RESOURCES=/var/db/tor
TOR_PKG=tor
@ -195,7 +199,6 @@ case $OS in
ROOT_USER=root
ROOT_GROUP=root
ROOT_HOME=/root
TOR_HOME=/etc/tor
TOR_CONFIGURATION=/etc/tor/torrc
TOR_RESOURCES=/var/lib/tor
TOR_PKG=tor
@ -285,6 +288,14 @@ BISQ_USER=bisq
BISQ_GROUP=bisq
# bisq home folder, needs about 1GB
BISQ_HOME=/bisq
# tor HS folder
BISQ_TOR_HS=bisq
# Unfurl user/group
UNFURL_USER=unfurl
UNFURL_GROUP=unfurl
# Unfurl home folder
UNFURL_HOME=/unfurl
# liquid user/group
ELEMENTS_USER=elements
@ -295,6 +306,8 @@ ELEMENTS_HOME=/elements
ELECTRS_HOME=/electrs
# elements electrs source/binaries
ELEMENTS_ELECTRS_HOME=${ELEMENTS_HOME}/electrs
# tor HS folder
LIQUID_TOR_HS=liquid
# minfee user/group
MINFEE_USER=minfee
@ -323,6 +336,13 @@ BISQ_REPO_BRANCH=master
BISQ_LATEST_RELEASE=master
echo -n '.'
UNFURL_REPO_URL=https://github.com/mempool/mempool
UNFURL_REPO_NAME=unfurl
UNFURL_REPO_BRANCH=master
#UNFURL_LATEST_RELEASE=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
UNFURL_LATEST_RELEASE=master
echo -n '.'
ELEMENTS_REPO_URL=https://github.com/ElementsProject/elements
ELEMENTS_REPO_NAME=elements
ELEMENTS_REPO_BRANCH=master
@ -359,6 +379,10 @@ DEBIAN_PKG+=(libboost-system-dev libboost-filesystem-dev libboost-chrono-dev lib
DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python3-certbot-nginx rsync ufw)
DEBIAN_PKG+=(geoipupdate)
DEBIAN_UNFURL_PKG=()
DEBIAN_UNFURL_PKG+=(cups chromium-bsu libatk1.0 libatk-bridge2.0 libxkbcommon-dev libxcomposite-dev)
DEBIAN_UNFURL_PKG+=(libxdamage-dev libxrandr-dev libgbm-dev libpango1.0-dev libasound-dev)
# packages needed for mempool ecosystem
FREEBSD_PKG=()
FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim)
@ -729,6 +753,7 @@ Liquid:Enable Elements Liquid:ON
Liquidtestnet:Enable Elements Liquidtestnet:ON
CoreLN:Enable Core Lightning:ON
Bisq:Enable Bisq:ON
Unfurl:Enable Unfurl:ON
EOF
cat $input | sed -e 's/^/"/' -e 's/:/" "/g' -e 's/$/"/' >$output
@ -806,6 +831,17 @@ if grep CoreLN $tempfile >/dev/null 2>&1;then
CLN_INSTALL=ON
else
CLN_INSTALL=OFF
if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then
BITCOIN_ELECTRS_INSTALL=ON
else
BITCOIN_ELECTRS_INSTALL=OFF
fi
if [ "${ELEMENTS_LIQUID_ENABLE}" = ON -o "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
ELEMENTS_ELECTRS_INSTALL=ON
else
ELEMENTS_ELECTRS_INSTALL=OFF
fi
if grep Bisq $tempfile >/dev/null 2>&1;then
@ -816,6 +852,12 @@ else
BISQ_MAINNET_ENABLE=OFF
fi
if grep Unfurl $tempfile >/dev/null 2>&1;then
UNFURL_INSTALL=ON
else
UNFURL_INSTALL=OFF
fi
##################
## dialog part 2 #
##################
@ -963,15 +1005,34 @@ if [ "${TOR_INSTALL}" = ON ];then
osPackageInstall "${TOR_PKG}"
echo "[*] Installing Tor base configuration"
osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/torrc" "${TOR_HOME}/torrc"
osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/torrc" "${TOR_CONFIGURATION}"
osSudo "${ROOT_USER}" sed -i.orig "s!__TOR_RESOURCES__!${TOR_RESOURCES}!" "${TOR_CONFIGURATION}"
echo "[*] Adding Tor HS configuration"
if ! grep "${MEMPOOL_TOR_HS}" /etc/tor/torrc >/dev/null 2>&1;then
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${MEMPOOL_TOR_HS}/ >> ${TOR_CONFIGURATION}"
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:81 >> ${TOR_CONFIGURATION}"
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}"
else
osSudo "${ROOT_USER}" sed -i.orig "s!__TOR_RESOURCES__!${TOR_RESOURCES}!" "${TOR_CONFIGURATION}"
echo "[*] Adding Tor HS configuration for Mempool"
if [ "${MEMPOOL_ENABLE}" = "ON" ];then
if ! grep "${MEMPOOL_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${MEMPOOL_TOR_HS}/ >> ${TOR_CONFIGURATION}"
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:81 >> ${TOR_CONFIGURATION}"
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}"
fi
fi
echo "[*] Adding Tor HS configuration for Bisq"
if [ "${BISQ_ENABLE}" = "ON" ];then
if ! grep "${BISQ_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${BISQ_TOR_HS}/ >> ${TOR_CONFIGURATION}"
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:82 >> ${TOR_CONFIGURATION}"
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}"
fi
fi
echo "[*] Adding Tor HS configuration for Liquid"
if [ "${LIQUID_ENABLE}" = "ON" ];then
if ! grep "${LIQUID_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${LIQUID_TOR_HS}/ >> ${TOR_CONFIGURATION}"
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:83 >> ${TOR_CONFIGURATION}"
osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}"
fi
fi
case $OS in
@ -1097,65 +1158,72 @@ fi
# Bitcoin -> Electrs installation #
###################################
echo "[*] Creating Bitcoin Electrs data folder"
osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}"
osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_ELECTRS_HOME}"
if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_MAINNET_DATA}"
if [ "${BITCOIN_ELECTRS_INSTALL}" = ON ];then
echo "[*] Creating Bitcoin Electrs data folder"
osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}"
osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_ELECTRS_HOME}"
if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_MAINNET_DATA}"
fi
if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_TESTNET_DATA}"
fi
if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then
osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_SIGNET_DATA}"
fi
echo "[*] Cloning Bitcoin Electrs repo from ${BITCOIN_ELECTRS_REPO_URL}"
osSudo "${BITCOIN_USER}" git config --global advice.detachedHead false
osSudo "${BITCOIN_USER}" git clone --branch "${BITCOIN_ELECTRS_REPO_BRANCH}" "${BITCOIN_ELECTRS_REPO_URL}" "${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME}"
echo "[*] Checking out Electrs ${BITCOIN_ELECTRS_LATEST_RELEASE}"
osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME} && git checkout ${BITCOIN_ELECTRS_LATEST_RELEASE}"
case $OS in
FreeBSD)
echo "[*] Installing Rust from pkg install"
;;
Debian)
echo "[*] Installing Rust from rustup.rs"
osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y"
;;
esac
echo "[*] Building Bitcoin Electrs release binary"
osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" || true
case $OS in
FreeBSD)
echo "[*] Patching Bitcoin Electrs code for FreeBSD"
osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\""
osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/new_index/\" && sed -i.bak -e s/Snappy/None/ db.rs && rm db.rs.bak"
osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i.bak -e 's/from_secs(5)/from_secs(1)/' electrs.rs && rm electrs.rs.bak"
;;
Debian)
;;
esac
echo "[*] Building Bitcoin Electrs release binary"
osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version"
fi
if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_TESTNET_DATA}"
fi
if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then
osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_SIGNET_DATA}"
fi
echo "[*] Cloning Bitcoin Electrs repo from ${BITCOIN_ELECTRS_REPO_URL}"
osSudo "${BITCOIN_USER}" git config --global advice.detachedHead false
osSudo "${BITCOIN_USER}" git clone --branch "${BITCOIN_ELECTRS_REPO_BRANCH}" "${BITCOIN_ELECTRS_REPO_URL}" "${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME}"
echo "[*] Checking out Electrs ${BITCOIN_ELECTRS_LATEST_RELEASE}"
osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME} && git checkout ${BITCOIN_ELECTRS_LATEST_RELEASE}"
case $OS in
FreeBSD)
echo "[*] Installing Rust from pkg install"
;;
Debian)
echo "[*] Installing Rust from rustup.rs"
osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y"
;;
esac
echo "[*] Building Bitcoin Electrs release binary"
osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" || true
case $OS in
FreeBSD)
echo "[*] Patching Bitcoin Electrs code for FreeBSD"
osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\""
osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/new_index/\" && sed -i.bak -e s/Snappy/None/ db.rs && rm db.rs.bak"
osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i.bak -e 's/from_secs(5)/from_secs(1)/' electrs.rs && rm electrs.rs.bak"
;;
Debian)
;;
esac
echo "[*] Building Bitcoin Electrs release binary"
osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version"
##################################
# Liquid -> Electrs installation #
##################################
if [ "${ELEMENTS_INSTALL}" = ON ;then
if [ "${ELEMENTS_ELECTRS_INSTALL}" = ON ];then
echo "[*] Creating Liquid Electrs data folder"
osSudo "${ROOT_USER}" mkdir -p "${ELEMENTS_ELECTRS_HOME}"
osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}"
osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_ELECTRS_HOME}"
osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUID_DATA}"
osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUIDTESTNET_DATA}"
if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUID_DATA}"
fi
if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUIDTESTNET_DATA}"
fi
echo "[*] Cloning Liquid Electrs repo from ${ELEMENTS_ELECTRS_REPO_URL}"
osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
@ -1296,6 +1364,50 @@ if [ "${BISQ_INSTALL}" = ON ];then
esac
fi
#######################
# Unfurl installation #
#######################
if [ "${UNFURL_INSTALL}" = ON ];then
echo "[*] Creating Unfurl user"
osGroupCreate "${UNFURL_GROUP}"
osUserCreate "${UNFURL_USER}" "${UNFURL_HOME}" "${UNFURL_GROUP}"
osSudo "${ROOT_USER}" chsh -s `which zsh` "${UNFURL_USER}"
echo "[*] Creating Unfurl folder"
osSudo "${ROOT_USER}" mkdir -p "${UNFURL_HOME}"
osSudo "${ROOT_USER}" chown -R "${UNFURL_USER}:${UNFURL_GROUP}" "${UNFURL_HOME}"
osSudo "${UNFURL_USER}" touch "${UNFURL_HOME}/.zshrc"
echo "[*] Insalling Unfurl source"
case $OS in
FreeBSD)
echo "[*] FIXME: Unfurl must be installed manually on FreeBSD"
;;
Debian)
echo "[*] Installing packages for Unfurl"
osPackageInstall ${DEBIAN_UNFURL_PKG[@]}
echo "[*] Cloning Mempool (Unfurl) repo from ${UNFURL_REPO_URL}"
osSudo "${UNFURL_USER}" git config --global pull.rebase true
osSudo "${UNFURL_USER}" git config --global advice.detachedHead false
osSudo "${UNFURL_USER}" git clone --branch "${UNFURL_REPO_BRANCH}" "${UNFURL_REPO_URL}" "${UNFURL_HOME}/${UNFURL_REPO_NAME}"
osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-build upgrade
osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-kill stop
osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-start start
echo "[*] Installing nvm.sh from GitHub"
osSudo "${UNFURL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh'
echo "[*] Building NodeJS via nvm.sh"
osSudo "${UNFURL_USER}" zsh -c 'source ~/.zshrc ; nvm install v16.16.0 --shared-zlib'
;;
esac
fi
################################
# Bitcoin instance for Mainnet #
################################

13
production/mempool-config.unfurl.json Normal file
View File

@ -0,0 +1,13 @@
{
"SERVER": {
"HOST": "https://mempool.space",
"HTTP_PORT": 8001
},
"MEMPOOL": {
"HTTP_HOST": "https://mempool.space",
"HTTP_PORT": 443
},
"PUPPETEER": {
"CLUSTER_SIZE": 8
}
}

11
production/mempool.crontab
View File

@ -1,3 +1,12 @@
# start on reboot
@reboot sleep 10 ; $HOME/start
37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 &
# start cache warmer on reboot
@reboot sleep 180 ; /mempool/mempool/production/nginx-cache-warmer >/dev/null 2>&1 &
# daily backup
37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 &
# hourly liquid asset update
6 * * * * cd $HOME/liquid/frontend && npm run sync-assets && rsync -av $HOME/liquid/frontend/dist/mempool/browser/en-US/resources/assets* $HOME/public_html/liquid/en-US/resources/ >/dev/null 2>&1

4
production/newsyslog-mempool-backend.conf
View File

@ -1,2 +1,2 @@
/var/log/mempool 640 10 * @T00 C
/var/log/mempool.debug 640 10 * @T00 C
/var/log/mempool 644 10 * @T00 C
/var/log/mempool.debug 644 10 * @T00 C

8
production/torrc
View File

@ -13,11 +13,3 @@ CookieAuthFileGroupReadable 1
HiddenServiceDir __TOR_RESOURCES__/mempool
HiddenServicePort 80 127.0.0.1:81
HiddenServiceVersion 3
HiddenServiceDir __TOR_RESOURCES__/bisq
HiddenServicePort 80 127.0.0.1:82
HiddenServiceVersion 3
HiddenServiceDir __TOR_RESOURCES__/liquid
HiddenServicePort 80 127.0.0.1:83
HiddenServiceVersion 3

62
production/unfurl-build Executable file
View File

@ -0,0 +1,62 @@
#!/usr/bin/env zsh
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin
HOSTNAME=$(hostname)
LOCATION=$(hostname|cut -d . -f2)
LOCKFILE="${HOME}/lock"
REF=$(echo "${1:=origin/master}"|sed -e 's!:!/!')
if [ -f "${LOCKFILE}" ];then
echo "upgrade already running? check lockfile ${LOCKFILE}"
exit 1
fi
# on exit, remove lockfile but preserve exit code
trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT
# create lockfile
touch "${LOCKFILE}"
# notify logged in users
echo "Upgrading unfurler to ${REF}" | wall
update_repo()
{
echo "[*] Upgrading unfurler to ${REF}"
cd "$HOME/unfurl/unfurler" || exit 1
git fetch origin || exit 1
for remote in origin;do
git remote add "${remote}" "https://github.com/${remote}/mempool" >/dev/null 2>&1
git fetch "${remote}" || exit 1
done
if [ $(git tag -l "${REF}") ];then
git reset --hard "tags/${REF}" || exit 1
elif [ $(git branch -r -l "origin/${REF}") ];then
git reset --hard "origin/${REF}" || exit 1
else
git reset --hard "${REF}" || exit 1
fi
export HASH=$(git rev-parse HEAD)
}
build_backend()
{
echo "[*] Building backend for unfurler"
[ -z "${HASH}" ] && exit 1
cd "$HOME/unfurl/unfurler" || exit 1
if [ ! -e "config.json" ];then
cp "${HOME}/unfurl/production/mempool-config.unfurl.json" "config.json"
fi
npm install || exit 1
npm run build || exit 1
}
update_repo
build_backend
# notify everyone
echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev
echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}"
exit 0

2
production/unfurl-kill Executable file
View File

@ -0,0 +1,2 @@
#!/usr/bin/env zsh
killall sh node

6
production/unfurl-start Executable file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env zsh
export NVM_DIR="$HOME/.nvm"
source "$NVM_DIR/nvm.sh"
cd "${HOME}/unfurl/unfurler/" && \
screen -dmS "unfurl" sh -c 'while true;do npm run start-production;sleep 1;done'