Merge pull request #1213 from knorrium/fetch_conversion_rates_over_tor

Fetch conversion rates over Tor
This commit is contained in:
wiz 2022-02-06 08:04:50 +00:00 committed by GitHub
commit b383f9fc67
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 310 additions and 8 deletions

View File

@ -88,7 +88,7 @@ JSON:
"BLOCK_WEIGHT_UNITS": 4000000,
"INITIAL_BLOCKS_AMOUNT": 8,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"PRICE_FEED_UPDATE_INTERVAL": 3600,
"PRICE_FEED_UPDATE_INTERVAL": 600,
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": []
},
@ -245,6 +245,39 @@ docker-compose overrides:
BISQ_DATA_PATH: ""
```
JSON:
```
"SOCKS5PROXY": {
"ENABLED": false,
"HOST": "127.0.0.1",
"PORT": "9050",
"USERNAME": "",
"PASSWORD": ""
}
```
docker-compose overrides:
```
SOCKS5PROXY_ENABLED: ""
SOCKS5PROXY_HOST: ""
SOCKS5PROXY_PORT: ""
SOCKS5PROXY_USERNAME: ""
SOCKS5PROXY_PASSWORD: ""
```
JSON:
```
"PRICE_DATA_SERVER": {
"TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
"CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
}
```
docker-compose overrides:
```
PRICE_DATA_SERVER_TOR_URL: ""
PRICE_DATA_SERVER_CLEARNET_URL: ""
```
# Manual Installation

View File

@ -13,7 +13,7 @@
"INITIAL_BLOCKS_AMOUNT": 8,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"INDEXING_BLOCKS_AMOUNT": 1100,
"PRICE_FEED_UPDATE_INTERVAL": 3600,
"PRICE_FEED_UPDATE_INTERVAL": 600,
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": [
"https://mempool.space/resources/pools.json"
@ -61,5 +61,16 @@
"BISQ": {
"ENABLED": false,
"DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
},
"SOCKS5PROXY": {
"ENABLED": false,
"HOST": "127.0.0.1",
"PORT": 9050,
"USERNAME": "",
"PASSWORD": ""
},
"PRICE_DATA_SERVER": {
"TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
"CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
}
}

View File

@ -19,6 +19,7 @@
"locutus": "^2.0.12",
"mysql2": "2.3.3",
"node-worker-threads-pool": "^1.4.3",
"socks-proxy-agent": "^6.1.1",
"typescript": "4.4.4",
"ws": "8.3.0"
},
@ -181,6 +182,38 @@
"node": ">= 0.6"
}
},
"node_modules/agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"dependencies": {
"debug": "4"
},
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/agent-base/node_modules/debug": {
"version": "4.3.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz",
"integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/agent-base/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
@ -731,6 +764,11 @@
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"node_modules/ip": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
},
"node_modules/ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@ -1189,6 +1227,62 @@
"sha.js": "bin.js"
}
},
"node_modules/smart-buffer": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
"engines": {
"node": ">= 6.0.0",
"npm": ">= 3.0.0"
}
},
"node_modules/socks": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.6.1.tgz",
"integrity": "sha512-kLQ9N5ucj8uIcxrDwjm0Jsqk06xdpBjGNQtpXy4Q8/QY2k+fY7nZH8CARy+hkbG+SGAovmzzuauCpBlb8FrnBA==",
"dependencies": {
"ip": "^1.1.5",
"smart-buffer": "^4.1.0"
},
"engines": {
"node": ">= 10.13.0",
"npm": ">= 3.0.0"
}
},
"node_modules/socks-proxy-agent": {
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.1.1.tgz",
"integrity": "sha512-t8J0kG3csjA4g6FTbsMOWws+7R7vuRC8aQ/wy3/1OWmsgwA68zs/+cExQ0koSitUDXqhufF/YJr9wtNMZHw5Ew==",
"dependencies": {
"agent-base": "^6.0.2",
"debug": "^4.3.1",
"socks": "^2.6.1"
},
"engines": {
"node": ">= 10"
}
},
"node_modules/socks-proxy-agent/node_modules/debug": {
"version": "4.3.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz",
"integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/socks-proxy-agent/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@ -1538,6 +1632,29 @@
"negotiator": "0.6.2"
}
},
"agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"requires": {
"debug": "4"
},
"dependencies": {
"debug": {
"version": "4.3.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz",
"integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==",
"requires": {
"ms": "2.1.2"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}
}
},
"ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
@ -1992,6 +2109,11 @@
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"ip": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
},
"ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@ -2352,6 +2474,45 @@
"safe-buffer": "^5.0.1"
}
},
"smart-buffer": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg=="
},
"socks": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.6.1.tgz",
"integrity": "sha512-kLQ9N5ucj8uIcxrDwjm0Jsqk06xdpBjGNQtpXy4Q8/QY2k+fY7nZH8CARy+hkbG+SGAovmzzuauCpBlb8FrnBA==",
"requires": {
"ip": "^1.1.5",
"smart-buffer": "^4.1.0"
}
},
"socks-proxy-agent": {
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.1.1.tgz",
"integrity": "sha512-t8J0kG3csjA4g6FTbsMOWws+7R7vuRC8aQ/wy3/1OWmsgwA68zs/+cExQ0koSitUDXqhufF/YJr9wtNMZHw5Ew==",
"requires": {
"agent-base": "^6.0.2",
"debug": "^4.3.1",
"socks": "^2.6.1"
},
"dependencies": {
"debug": {
"version": "4.3.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz",
"integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==",
"requires": {
"ms": "2.1.2"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}
}
},
"sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",

View File

@ -38,6 +38,7 @@
"locutus": "^2.0.12",
"mysql2": "2.3.3",
"node-worker-threads-pool": "^1.4.3",
"socks-proxy-agent": "^6.1.1",
"typescript": "4.4.4",
"ws": "8.3.0"
},

View File

@ -1,7 +1,9 @@
import logger from '../logger';
import axios from 'axios';
import axios, { AxiosResponse } from 'axios';
import { IConversionRates } from '../mempool.interfaces';
import config from '../config';
import backendInfo from './backend-info';
import { SocksProxyAgent } from 'socks-proxy-agent';
class FiatConversion {
private conversionRates: IConversionRates = {
@ -17,6 +19,11 @@ class FiatConversion {
public startService() {
logger.info('Starting currency rates service');
if (config.SOCKS5PROXY.ENABLED) {
logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICE_DATA_SERVER.TOR_URL}`);
} else {
logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
}
setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
this.updateCurrency();
}
@ -26,12 +33,43 @@ class FiatConversion {
}
private async updateCurrency(): Promise<void> {
const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` };
let fiatConversionUrl: string;
let response: AxiosResponse;
try {
const response = await axios.get('https://price.bisq.wiz.biz/getAllMarketPrices', { timeout: 10000 });
if (config.SOCKS5PROXY.ENABLED) {
let socksOptions: any = {
agentOptions: {
keepAlive: true,
},
host: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
}
const agent = new SocksProxyAgent(socksOptions);
fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL;
logger.debug('Querying currency rates service...');
response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 });
} else {
fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL;
logger.debug('Querying currency rates service...');
response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 });
}
const usd = response.data.data.find((item: any) => item.currencyCode === 'USD');
this.conversionRates = {
'USD': usd.price,
};
logger.debug(`USD Conversion Rate: ${usd.price}`);
if (this.ratesChangedCallback) {
this.ratesChangedCallback(this.conversionRates);
}

View File

@ -51,7 +51,7 @@ interface IConfig {
ENABLED: boolean;
HOST: string;
PORT: number;
MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' |'warn' | 'notice' | 'info' | 'debug';
MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
FACILITY: string;
};
STATISTICS: {
@ -62,6 +62,17 @@ interface IConfig {
ENABLED: boolean;
DATA_PATH: string;
};
SOCKS5PROXY: {
ENABLED: boolean;
HOST: string;
PORT: number;
USERNAME: string;
PASSWORD: string;
};
PRICE_DATA_SERVER: {
TOR_URL: string;
CLEARNET_URL: string;
};
}
const defaults: IConfig = {
@ -79,7 +90,7 @@ const defaults: IConfig = {
'INITIAL_BLOCKS_AMOUNT': 8,
'MEMPOOL_BLOCKS_AMOUNT': 8,
'INDEXING_BLOCKS_AMOUNT': 1100, // 0 = disable indexing, -1 = index all blocks
'PRICE_FEED_UPDATE_INTERVAL': 3600,
'PRICE_FEED_UPDATE_INTERVAL': 600,
'USE_SECOND_NODE_FOR_MINFEE': false,
'EXTERNAL_ASSETS': [
'https://mempool.space/resources/pools.json'
@ -128,6 +139,17 @@ const defaults: IConfig = {
'ENABLED': false,
'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db'
},
'SOCKS5PROXY': {
'ENABLED': false,
'HOST': '127.0.0.1',
'PORT': 9050,
'USERNAME': '',
'PASSWORD': ''
},
"PRICE_DATA_SERVER": {
'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
}
};
class Config implements IConfig {
@ -140,6 +162,8 @@ class Config implements IConfig {
SYSLOG: IConfig['SYSLOG'];
STATISTICS: IConfig['STATISTICS'];
BISQ: IConfig['BISQ'];
SOCKS5PROXY: IConfig['SOCKS5PROXY'];
PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
constructor() {
const configs = this.merge(configFile, defaults);
@ -152,6 +176,8 @@ class Config implements IConfig {
this.SYSLOG = configs.SYSLOG;
this.STATISTICS = configs.STATISTICS;
this.BISQ = configs.BISQ;
this.SOCKS5PROXY = configs.SOCKS5PROXY;
this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
}
merge = (...objects: object[]): IConfig => {

View File

@ -10,7 +10,8 @@
"moduleResolution": "node",
"typeRoots": [
"node_modules/@types"
]
],
"allowSyntheticDefaultImports": true
},
"include": [
"src/**/*.ts"

View File

@ -58,5 +58,16 @@
"BISQ": {
"ENABLED": __BISQ_ENABLED__,
"DATA_PATH": "__BISQ_DATA_PATH__"
},
"SOCKS5PROXY": {
"ENABLED": __SOCKS5PROXY_ENABLED__,
"HOST": "__SOCKS5PROXY_HOST__",
"PORT": "__SOCKS5PROXY_PORT__",
"USERNAME": "__SOCKS5PROXY_USERNAME__",
"PASSWORD": "__SOCKS5PROXY_PASSWORD__"
},
"PRICE_DATA_SERVER": {
"TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
"CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
}
}

View File

@ -14,7 +14,7 @@ __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000}
__MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8}
__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=1100}
__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=3600}
__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
@ -61,6 +61,17 @@ __STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__=${STATISTICS_TX_PER_SECOND_SAMPLE_PER
__BISQ_ENABLED__=${BISQ_ENABLED:=false}
__BISQ_DATA_PATH__=${BISQ_DATA_PATH:=/bisq/statsnode-data/btc_mainnet/db}
# SOCKS5PROXY
__SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false}
__SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost}
__SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050}
__SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""}
__SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""}
# PRICE_DATA_SERVER
__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices}
__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices}
mkdir -p "${__MEMPOOL_CACHE_DIR__}"
sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
@ -115,4 +126,13 @@ sed -i "s/__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__/${__STATISTICS_TX_PER_SECON
sed -i "s/__BISQ_ENABLED__/${__BISQ_ENABLED__}/g" mempool-config.json
sed -i "s!__BISQ_DATA_PATH__!${__BISQ_DATA_PATH__}!g" mempool-config.json
sed -i "s/__SOCKS5PROXY_ENABLED__/${__SOCKS5PROXY_ENABLED__}/g" mempool-config.json
sed -i "s/__SOCKS5PROXY_HOST__/${__SOCKS5PROXY_HOST__}/g" mempool-config.json
sed -i "s/__SOCKS5PROXY_PORT__/${__SOCKS5PROXY_PORT__}/g" mempool-config.json
sed -i "s/__SOCKS5PROXY_USERNAME__/${__SOCKS5PROXY_USERNAME__}/g" mempool-config.json
sed -i "s/__SOCKS5PROXY_PASSWORD__/${__SOCKS5PROXY_PASSWORD__}/g" mempool-config.json
sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json
sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json
node /backend/dist/index.js