diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 2e7e09316..473622ef5 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -1,6 +1,7 @@ { "MEMPOOL": { "NETWORK": "mainnet", + "BACKEND": "electrs", "HTTP_PORT": 8999, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", @@ -8,7 +9,15 @@ }, "ELECTRS": { "REST_API_URL": "http://127.0.0.1:3000", - "POLL_RATE_MS": 2000 + "POLL_RATE_MS": 2000, + "HOST": "127.0.0.1", + "PORT": 50002 + }, + "BITCOIND": { + "HOST": "127.0.0.1", + "PORT": 3306, + "USERNAME": "mempool", + "PASSWORD": "mempool" }, "DATABASE": { "ENABLED": true, diff --git a/backend/package.json b/backend/package.json index 369308e71..fa2c79c11 100644 --- a/backend/package.json +++ b/backend/package.json @@ -27,6 +27,8 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "dependencies": { + "@codewarriorr/electrum-client-js": "^0.1.1", + "@mempool/bitcoin": "^3.0.2", "axios": "^0.21.0", "express": "^4.17.1", "locutus": "^2.0.12", diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts new file mode 100644 index 000000000..01347884a --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -0,0 +1,16 @@ +import { MempoolInfo, Transaction, Block, MempoolEntries, MempoolEntry } from '../../interfaces'; + +export interface AbstractBitcoinApi { + getMempoolInfo(): Promise; + getRawMempool(): Promise; + getRawTransaction(txId: string): Promise; + getBlockHeightTip(): Promise; + getTxIdsForBlock(hash: string): Promise; + getBlockHash(height: number): Promise; + getBlock(hash: string): Promise; + getMempoolEntry(txid: string): Promise; + + // Custom + getRawMempoolVerbose(): Promise; + getRawTransactionBitcond(txId: string): Promise; +} diff --git a/backend/src/api/bitcoin/bitcoin-api-factory.ts b/backend/src/api/bitcoin/bitcoin-api-factory.ts new file mode 100644 index 000000000..f625e99e1 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-api-factory.ts @@ -0,0 +1,19 @@ +import config from '../../config'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import BitcoindElectrsApi from './bitcoind-electrs-api'; +import BitcoindApi from './bitcoind-api'; +import ElectrsApi from './electrs-api'; + +function bitcoinApiFactory(): AbstractBitcoinApi { + switch (config.MEMPOOL.BACKEND) { + case 'electrs': + return new ElectrsApi(); + case 'bitcoind-electrs': + return new BitcoindElectrsApi(); + case 'bitcoind': + default: + return new BitcoindApi(); + } +} + +export default bitcoinApiFactory(); diff --git a/backend/src/api/bitcoin/bitcoind-api.ts b/backend/src/api/bitcoin/bitcoind-api.ts new file mode 100644 index 000000000..d9c7cb4a3 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoind-api.ts @@ -0,0 +1,83 @@ +import config from '../../config'; +import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry } from '../../interfaces'; +import * as bitcoin from '@mempool/bitcoin'; + +class BitcoindApi { + bitcoindClient: any; + + constructor() { + this.bitcoindClient = new bitcoin.Client({ + host: config.BITCOIND.HOST, + port: config.BITCOIND.PORT, + user: config.BITCOIND.USERNAME, + pass: config.BITCOIND.PASSWORD, + timeout: 60000, + }); + } + + getMempoolInfo(): Promise { + return this.bitcoindClient.getMempoolInfo(); + } + + getRawMempool(): Promise { + return this.bitcoindClient.getRawMemPool(); + } + + getRawMempoolVerbose(): Promise { + return this.bitcoindClient.getRawMemPool(true); + } + + getMempoolEntry(txid: string): Promise { + return this.bitcoindClient.getMempoolEntry(txid,); + } + + getRawTransaction(txId: string): Promise { + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: Transaction) => { + transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); + return transaction; + }); + } + + getBlockHeightTip(): Promise { + return this.bitcoindClient.getChainTips() + .then((result) => result[0].height); + } + + getTxIdsForBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash, 1) + .then((rpcBlock: RpcBlock) => { + return rpcBlock.tx; + }); + } + + getBlockHash(height: number): Promise { + return this.bitcoindClient.getBlockHash(height) + } + + getBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash) + .then((rpcBlock: RpcBlock) => { + return { + id: rpcBlock.hash, + height: rpcBlock.height, + version: rpcBlock.version, + timestamp: rpcBlock.time, + bits: rpcBlock.bits, + nonce: rpcBlock.nonce, + difficulty: rpcBlock.difficulty, + merkle_root: rpcBlock.merkleroot, + tx_count: rpcBlock.nTx, + size: rpcBlock.size, + weight: rpcBlock.weight, + previousblockhash: rpcBlock.previousblockhash, + }; + }); + } + + getRawTransactionBitcond(txId: string): Promise { + throw new Error('Method not implemented.'); + } +} + +export default BitcoindApi; diff --git a/backend/src/api/bitcoin/bitcoind-electrs-api.ts b/backend/src/api/bitcoin/bitcoind-electrs-api.ts new file mode 100644 index 000000000..b99321a78 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoind-electrs-api.ts @@ -0,0 +1,108 @@ +import config from '../../config'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { Transaction, Block, MempoolInfo, RpcBlock, MempoolEntries, MempoolEntry } from '../../interfaces'; +import * as bitcoin from '@mempool/bitcoin'; +import * as ElectrumClient from '@codewarriorr/electrum-client-js'; +import logger from '../../logger'; + +class BitcoindElectrsApi implements AbstractBitcoinApi { + bitcoindClient: any; + electrumClient: any; + + constructor() { + this.bitcoindClient = new bitcoin.Client({ + host: config.BITCOIND.HOST, + port: config.BITCOIND.PORT, + user: config.BITCOIND.USERNAME, + pass: config.BITCOIND.PASSWORD, + timeout: 60000, + }); + + this.electrumClient = new ElectrumClient( + config.ELECTRS.HOST, + config.ELECTRS.PORT, + 'ssl' + ); + + this.electrumClient.connect( + 'electrum-client-js', + '1.4' + ) + } + + getMempoolInfo(): Promise { + return this.bitcoindClient.getMempoolInfo(); + } + + getRawMempool(): Promise { + return this.bitcoindClient.getRawMemPool(); + } + + getRawMempoolVerbose(): Promise { + return this.bitcoindClient.getRawMemPool(true); + } + + getMempoolEntry(txid: string): Promise { + return this.bitcoindClient.getMempoolEntry(txid,); + } + + async getRawTransaction(txId: string): Promise { + try { + const transaction: Transaction = await this.electrumClient.blockchain_transaction_get(txId, true); + if (!transaction) { + throw new Error('not found'); + } + transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); + return transaction; + } catch (e) { + logger.debug('getRawTransaction error: ' + (e.message || e)); + throw new Error(e); + } + } + + getRawTransactionBitcond(txId: string): Promise { + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: Transaction) => { + transaction.vout.forEach((vout) => vout.value = vout.value * 100000000); + return transaction; + }); + } + + getBlockHeightTip(): Promise { + return this.bitcoindClient.getChainTips() + .then((result) => result[0].height); + } + + getTxIdsForBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash, 1) + .then((rpcBlock: RpcBlock) => { + return rpcBlock.tx; + }); + } + + getBlockHash(height: number): Promise { + return this.bitcoindClient.getBlockHash(height) + } + + getBlock(hash: string): Promise { + return this.bitcoindClient.getBlock(hash) + .then((rpcBlock: RpcBlock) => { + return { + id: rpcBlock.hash, + height: rpcBlock.height, + version: rpcBlock.version, + timestamp: rpcBlock.time, + bits: rpcBlock.bits, + nonce: rpcBlock.nonce, + difficulty: rpcBlock.difficulty, + merkle_root: rpcBlock.merkleroot, + tx_count: rpcBlock.nTx, + size: rpcBlock.size, + weight: rpcBlock.weight, + previousblockhash: rpcBlock.previousblockhash, + }; + }); + } +} + +export default BitcoindElectrsApi; diff --git a/backend/src/api/bitcoin/electrs-api.ts b/backend/src/api/bitcoin/electrs-api.ts index 0530f0cfa..a7028ab59 100644 --- a/backend/src/api/bitcoin/electrs-api.ts +++ b/backend/src/api/bitcoin/electrs-api.ts @@ -1,8 +1,9 @@ import config from '../../config'; -import { Transaction, Block, MempoolInfo } from '../../interfaces'; +import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { Transaction, Block, MempoolInfo, MempoolEntry, MempoolEntries } from '../../interfaces'; import axios from 'axios'; -class ElectrsApi { +class ElectrsApi implements AbstractBitcoinApi { constructor() { } @@ -42,15 +43,22 @@ class ElectrsApi { .then((response) => response.data); } - getBlocksFromHeight(height: number): Promise { - return axios.get(config.ELECTRS.REST_API_URL + '/blocks/' + height) - .then((response) => response.data); - } - getBlock(hash: string): Promise { return axios.get(config.ELECTRS.REST_API_URL + '/block/' + hash) .then((response) => response.data); } + + getRawMempoolVerbose(): Promise { + throw new Error('Method not implemented.'); + } + + getMempoolEntry(): Promise { + throw new Error('Method not implemented.'); + } + + getRawTransactionBitcond(txId: string): Promise { + throw new Error('Method not implemented.'); + } } -export default new ElectrsApi(); +export default ElectrsApi; diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index de1d702aa..31e2525f6 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -1,7 +1,8 @@ -import bitcoinApi from './bitcoin/electrs-api'; +import config from '../config'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { Block, TransactionExtended, TransactionMinerInfo } from '../interfaces'; +import { Block, Transaction, TransactionExtended, TransactionMinerInfo } from '../interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; @@ -55,31 +56,38 @@ class Blocks { logger.debug(`New block found (#${this.currentBlockHeight})!`); } + let transactions: TransactionExtended[] = []; + const blockHash = await bitcoinApi.getBlockHash(this.currentBlockHeight); const block = await bitcoinApi.getBlock(blockHash); - const txIds = await bitcoinApi.getTxIdsForBlock(blockHash); + let txIds: string[] = await bitcoinApi.getTxIdsForBlock(blockHash); const mempool = memPool.getMempool(); let found = 0; - let notFound = 0; - - const transactions: TransactionExtended[] = []; for (let i = 0; i < txIds.length; i++) { if (mempool[txIds[i]]) { transactions.push(mempool[txIds[i]]); found++; } else { - logger.debug(`Fetching block tx ${i} of ${txIds.length}`); - const tx = await memPool.getTransactionExtended(txIds[i]); - if (tx) { - transactions.push(tx); + if (config.MEMPOOL.BACKEND === 'electrs') { + logger.debug(`Fetching block tx ${i} of ${txIds.length}`); + const tx = await memPool.getTransactionExtended(txIds[i]); + if (tx) { + transactions.push(tx); + } + } else { // When using bitcoind, just skip parsing past block tx's for now + if (i === 0) { + const tx = await memPool.getTransactionExtended(txIds[i], true); + if (tx) { + transactions.push(tx); + } + } } - notFound++; } } - logger.debug(`${found} of ${txIds.length} found in mempool. ${notFound} not found.`); + logger.debug(`${found} of ${txIds.length} found in mempool. ${txIds.length - found} not found.`); block.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); block.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]); @@ -110,10 +118,13 @@ class Blocks { private stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { return { vin: [{ - scriptsig: tx.vin[0].scriptsig + scriptsig: tx.vin[0].scriptsig || tx.vin[0]['coinbase'] }], vout: tx.vout - .map((vout) => ({ scriptpubkey_address: vout.scriptpubkey_address, value: vout.value })) + .map((vout) => ({ + scriptpubkey_address: vout.scriptpubkey_address || (vout['scriptPubKey']['addresses'] && vout['scriptPubKey']['addresses'][0]) || null, + value: vout.value + })) .filter((vout) => vout.value) }; } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 63256be9b..6e87c9896 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,4 +1,4 @@ -import { Transaction, TransactionExtended, TransactionStripped } from '../interfaces'; +import { TransactionExtended, TransactionStripped } from '../interfaces'; export class Common { static median(numbers: number[]) { @@ -53,7 +53,7 @@ export class Common { txid: tx.txid, fee: tx.fee, weight: tx.weight, - value: tx.vin.reduce((acc, vin) => acc + (vin.prevout ? vin.prevout.value : 0), 0), + value: tx.vout ? tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) : 0, }; } } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 2a509aa3f..6a477d5f7 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,6 +1,6 @@ import config from '../config'; -import bitcoinApi from './bitcoin/electrs-api'; -import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond } from '../interfaces'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; +import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond, MempoolEntry, MempoolEntries } from '../interfaces'; import logger from '../logger'; import { Common } from './common'; @@ -18,6 +18,7 @@ class Mempool { private vBytesPerSecond: number = 0; private mempoolProtection = 0; private latestTransactions: any[] = []; + private mempoolEntriesCache: MempoolEntries | null = null; constructor() { setInterval(this.updateTxPerSecond.bind(this), 1000); @@ -75,20 +76,47 @@ class Mempool { return txTimes; } - public async getTransactionExtended(txId: string): Promise { + public async getTransactionExtended(txId: string, isCoinbase = false): Promise { try { - const transaction: Transaction = await bitcoinApi.getRawTransaction(txId); - return Object.assign({ - vsize: transaction.weight / 4, - feePerVsize: (transaction.fee || 0) / (transaction.weight / 4), - firstSeen: Math.round((new Date().getTime() / 1000)), - }, transaction); + let transaction: Transaction; + if (!isCoinbase && config.MEMPOOL.BACKEND === 'bitcoind-electrs') { + transaction = await bitcoinApi.getRawTransactionBitcond(txId); + } else { + transaction = await bitcoinApi.getRawTransaction(txId); + } + if (config.MEMPOOL.BACKEND !== 'electrs' && !isCoinbase) { + transaction = await this.$appendFeeData(transaction); + } + return this.extendTransaction(transaction); } catch (e) { logger.debug(txId + ' not found'); return false; } } + private async $appendFeeData(transaction: Transaction): Promise { + let mempoolEntry: MempoolEntry; + if (!this.inSync && !this.mempoolEntriesCache) { + this.mempoolEntriesCache = await bitcoinApi.getRawMempoolVerbose(); + } + if (this.mempoolEntriesCache && this.mempoolEntriesCache[transaction.txid]) { + mempoolEntry = this.mempoolEntriesCache[transaction.txid]; + } else { + mempoolEntry = await bitcoinApi.getMempoolEntry(transaction.txid); + } + transaction.fee = mempoolEntry.fees.base * 100000000; + return transaction; + } + + private extendTransaction(transaction: Transaction | MempoolEntry): TransactionExtended { + // @ts-ignore + return Object.assign({ + vsize: Math.round(transaction.weight / 4), + feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)), + firstSeen: Math.round((new Date().getTime() / 1000)), + }, transaction); + } + public async $updateMempool() { logger.debug('Updating mempool'); const start = new Date().getTime(); @@ -169,6 +197,7 @@ class Mempool { if (!this.inSync && transactions.length === Object.keys(newMempool).length) { this.inSync = true; + this.mempoolEntriesCache = null; logger.info('The mempool is now in sync!'); } diff --git a/backend/src/config.ts b/backend/src/config.ts index 7aa1d2a84..6e2e3dc9d 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -3,6 +3,7 @@ const configFile = require('../mempool-config.json'); interface IConfig { MEMPOOL: { NETWORK: 'mainnet' | 'testnet' | 'liquid'; + BACKEND: 'electrs' | 'bitcoind' | 'bitcoind-electrs'; HTTP_PORT: number; SPAWN_CLUSTER_PROCS: number; API_URL_PREFIX: string; @@ -11,7 +12,15 @@ interface IConfig { ELECTRS: { REST_API_URL: string; POLL_RATE_MS: number; + HOST: string; + PORT: number; }; + BITCOIND: { + HOST: string; + PORT: number; + USERNAME: string; + PASSWORD: string; + }, DATABASE: { ENABLED: boolean; HOST: string, @@ -44,6 +53,7 @@ interface IConfig { const defaults: IConfig = { 'MEMPOOL': { 'NETWORK': 'mainnet', + 'BACKEND': 'electrs', 'HTTP_PORT': 8999, 'SPAWN_CLUSTER_PROCS': 0, 'API_URL_PREFIX': '/api/v1/', @@ -51,7 +61,15 @@ const defaults: IConfig = { }, 'ELECTRS': { 'REST_API_URL': 'http://127.0.0.1:3000', - 'POLL_RATE_MS': 2000 + 'POLL_RATE_MS': 2000, + 'HOST': '127.0.0.1', + 'PORT': 3306 + }, + 'BITCOIND': { + 'HOST': "127.0.0.1", + 'PORT': 8332, + 'USERNAME': "mempoo", + 'PASSWORD': "mempool" }, 'DATABASE': { 'ENABLED': true, @@ -85,6 +103,7 @@ const defaults: IConfig = { class Config implements IConfig { MEMPOOL: IConfig['MEMPOOL']; ELECTRS: IConfig['ELECTRS']; + BITCOIND: IConfig['BITCOIND']; DATABASE: IConfig['DATABASE']; STATISTICS: IConfig['STATISTICS']; BISQ_BLOCKS: IConfig['BISQ_BLOCKS']; @@ -95,6 +114,7 @@ class Config implements IConfig { const configs = this.merge(configFile, defaults); this.MEMPOOL = configs.MEMPOOL; this.ELECTRS = configs.ELECTRS; + this.BITCOIND = configs.BITCOIND; this.DATABASE = configs.DATABASE; this.STATISTICS = configs.STATISTICS; this.BISQ_BLOCKS = configs.BISQ_BLOCKS; diff --git a/backend/src/interfaces.ts b/backend/src/interfaces.ts index 159ad868d..e4fab5181 100644 --- a/backend/src/interfaces.ts +++ b/backend/src/interfaces.ts @@ -119,7 +119,7 @@ export interface Block { version: number; timestamp: number; bits: number; - nounce: number; + nonce: number; difficulty: number; merkle_root: string; tx_count: number; @@ -132,8 +132,58 @@ export interface Block { feeRange?: number[]; reward?: number; coinbaseTx?: TransactionMinerInfo; - matchRate: number; - stage: number; + matchRate?: number; +} + +export interface RpcBlock { + hash: string; + confirmations: number; + size: number; + strippedsize: number; + weight: number; + height: number; + version: number, + versionHex: string; + merkleroot: string; + tx: Transaction[]; + time: number; + mediantime: number; + nonce: number; + bits: number; + difficulty: number; + chainwork: string; + nTx: number, + previousblockhash: string; + nextblockhash: string; +} + +export interface MempoolEntries { [txId: string]: MempoolEntry }; + +export interface MempoolEntry { + fees: Fees + vsize: number + weight: number + fee: number + modifiedfee: number + time: number + height: number + descendantcount: number + descendantsize: number + descendantfees: number + ancestorcount: number + ancestorsize: number + ancestorfees: number + wtxid: string + depends: any[] + spentby: any[] + 'bip125-replaceable': boolean +} + +export interface Fees { + base: number + modified: number + ancestor: number + descendant: number } export interface Address {