mirror of
https://github.com/mempool/mempool.git
synced 2025-04-08 03:48:31 +02:00
Implement Redis cache for block and mempool data
This commit is contained in:
Mononaut
parent
8cfa4ef1a1
commit
5138f9a254
158
backend/package-lock.json
generated
158
backend/package-lock.json
generated
@ -19,6 +19,7 @@
|
||||
"maxmind": "~4.3.11",
|
||||
"mysql2": "~3.5.2",
|
||||
"rust-gbt": "file:./rust-gbt",
|
||||
"redis": "^4.6.6",
|
||||
"socks-proxy-agent": "~7.0.0",
|
||||
"typescript": "~4.9.3",
|
||||
"ws": "~8.13.0"
|
||||
@ -1555,6 +1556,64 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/bloom": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
|
||||
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/client": {
|
||||
"version": "1.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.7.tgz",
|
||||
"integrity": "sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw==",
|
||||
"dependencies": {
|
||||
"cluster-key-slot": "1.1.2",
|
||||
"generic-pool": "3.9.0",
|
||||
"yallist": "4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/client/node_modules/yallist": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
|
||||
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
|
||||
},
|
||||
"node_modules/@redis/graph": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz",
|
||||
"integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/json": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz",
|
||||
"integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/search": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.2.tgz",
|
||||
"integrity": "sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@redis/time-series": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz",
|
||||
"integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==",
|
||||
"peerDependencies": {
|
||||
"@redis/client": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@sinclair/typebox": {
|
||||
"version": "0.25.24",
|
||||
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz",
|
||||
@ -2718,6 +2777,14 @@
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/cluster-key-slot": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
|
||||
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/co": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
|
||||
@ -3678,6 +3745,14 @@
|
||||
"is-property": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/generic-pool": {
|
||||
"version": "3.9.0",
|
||||
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
|
||||
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
|
||||
"engines": {
|
||||
"node": ">= 4"
|
||||
}
|
||||
},
|
||||
"node_modules/gensync": {
|
||||
"version": "1.0.0-beta.2",
|
||||
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
|
||||
@ -6577,6 +6652,19 @@
|
||||
"integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/redis": {
|
||||
"version": "4.6.6",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-4.6.6.tgz",
|
||||
"integrity": "sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA==",
|
||||
"dependencies": {
|
||||
"@redis/bloom": "1.2.0",
|
||||
"@redis/client": "1.5.7",
|
||||
"@redis/graph": "1.1.0",
|
||||
"@redis/json": "1.0.4",
|
||||
"@redis/search": "1.1.2",
|
||||
"@redis/time-series": "1.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
@ -8704,6 +8792,53 @@
|
||||
"fastq": "^1.6.0"
|
||||
}
|
||||
},
|
||||
"@redis/bloom": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
|
||||
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
|
||||
"requires": {}
|
||||
},
|
||||
"@redis/client": {
|
||||
"version": "1.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.7.tgz",
|
||||
"integrity": "sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw==",
|
||||
"requires": {
|
||||
"cluster-key-slot": "1.1.2",
|
||||
"generic-pool": "3.9.0",
|
||||
"yallist": "4.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"yallist": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
|
||||
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"@redis/graph": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz",
|
||||
"integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==",
|
||||
"requires": {}
|
||||
},
|
||||
"@redis/json": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz",
|
||||
"integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==",
|
||||
"requires": {}
|
||||
},
|
||||
"@redis/search": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.2.tgz",
|
||||
"integrity": "sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==",
|
||||
"requires": {}
|
||||
},
|
||||
"@redis/time-series": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz",
|
||||
"integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==",
|
||||
"requires": {}
|
||||
},
|
||||
"@sinclair/typebox": {
|
||||
"version": "0.25.24",
|
||||
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz",
|
||||
@ -9604,6 +9739,11 @@
|
||||
"wrap-ansi": "^7.0.0"
|
||||
}
|
||||
},
|
||||
"cluster-key-slot": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
|
||||
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="
|
||||
},
|
||||
"co": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
|
||||
@ -10332,6 +10472,11 @@
|
||||
"is-property": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"generic-pool": {
|
||||
"version": "3.9.0",
|
||||
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
|
||||
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g=="
|
||||
},
|
||||
"gensync": {
|
||||
"version": "1.0.0-beta.2",
|
||||
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
|
||||
@ -12454,6 +12599,19 @@
|
||||
"integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
|
||||
"dev": true
|
||||
},
|
||||
"redis": {
|
||||
"version": "4.6.6",
|
||||
"resolved": "https://registry.npmjs.org/redis/-/redis-4.6.6.tgz",
|
||||
"integrity": "sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA==",
|
||||
"requires": {
|
||||
"@redis/bloom": "1.2.0",
|
||||
"@redis/client": "1.5.7",
|
||||
"@redis/graph": "1.1.0",
|
||||
"@redis/json": "1.0.4",
|
||||
"@redis/search": "1.1.2",
|
||||
"@redis/time-series": "1.0.4"
|
||||
}
|
||||
},
|
||||
"require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
|
||||
@ -47,13 +47,14 @@
|
||||
"maxmind": "~4.3.11",
|
||||
"mysql2": "~3.5.2",
|
||||
"rust-gbt": "file:./rust-gbt",
|
||||
"redis": "^4.6.6",
|
||||
"socks-proxy-agent": "~7.0.0",
|
||||
"typescript": "~4.9.3",
|
||||
"ws": "~8.13.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "^7.21.3",
|
||||
"@babel/code-frame": "^7.18.6",
|
||||
"@babel/core": "^7.21.3",
|
||||
"@types/compression": "^1.7.2",
|
||||
"@types/crypto-js": "^4.1.1",
|
||||
"@types/express": "^4.17.17",
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
"AUTOMATIC_BLOCK_REINDEXING": false,
|
||||
"POLL_RATE_MS": 3,
|
||||
"CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
|
||||
"CACHE_ENABLED": true,
|
||||
"CLEAR_PROTECTION_MINUTES": 4,
|
||||
"RECOMMENDED_FEE_PERCENTILE": 5,
|
||||
"BLOCK_WEIGHT_UNITS": 6,
|
||||
@ -127,5 +128,9 @@
|
||||
"AUDIT": false,
|
||||
"AUDIT_START_HEIGHT": 774000,
|
||||
"SERVERS": []
|
||||
},
|
||||
"REDIS": {
|
||||
"ENABLED": false,
|
||||
"UNIX_SOCKET_PATH": "/tmp/redis.sock"
|
||||
}
|
||||
}
|
||||
|
||||
@ -23,6 +23,7 @@ describe('Mempool Backend Config', () => {
|
||||
AUTOMATIC_BLOCK_REINDEXING: false,
|
||||
POLL_RATE_MS: 2000,
|
||||
CACHE_DIR: './cache',
|
||||
CACHE_ENABLED: true,
|
||||
CLEAR_PROTECTION_MINUTES: 20,
|
||||
RECOMMENDED_FEE_PERCENTILE: 50,
|
||||
BLOCK_WEIGHT_UNITS: 4000000,
|
||||
@ -127,6 +128,11 @@ describe('Mempool Backend Config', () => {
|
||||
AUDIT_START_HEIGHT: 774000,
|
||||
SERVERS: []
|
||||
});
|
||||
|
||||
expect(config.REDIS).toStrictEqual({
|
||||
ENABLED: false,
|
||||
UNIX_SOCKET_PATH: ''
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -160,6 +166,8 @@ describe('Mempool Backend Config', () => {
|
||||
expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
|
||||
|
||||
expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
|
||||
|
||||
expect(config.REDIS).toStrictEqual(fixture.REDIS);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@ -26,6 +26,8 @@ import PricesRepository from '../repositories/PricesRepository';
|
||||
import priceUpdater from '../tasks/price-updater';
|
||||
import chainTips from './chain-tips';
|
||||
import websocketHandler from './websocket-handler';
|
||||
import redisCache from './redis-cache';
|
||||
import rbfCache from './rbf-cache';
|
||||
|
||||
class Blocks {
|
||||
private blocks: BlockExtended[] = [];
|
||||
@ -804,10 +806,18 @@ class Blocks {
|
||||
if (this.newBlockCallbacks.length) {
|
||||
this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
|
||||
}
|
||||
if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
|
||||
if (config.MEMPOOL.CACHE_ENABLED && !memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
|
||||
diskCache.$saveCacheToDisk();
|
||||
}
|
||||
|
||||
// Update Redis cache
|
||||
if (config.REDIS.ENABLED) {
|
||||
await redisCache.$updateBlocks(this.blocks);
|
||||
await redisCache.$updateBlockSummaries(this.blockSummaries);
|
||||
await redisCache.$removeTransactions(txIds);
|
||||
await rbfCache.updateCache();
|
||||
}
|
||||
|
||||
handledBlocks++;
|
||||
}
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ class DiskCache {
|
||||
};
|
||||
|
||||
constructor() {
|
||||
if (!cluster.isPrimary) {
|
||||
if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
|
||||
return;
|
||||
}
|
||||
process.on('SIGINT', (e) => {
|
||||
@ -39,7 +39,7 @@ class DiskCache {
|
||||
}
|
||||
|
||||
async $saveCacheToDisk(sync: boolean = false): Promise<void> {
|
||||
if (!cluster.isPrimary) {
|
||||
if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
|
||||
return;
|
||||
}
|
||||
if (this.isWritingCache) {
|
||||
@ -175,7 +175,7 @@ class DiskCache {
|
||||
}
|
||||
|
||||
async $loadMempoolCache(): Promise<void> {
|
||||
if (!fs.existsSync(DiskCache.FILE_NAME)) {
|
||||
if (!config.MEMPOOL.CACHE_ENABLED || !fs.existsSync(DiskCache.FILE_NAME)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
||||
@ -9,7 +9,7 @@ import loadingIndicators from './loading-indicators';
|
||||
import bitcoinClient from './bitcoin/bitcoin-client';
|
||||
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
|
||||
import rbfCache from './rbf-cache';
|
||||
import { IEsploraApi } from './bitcoin/esplora-api.interface';
|
||||
import redisCache from './redis-cache';
|
||||
|
||||
class Mempool {
|
||||
private inSync: boolean = false;
|
||||
@ -102,6 +102,10 @@ class Mempool {
|
||||
await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], []);
|
||||
}
|
||||
this.addToSpendMap(Object.values(this.mempoolCache));
|
||||
if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
|
||||
logger.debug('copying mempool from disk cache into Redis');
|
||||
await redisCache.$addTransactions(Object.values(mempoolData));
|
||||
}
|
||||
}
|
||||
|
||||
public async $reloadMempool(expectedCount: number): Promise<MempoolTransactionExtended[]> {
|
||||
@ -318,6 +322,12 @@ class Mempool {
|
||||
loadingIndicators.setProgress('mempool', 100);
|
||||
}
|
||||
|
||||
// Update Redis cache
|
||||
if (config.REDIS.ENABLED) {
|
||||
await redisCache.$addTransactions(newTransactions);
|
||||
await redisCache.$removeTransactions(deletedTransactions.map(tx => tx.txid));
|
||||
}
|
||||
|
||||
const end = new Date().getTime();
|
||||
const time = end - start;
|
||||
logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
|
||||
|
||||
140
backend/src/api/redis-cache.ts
Normal file
140
backend/src/api/redis-cache.ts
Normal file
@ -0,0 +1,140 @@
|
||||
import { createClient } from 'redis';
|
||||
import memPool from './mempool';
|
||||
import blocks from './blocks';
|
||||
import logger from '../logger';
|
||||
import config from '../config';
|
||||
import { BlockExtended, BlockSummary, TransactionExtended } from '../mempool.interfaces';
|
||||
|
||||
class RedisCache {
|
||||
private client;
|
||||
private connected = false;
|
||||
|
||||
constructor() {
|
||||
if (config.REDIS.ENABLED) {
|
||||
const redisConfig = {
|
||||
socket: {
|
||||
path: config.REDIS.UNIX_SOCKET_PATH
|
||||
}
|
||||
};
|
||||
this.client = createClient(redisConfig);
|
||||
this.client.on('error', (e) => {
|
||||
logger.err(`Error in Redis client: ${e instanceof Error ? e.message : e}`);
|
||||
});
|
||||
this.$ensureConnected();
|
||||
}
|
||||
}
|
||||
|
||||
private async $ensureConnected(): Promise<void> {
|
||||
if (!this.connected && config.REDIS.ENABLED) {
|
||||
return this.client.connect().then(() => {
|
||||
this.connected = true;
|
||||
logger.info(`Redis client connected`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async $updateBlocks(blocks: BlockExtended[]) {
|
||||
try {
|
||||
await this.$ensureConnected();
|
||||
await this.client.json.set('blocks', '$', blocks);
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to update blocks in Redis cache: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
}
|
||||
|
||||
async $updateBlockSummaries(summaries: BlockSummary[]) {
|
||||
try {
|
||||
await this.$ensureConnected();
|
||||
await this.client.json.set('block-summaries', '$', summaries);
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to update blocks in Redis cache: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
}
|
||||
|
||||
async $addTransactions(newTransactions: TransactionExtended[]) {
|
||||
try {
|
||||
await this.$ensureConnected();
|
||||
await Promise.all(newTransactions.map(tx => {
|
||||
return this.client.json.set('tx:' + tx.txid, '$', tx);
|
||||
}));
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to add ${newTransactions.length} transactions to Redis cache: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
}
|
||||
|
||||
async $removeTransactions(transactions: string[]) {
|
||||
try {
|
||||
await this.$ensureConnected();
|
||||
await Promise.all(transactions.map(txid => {
|
||||
return this.client.del('tx:' + txid);
|
||||
}));
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to remove ${transactions.length} transactions from Redis cache: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
}
|
||||
|
||||
async $getBlocks(): Promise<BlockExtended[]> {
|
||||
try {
|
||||
await this.$ensureConnected();
|
||||
return this.client.json.get('blocks');
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async $getBlockSummaries(): Promise<BlockSummary[]> {
|
||||
try {
|
||||
await this.$ensureConnected();
|
||||
return this.client.json.get('block-summaries');
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async $getMempool(): Promise<{ [txid: string]: TransactionExtended }> {
|
||||
const mempool = {};
|
||||
try {
|
||||
await this.$ensureConnected();
|
||||
const keys = await this.client.keys('tx:*');
|
||||
const promises: Promise<TransactionExtended[]>[] = [];
|
||||
for (let i = 0; i < keys.length; i += 10000) {
|
||||
const keySlice = keys.slice(i, i + 10000);
|
||||
if (!keySlice.length) {
|
||||
continue;
|
||||
}
|
||||
promises.push(this.client.json.mGet(keySlice, '$').then(chunk => {
|
||||
for (const txs of chunk) {
|
||||
for (const tx of txs) {
|
||||
if (tx) {
|
||||
mempool[tx.txid] = tx;
|
||||
}
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
await Promise.all(promises);
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to retrieve mempool from Redis cache: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
return mempool;
|
||||
}
|
||||
|
||||
async $loadCache() {
|
||||
logger.info('Restoring mempool and blocks data from Redis cache');
|
||||
// Load block data
|
||||
const loadedBlocks = await this.$getBlocks();
|
||||
const loadedBlockSummaries = await this.$getBlockSummaries();
|
||||
// Load mempool
|
||||
const loadedMempool = await this.$getMempool();
|
||||
|
||||
// Set loaded data
|
||||
blocks.setBlocks(loadedBlocks || []);
|
||||
blocks.setBlockSummaries(loadedBlockSummaries || []);
|
||||
await memPool.$setMempool(loadedMempool);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default new RedisCache();
|
||||
@ -12,6 +12,7 @@ interface IConfig {
|
||||
API_URL_PREFIX: string;
|
||||
POLL_RATE_MS: number;
|
||||
CACHE_DIR: string;
|
||||
CACHE_ENABLED: boolean;
|
||||
CLEAR_PROTECTION_MINUTES: number;
|
||||
RECOMMENDED_FEE_PERCENTILE: number;
|
||||
BLOCK_WEIGHT_UNITS: number;
|
||||
@ -137,7 +138,11 @@ interface IConfig {
|
||||
AUDIT: boolean;
|
||||
AUDIT_START_HEIGHT: number;
|
||||
SERVERS: string[];
|
||||
}
|
||||
},
|
||||
REDIS: {
|
||||
ENABLED: boolean;
|
||||
UNIX_SOCKET_PATH: string;
|
||||
},
|
||||
}
|
||||
|
||||
const defaults: IConfig = {
|
||||
@ -150,6 +155,7 @@ const defaults: IConfig = {
|
||||
'API_URL_PREFIX': '/api/v1/',
|
||||
'POLL_RATE_MS': 2000,
|
||||
'CACHE_DIR': './cache',
|
||||
'CACHE_ENABLED': true,
|
||||
'CLEAR_PROTECTION_MINUTES': 20,
|
||||
'RECOMMENDED_FEE_PERCENTILE': 50,
|
||||
'BLOCK_WEIGHT_UNITS': 4000000,
|
||||
@ -275,7 +281,11 @@ const defaults: IConfig = {
|
||||
'AUDIT': false,
|
||||
'AUDIT_START_HEIGHT': 774000,
|
||||
'SERVERS': [],
|
||||
}
|
||||
},
|
||||
'REDIS': {
|
||||
'ENABLED': false,
|
||||
'UNIX_SOCKET_PATH': '',
|
||||
},
|
||||
};
|
||||
|
||||
class Config implements IConfig {
|
||||
@ -296,6 +306,7 @@ class Config implements IConfig {
|
||||
EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
|
||||
MAXMIND: IConfig['MAXMIND'];
|
||||
REPLICATION: IConfig['REPLICATION'];
|
||||
REDIS: IConfig['REDIS'];
|
||||
|
||||
constructor() {
|
||||
const configs = this.merge(configFromFile, defaults);
|
||||
@ -316,6 +327,7 @@ class Config implements IConfig {
|
||||
this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
|
||||
this.MAXMIND = configs.MAXMIND;
|
||||
this.REPLICATION = configs.REPLICATION;
|
||||
this.REDIS = configs.REDIS;
|
||||
}
|
||||
|
||||
merge = (...objects: object[]): IConfig => {
|
||||
|
||||
@ -41,6 +41,7 @@ import chainTips from './api/chain-tips';
|
||||
import { AxiosError } from 'axios';
|
||||
import v8 from 'v8';
|
||||
import { formatBytes, getBytesUnit } from './utils/format';
|
||||
import redisCache from './api/redis-cache';
|
||||
|
||||
class Server {
|
||||
private wss: WebSocket.Server | undefined;
|
||||
@ -122,7 +123,11 @@ class Server {
|
||||
await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
|
||||
await syncAssets.syncAssets$();
|
||||
if (config.MEMPOOL.ENABLED) {
|
||||
await diskCache.$loadMempoolCache();
|
||||
if (config.MEMPOOL.CACHE_ENABLED) {
|
||||
await diskCache.$loadMempoolCache();
|
||||
} else if (config.REDIS.ENABLED) {
|
||||
await redisCache.$loadCache();
|
||||
}
|
||||
}
|
||||
|
||||
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user