Merge branch 'master' into nymkappa/configurable-timeout

2025-04-07 19:38:32 +02:00 · 2023-03-20 17:38:40 +09:00 · 2023-03-20 17:38:40 +09:00 · cce7dd917f
commit cce7dd917f
parent c6e063ea2f 3dafb284a9
15 changed files with 64 additions and 75 deletions
--- a/README.md
+++ b/README.md
@ -1,13 +1,13 @@
 # The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)

-https://user-images.githubusercontent.com/232186/222445818-234aa6c9-c233-4c52-b3f0-e32b8232893b.mp4
+https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4
+
+<br>

 Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 

 It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.

-![mempool](https://mempool.space/resources/screenshots/v2.4.0-dashboard.png)
-
 # Installation Methods

 Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 
--- a/backend/mempool-config.sample.json
+++ b/backend/mempool-config.sample.json
@ -27,7 +27,8 @@
    "AUDIT": false,
    "ADVANCED_GBT_AUDIT": false,
    "ADVANCED_GBT_MEMPOOL": false,
-    "CPFP_INDEXING": false
+    "CPFP_INDEXING": false,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
  },
  "CORE_RPC": {
    "HOST": "127.0.0.1",
--- a/backend/src/fixtures/mempool-config.template.json
+++ b/backend/src/fixtures/mempool-config.template.json
@ -28,7 +28,8 @@
    "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
    "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
-    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
+    "DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
  },
  "CORE_RPC": {
    "HOST": "__CORE_RPC_HOST__",
--- a/backend/src/tests/config.test.ts
+++ b/backend/src/tests/config.test.ts
@ -42,6 +42,7 @@ describe('Mempool Backend Config', () => {
        ADVANCED_GBT_MEMPOOL: false,
        CPFP_INDEXING: false,
        MAX_BLOCKS_BULK_QUERY: 0,
+        DISK_CACHE_BLOCK_INTERVAL: 6,
      });

      expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@ -651,7 +651,7 @@ class Blocks {
      if (this.newBlockCallbacks.length) {
        this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
      }
-      if (!memPool.hasPriority() && (block.height % 6 === 0)) {
+      if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
        diskCache.$saveCacheToDisk();
      }

--- a/backend/src/api/disk-cache.ts
+++ b/backend/src/api/disk-cache.ts
@ -19,20 +19,16 @@ class DiskCache {
  private isWritingCache = false;

  constructor() {
-    if (!cluster.isMaster) {
+    if (!cluster.isPrimary) {
      return;
    }
    process.on('SIGINT', (e) => {
-      this.saveCacheToDiskSync();
-      process.exit(2);
-    });
-    process.on('SIGTERM', (e) => {
-      this.saveCacheToDiskSync();
-      process.exit(2);
+      this.$saveCacheToDisk(true);
+      process.exit(0);
    });
  }

-  async $saveCacheToDisk(): Promise<void> {
+  async $saveCacheToDisk(sync: boolean = false): Promise<void> {
    if (!cluster.isPrimary) {
      return;
    }
@ -41,7 +37,7 @@ class DiskCache {
      return;
    }
    try {
-      logger.debug('Writing mempool and blocks data to disk cache (async)...');
+      logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
      this.isWritingCache = true;

      const mempool = memPool.getMempool();
@ -54,68 +50,46 @@ class DiskCache {

      const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);

-      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
-        network: config.MEMPOOL.NETWORK,
-        cacheSchemaVersion: this.cacheSchemaVersion,
-        blocks: blocks.getBlocks(),
-        blockSummaries: blocks.getBlockSummaries(),
-        mempool: {},
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), { flag: 'w' });
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+      if (sync) {
+        fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
          mempool: {},
          mempoolArray: mempoolArray.splice(0, chunkSize),
        }), { flag: 'w' });
-      }
-      logger.debug('Mempool and blocks data saved to disk cache');
-      this.isWritingCache = false;
-    } catch (e) {
-      logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
-      this.isWritingCache = false;
-    }
-  }
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }

-  saveCacheToDiskSync(): void {
-    if (!cluster.isPrimary) {
-      return;
-    }
-    if (this.isWritingCache) {
-      logger.debug('Saving cache already in progress. Skipping.');
-      return;
-    }
-    try {
-      logger.debug('Writing mempool and blocks data to disk cache (sync)...');
-      this.isWritingCache = true;
-
-      const mempool = memPool.getMempool();
-      const mempoolArray: TransactionExtended[] = [];
-      for (const tx in mempool) {
-        mempoolArray.push(mempool[tx]);
-      }
-
-      Common.shuffleArray(mempoolArray);
-
-      const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
-
-      fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
-        network: config.MEMPOOL.NETWORK,
-        cacheSchemaVersion: this.cacheSchemaVersion,
-        blocks: blocks.getBlocks(),
-        blockSummaries: blocks.getBlockSummaries(),
-        mempool: {},
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), { flag: 'w' });
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+        fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
+      } else {
+        await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
          mempool: {},
          mempoolArray: mempoolArray.splice(0, chunkSize),
        }), { flag: 'w' });
-      }
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }

-      fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
      }

      logger.debug('Mempool and blocks data saved to disk cache');
--- a/backend/src/config.ts
+++ b/backend/src/config.ts
@ -33,6 +33,7 @@ interface IConfig {
    ADVANCED_GBT_MEMPOOL: boolean;
    CPFP_INDEXING: boolean;
    MAX_BLOCKS_BULK_QUERY: number;
+    DISK_CACHE_BLOCK_INTERVAL: number;
  };
  ESPLORA: {
    REST_API_URL: string;
@ -158,6 +159,7 @@ const defaults: IConfig = {
    'ADVANCED_GBT_MEMPOOL': false,
    'CPFP_INDEXING': false,
    'MAX_BLOCKS_BULK_QUERY': 0,
+    'DISK_CACHE_BLOCK_INTERVAL': 6,
  },
  'ESPLORA': {
    'REST_API_URL': 'http://127.0.0.1:3000',
--- a/docker/README.md
+++ b/docker/README.md
@ -113,6 +113,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
    "ADVANCED_GBT_MEMPOOL": false,
    "CPFP_INDEXING": false,
    "MAX_BLOCKS_BULK_QUERY": 0,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
  },
 ```

@ -144,6 +145,7 @@ Corresponding `docker-compose.yml` overrides:
      MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
      MEMPOOL_CPFP_INDEXING: ""
      MAX_BLOCKS_BULK_QUERY: ""
+      DISK_CACHE_BLOCK_INTERVAL: ""
      ...
 ```

--- a/docker/backend/mempool-config.json
+++ b/docker/backend/mempool-config.json
@ -26,7 +26,8 @@
    "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
    "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
-    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__
+    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
+    "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__
  },
  "CORE_RPC": {
    "HOST": "__CORE_RPC_HOST__",
--- a/docker/backend/start.sh
+++ b/docker/backend/start.sh
@ -30,6 +30,7 @@ __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
+__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}

 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@ -152,6 +153,7 @@ sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json

 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json
--- a/production/mempool-config.liquid.json
+++ b/production/mempool-config.liquid.json
@ -6,7 +6,8 @@
    "MINED_BLOCKS_CACHE": 144,
    "SPAWN_CLUSTER_PROCS": 0,
    "API_URL_PREFIX": "/api/v1/",
-    "POLL_RATE_MS": 1000
+    "POLL_RATE_MS": 1000,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
  },
  "SYSLOG" : {
    "MIN_PRIORITY": "debug"
--- a/production/mempool-config.liquidtestnet.json
+++ b/production/mempool-config.liquidtestnet.json
@ -6,7 +6,8 @@
    "MINED_BLOCKS_CACHE": 144,
    "SPAWN_CLUSTER_PROCS": 0,
    "API_URL_PREFIX": "/api/v1/",
-    "POLL_RATE_MS": 1000
+    "POLL_RATE_MS": 1000,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
  },
  "SYSLOG" : {
    "MIN_PRIORITY": "debug"
--- a/production/mempool-config.mainnet.json
+++ b/production/mempool-config.mainnet.json
@ -14,7 +14,8 @@
    "CPFP_INDEXING": true,
    "ADVANCED_GBT_AUDIT": true,
    "ADVANCED_GBT_MEMPOOL": true,
-    "USE_SECOND_NODE_FOR_MINFEE": true
+    "USE_SECOND_NODE_FOR_MINFEE": true,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
  },
  "SYSLOG" : {
    "MIN_PRIORITY": "debug"
--- a/production/mempool-config.signet.json
+++ b/production/mempool-config.signet.json
@ -10,7 +10,8 @@
    "AUDIT": true,
    "ADVANCED_GBT_AUDIT": true,
    "ADVANCED_GBT_MEMPOOL": true,
-    "POLL_RATE_MS": 1000
+    "POLL_RATE_MS": 1000,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
  },
  "SYSLOG" : {
    "MIN_PRIORITY": "debug"
--- a/production/mempool-config.testnet.json
+++ b/production/mempool-config.testnet.json
@ -10,7 +10,8 @@
    "AUDIT": true,
    "ADVANCED_GBT_AUDIT": true,
    "ADVANCED_GBT_MEMPOOL": true,
-    "POLL_RATE_MS": 1000
+    "POLL_RATE_MS": 1000,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
  },
  "SYSLOG" : {
    "MIN_PRIORITY": "debug"