Merge pull request #2333 from mempool/fix/difficulty-api

Fix: Difficulty API (REST) with frontend fixes

backend/jest.config.ts

@@ -7,11 +7,14 @@ const config: Config.InitialOptions = {
   automock: false,
   collectCoverage: true,
   collectCoverageFrom: ["./src/**/**.ts"],
-  coverageProvider: "v8",
+  coverageProvider: "babel",
   coverageThreshold: {
     global: {
       lines: 1
     }
-  }
+  },
+  setupFiles: [
+    "./testSetup.ts",
+  ],
 }
 export default config;

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -0,0 +1,62 @@
+import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment';
+
+describe('Mempool Difficulty Adjustment', () => {
+  test('should calculate Difficulty Adjustments properly', () => {
+    const dt = (dtString) => {
+      return Math.floor(new Date(dtString).getTime() / 1000);
+    };
+
+    const vectors = [
+      [ // Vector 1
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          0,                              // If not testnet, not used
+        ],
+        { // Expected Result
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: 0,
+        },
+      ],
+      [ // Vector 2 (testnet)
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'testnet',                      // Network
+          dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds
+        ],
+        { // Expected Result is same other than timeOffset
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
+          // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
+        },
+      ],
+    ] as [[number, number, number, number, string, number], DifficultyAdjustment][];
+
+    for (const vector of vectors) {
+      const result = calcDifficultyAdjustment(...vector[0]);
+      // previousRetarget is passed through untouched
+      expect(result.previousRetarget).toStrictEqual(vector[0][3]);
+      expect(result).toStrictEqual(vector[1]);
+    }
+  });
+});

backend/src/__tests__/config.test.ts

@@ -136,5 +136,4 @@ describe('Mempool Backend Config', () => {
       expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
     });
   });
-
 });

backend/src/api/difficulty-adjustment.ts

@@ -2,6 +2,84 @@ import config from '../config';
 import { IDifficultyAdjustment } from '../mempool.interfaces';
 import blocks from './blocks';
 
+export interface DifficultyAdjustment {
+  progressPercent: number;       // Percent: 0 to 100
+  difficultyChange: number;      // Percent: -75 to 300
+  estimatedRetargetDate: number; // Unix time in ms
+  remainingBlocks: number;       // Block count
+  remainingTime: number;         // Duration of time in ms
+  previousRetarget: number;      // Percent: -75 to 300
+  nextRetargetHeight: number;    // Block Height
+  timeAvg: number;               // Duration of time in ms
+  timeOffset: number;            // (Testnet) Time since last block (cap @ 20min) in ms
+}
+
+export function calcDifficultyAdjustment(
+  DATime: number,
+  nowSeconds: number,
+  blockHeight: number,
+  previousRetarget: number,
+  network: string,
+  latestBlockTimestamp: number,
+): DifficultyAdjustment {
+  const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
+  const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
+  const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
+  const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
+
+  const diffSeconds = nowSeconds - DATime;
+  const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
+  const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
+  const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
+  const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
+
+  let difficultyChange = 0;
+  let timeAvgSecs = BLOCK_SECONDS_TARGET;
+  // Only calculate the estimate once we have 7.2% of blocks in current epoch
+  if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
+    timeAvgSecs = diffSeconds / blocksInEpoch;
+    difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+    // Max increase is x4 (+300%)
+    if (difficultyChange > 300) {
+      difficultyChange = 300;
+    }
+    // Max decrease is /4 (-75%)
+    if (difficultyChange < -75) {
+      difficultyChange = -75;
+    }
+  }
+
+  // Testnet difficulty is set to 1 after 20 minutes of no blocks,
+  // therefore the time between blocks will always be below 20 minutes (1200s).
+  let timeOffset = 0;
+  if (network === 'testnet') {
+    if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS;
+    }
+
+    const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp;
+    if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000;
+    }
+  }
+
+  const timeAvg = Math.floor(timeAvgSecs * 1000);
+  const remainingTime = remainingBlocks * timeAvg;
+  const estimatedRetargetDate = remainingTime + nowSeconds * 1000;
+
+  return {
+    progressPercent,
+    difficultyChange,
+    estimatedRetargetDate,
+    remainingBlocks,
+    remainingTime,
+    previousRetarget,
+    nextRetargetHeight,
+    timeAvg,
+    timeOffset,
+  };
+}
+
 class DifficultyAdjustmentApi {
   constructor() { }
 
@@ -11,56 +89,12 @@ class DifficultyAdjustmentApi {
     const blockHeight = blocks.getCurrentBlockHeight();
     const blocksCache = blocks.getBlocks();
     const latestBlock = blocksCache[blocksCache.length - 1];
+    const nowSeconds = Math.floor(new Date().getTime() / 1000);
 
-    const now = new Date().getTime() / 1000;
-    const diff = now - DATime;
-    const blocksInEpoch = blockHeight % 2016;
-    const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
-    const remainingBlocks = 2016 - blocksInEpoch;
-    const nextRetargetHeight = blockHeight + remainingBlocks;
-
-    let difficultyChange = 0;
-    if (remainingBlocks < 1870) {
-      if (blocksInEpoch > 0) {
-        difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100;
-      }
-      if (difficultyChange > 300) {
-        difficultyChange = 300;
-      }
-      if (difficultyChange < -75) {
-        difficultyChange = -75;
-      }
-    }
-
-    let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
-
-    // Testnet difficulty is set to 1 after 20 minutes of no blocks,
-    // therefore the time between blocks will always be below 20 minutes (1200s).
-    let timeOffset = 0;
-    if (config.MEMPOOL.NETWORK === 'testnet') {
-      if (timeAvgMins > 20) {
-        timeAvgMins = 20;
-      }
-      if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) {
-        timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000;
-      }
-    }
-
-    const timeAvg = timeAvgMins * 60 * 1000 ;
-    const remainingTime = (remainingBlocks * timeAvg) + (now * 1000);
-    const estimatedRetargetDate = remainingTime + now;
-
-    return {
-      progressPercent,
-      difficultyChange,
-      estimatedRetargetDate,
-      remainingBlocks,
-      remainingTime,
-      previousRetarget,
-      nextRetargetHeight,
-      timeAvg,
-      timeOffset,
-    };
+    return calcDifficultyAdjustment(
+      DATime, nowSeconds, blockHeight, previousRetarget,
+      config.MEMPOOL.NETWORK, latestBlock.timestamp
+    );
   }
 }
 

backend/testSetup.ts

@@ -0,0 +1,5 @@
+jest.mock('./mempool-config.json', () => ({}), { virtual: true });
+jest.mock('./src/logger.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/rbf-cache.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/mempool.ts', () => ({}), { virtual: true });
+jest.mock('./src/api/memory-cache.ts', () => ({}), { virtual: true });

frontend/src/app/components/difficulty/difficulty.component.html

@@ -10,7 +10,7 @@
             <ng-template #blocksPlural let-i i18n="shared.blocks">{{ i }} <span class="shared-block">blocks</span></ng-template>
             <ng-template #blocksSingular let-i i18n="shared.block">{{ i }} <span class="shared-block">block</span></ng-template>
           </div>
-          <div class="symbol"><app-time-until [time]="epochData.remainingTime" [fastRender]="true"></app-time-until></div>
+          <div class="symbol"><app-time-until [time]="epochData.estimatedRetargetDate" [fastRender]="true"></app-time-until></div>
         </div>
         <div class="item">
           <h5 class="card-title" i18n="difficulty-box.estimate">Estimate</h5>

frontend/src/app/components/difficulty/difficulty.component.ts

@@ -11,7 +11,7 @@ interface EpochProgress {
   newDifficultyHeight: number;
   colorAdjustments: string;
   colorPreviousAdjustments: string;
-  remainingTime: number;
+  estimatedRetargetDate: number;
   previousRetarget: number;
   blocksUntilHalving: number;
   timeUntilHalving: number;
@@ -74,7 +74,7 @@ export class DifficultyComponent implements OnInit {
           colorAdjustments,
           colorPreviousAdjustments,
           newDifficultyHeight: da.nextRetargetHeight,
-          remainingTime: da.remainingTime,
+          estimatedRetargetDate: da.estimatedRetargetDate,
           previousRetarget: da.previousRetarget,
           blocksUntilHalving,
           timeUntilHalving,

frontend/src/app/docs/api-docs/api-docs-data.ts

@@ -114,11 +114,14 @@ export const restApiDocsData = [
           curl: [],
           response: `{
   progressPercent: 44.397234501112074,
-  difficultyChange: 0.9845932018381687,
-  estimatedRetargetDate: 1627762478.9111245,
+  difficultyChange: 98.45932018381687,
+  estimatedRetargetDate: 1627762478,
   remainingBlocks: 1121,
-  remainingTime: 665977.6261244365,
-  previousRetarget: -4.807005268478962
+  remainingTime: 665977,
+  previousRetarget: -4.807005268478962,
+  nextRetargetHeight: 741888,
+  timeAvg: 302328,
+  timeOffset: 0
 }`
         },
         codeSampleTestnet: {
@@ -127,11 +130,14 @@ export const restApiDocsData = [
           curl: [],
           response: `{
   progressPercent: 44.397234501112074,
-  difficultyChange: 0.9845932018381687,
-  estimatedRetargetDate: 1627762478.9111245,
+  difficultyChange: 98.45932018381687,
+  estimatedRetargetDate: 1627762478,
   remainingBlocks: 1121,
-  remainingTime: 665977.6261244365,
-  previousRetarget: -4.807005268478962
+  remainingTime: 665977,
+  previousRetarget: -4.807005268478962,
+  nextRetargetHeight: 741888,
+  timeAvg: 302328,
+  timeOffset: 0
 }`
         },
         codeSampleSignet: {
@@ -140,11 +146,14 @@ export const restApiDocsData = [
           curl: [],
           response: `{
   progressPercent: 44.397234501112074,
-  difficultyChange: 0.9845932018381687,
-  estimatedRetargetDate: 1627762478.9111245,
+  difficultyChange: 98.45932018381687,
+  estimatedRetargetDate: 1627762478,
   remainingBlocks: 1121,
-  remainingTime: 665977.6261244365,
-  previousRetarget: -4.807005268478962
+  remainingTime: 665977,
+  previousRetarget: -4.807005268478962,
+  nextRetargetHeight: 741888,
+  timeAvg: 302328,
+  timeOffset: 0
 }`
         },
         codeSampleLiquid: {
@@ -153,11 +162,14 @@ export const restApiDocsData = [
           curl: [],
           response: `{
   progressPercent: 44.397234501112074,
-  difficultyChange: 0.9845932018381687,
-  estimatedRetargetDate: 1627762478.9111245,
+  difficultyChange: 98.45932018381687,
+  estimatedRetargetDate: 1627762478,
   remainingBlocks: 1121,
-  remainingTime: 665977.6261244365,
-  previousRetarget: -4.807005268478962
+  remainingTime: 665977,
+  previousRetarget: -4.807005268478962,
+  nextRetargetHeight: 741888,
+  timeAvg: 302328,
+  timeOffset: 0
 }`
         }
       }