Split network daily hashrate indexing and weekly pool hashrate indexing

backend/src/api/database-migration.ts

@@ -6,7 +6,7 @@ import logger from '../logger';
 const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
 
 class DatabaseMigration {
-  private static currentVersion = 8;
+  private static currentVersion = 9;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
 
@@ -133,6 +133,10 @@ class DatabaseMigration {
         await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
       }
 
+      if (databaseSchemaVersion < 9) {
+        await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)')
+      }
+
       connection.release();
     } catch (e) {
       connection.release();
@@ -276,6 +280,10 @@ class DatabaseMigration {
       queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
     }
 
+    if (version < 9) {
+      queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
+    }
+
     return queries;
   }
 

backend/src/api/mining.ts

@@ -8,6 +8,7 @@ import blocks from './blocks';
 
 class Mining {
   hashrateIndexingStarted = false;
+  weeklyHashrateIndexingStarted = false;
 
   constructor() {
   }
@@ -95,31 +96,112 @@ class Mining {
   }
 
   /**
-   * Generate daily hashrate data
+   * Generate weekly mining pool hashrate history
    */
-  public async $generateNetworkHashrateHistory(): Promise<void> {
+  public async $generatePoolHashrateHistory(): Promise<void> {
-    // We only run this once a day
+    if (!blocks.blockIndexingCompleted || this.weeklyHashrateIndexingStarted) {
-    const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp();
-    const now = new Date().getTime() / 1000;
-    if (now - latestTimestamp < 86400) {
       return;
     }
 
+    // We only run this once a week
+    const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing');
+    const now = new Date().getTime() / 1000;
+    if (now - latestTimestamp < 604800) {
+      return;
+    }
+
+    try {
+      this.weeklyHashrateIndexingStarted = true;
+
+      logger.info(`Indexing mining pools weekly hashrates`);
+
+      const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
+      const hashrates: any[] = [];
+      const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
+      const lastWeekMidnight = this.getDateMidnight(new Date());
+      lastWeekMidnight.setDate(lastWeekMidnight.getDate() - 7);
+      let toTimestamp = Math.round(lastWeekMidnight.getTime() / 1000);
+
+      while (toTimestamp > genesisTimestamp) {
+        const fromTimestamp = toTimestamp - 604800;
+
+        // Skip already indexed weeks
+        if (indexedTimestamp.includes(toTimestamp + 1)) {
+          toTimestamp -= 604800;
+          continue;
+        }
+
+        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, fromTimestamp, toTimestamp);
+        if (blockStats.blockCount === 0) { // We are done indexing, no blocks left
+          break;
+        }
+
+        const fromTxt = new Date(1000 * fromTimestamp);
+        const toTxt = new Date(1000 * toTimestamp);
+        logger.debug(`Indexing pools hashrate between ${fromTxt.toUTCString()} and ${toTxt.toUTCString()}`)
+
+        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+          blockStats.lastBlockHeight);
+
+        let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp, toTimestamp);
+        const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
+        pools = pools.map((pool: any) => {
+          pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
+          pool.share = (pool.blockCount / totalBlocks);
+          return pool;
+        });
+
+        for (const pool of pools) {
+          hashrates.push({
+            hashrateTimestamp: toTimestamp + 1,
+            avgHashrate: pool['hashrate'],
+            poolId: pool.poolId,
+            share: pool['share'],
+            type: 'weekly',
+          });
+        }
+
+        await HashratesRepository.$saveHashrates(hashrates);
+        hashrates.length = 0;
+
+        toTimestamp -= 604800;
+      }
+      this.weeklyHashrateIndexingStarted = false;
+      await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing');
+      logger.info(`Weekly pools hashrate indexing completed`);
+    } catch (e) {
+      this.weeklyHashrateIndexingStarted = false;
+      throw e;
+    }
+
+  }
+
+  /**
+   * Generate daily hashrate data
+   */
+  public async $generateNetworkHashrateHistory(): Promise<void> {
     if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) {
       return;
     }
 
+    // We only run this once a day
+    const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_hashrates_indexing');
+    const now = new Date().getTime() / 1000;
+    if (now - latestTimestamp < 86400) {
+      return;
+    }
+
     try {
       this.hashrateIndexingStarted = true;
 
-      logger.info(`Indexing hashrates`);
+      logger.info(`Indexing network daily hashrate`);
 
       const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
       const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
       let startedAt = new Date().getTime() / 1000;
       const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
-      const lastMidnight = new Date();
+      const lastMidnight = this.getDateMidnight(new Date());
-      lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0);
       let toTimestamp = Math.round(lastMidnight.getTime() / 1000);
       let indexedThisRun = 0;
       let totalIndexed = 0;
@@ -128,6 +210,8 @@ class Mining {
 
       while (toTimestamp > genesisTimestamp) {
         const fromTimestamp = toTimestamp - 86400;
+
+        // Skip already indexed weeks
         if (indexedTimestamp.includes(fromTimestamp)) {
           toTimestamp -= 86400;
           ++totalIndexed;
@@ -140,31 +224,9 @@ class Mining {
           break;
         }
 
-        let lastBlockHashrate = 0;
+        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
-        lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
           blockStats.lastBlockHeight);
 
-        if (totalIndexed > 7 && totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate
-          logger.debug("Indexing weekly hashrates for mining pools");
-          let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp);
-          const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
-          pools = pools.map((pool: any) => {
-            pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
-            pool.share = (pool.blockCount / totalBlocks);
-            return pool;
-          });
-
-          for (const pool of pools) {
-            hashrates.push({
-              hashrateTimestamp: fromTimestamp + 1,
-              avgHashrate: pool['hashrate'],
-              poolId: pool.poolId,
-              share: pool['share'],
-              type: 'weekly',
-            });
-          }
-        }
-
         hashrates.push({
           hashrateTimestamp: fromTimestamp,
           avgHashrate: lastBlockHashrate,
@@ -203,18 +265,25 @@ class Mining {
         });
       }
 
-      if (hashrates.length > 0) {
       await HashratesRepository.$saveHashrates(hashrates);
-      }
-      await HashratesRepository.$setLatestRunTimestamp();
-      this.hashrateIndexingStarted = false;
 
-      logger.info(`Hashrates indexing completed`);
+      await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing');
+      this.hashrateIndexingStarted = false;
+      logger.info(`Daily network hashrate indexing completed`);
     } catch (e) {
       this.hashrateIndexingStarted = false;
       throw e;
     }
   }
+
+  private getDateMidnight(date: Date): Date {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+
+    return date;
+  }
 }
 
 export default new Mining();

backend/src/index.ts

@@ -167,7 +167,8 @@ class Server {
   }
 
   async $resetHashratesIndexingState() {
-    return await HashratesRepository.$setLatestRunTimestamp(0);
+    await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing', 0);
+    await HashratesRepository.$setLatestRunTimestamp('last_week_hashrates_indexing', 0);
   }
 
   async $runIndexingWhenReady() {
@@ -178,6 +179,7 @@ class Server {
     try {
       await blocks.$generateBlockDatabase();
       await mining.$generateNetworkHashrateHistory();
+      await mining.$generatePoolHashrateHistory();
     } catch (e) {
       logger.err(`Unable to run indexing right now, trying again later. ` + e);
     }

backend/src/repositories/HashratesRepository.ts

@@ -8,6 +8,10 @@ class HashratesRepository {
    * Save indexed block data in the database
    */
   public async $saveHashrates(hashrates: any) {
+    if (hashrates.length === 0) {
+      return;
+    }
+
     let query = `INSERT INTO
       hashrates(hashrate_timestamp, avg_hashrate, pool_id, share, type) VALUES`;
 
@@ -53,6 +57,16 @@ class HashratesRepository {
     return rows;
   }
 
+  public async $getWeeklyHashrateTimestamps(): Promise<number[]> {
+    const connection = await DB.pool.getConnection();
+
+    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp FROM hashrates where type = 'weekly' GROUP BY hashrate_timestamp`;
+    const [rows]: any[] = await connection.query(query);
+    connection.release();
+
+    return rows.map(row => row.timestamp);
+  }
+
   /**
    * Returns the current biggest pool hashrate history
    */
@@ -83,18 +97,18 @@ class HashratesRepository {
     return rows;
   }
 
-  public async $setLatestRunTimestamp(val: any = null) {
+  public async $setLatestRunTimestamp(key: string, val: any = null) {
     const connection = await DB.pool.getConnection();
-    const query = `UPDATE state SET number = ? WHERE name = 'last_hashrates_indexing'`;
+    const query = `UPDATE state SET number = ? WHERE name = ?`;
 
-    await connection.query<any>(query, (val === null) ? [Math.round(new Date().getTime() / 1000)] : [val]);
+    await connection.query<any>(query, (val === null) ? [Math.round(new Date().getTime() / 1000), key] : [val, key]);
     connection.release();
   }
 
-  public async $getLatestRunTimestamp(): Promise<number> {
+  public async $getLatestRunTimestamp(key: string): Promise<number> {
     const connection = await DB.pool.getConnection();
-    const query = `SELECT number FROM state WHERE name = 'last_hashrates_indexing'`;
+    const query = `SELECT number FROM state WHERE name = ?`;
-    const [rows] = await connection.query<any>(query);
+    const [rows] = await connection.query<any>(query, [key]);
     connection.release();
     return rows[0]['number'];
   }