diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index c0b548b9a..2f4bcee85 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -4,6 +4,7 @@ import http from 'http'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; import logger from '../../logger'; +import { Common } from '../common'; interface FailoverHost { host: string, @@ -15,11 +16,13 @@ interface FailoverHost { outOfSync?: boolean, unreachable?: boolean, preferred?: boolean, + checked: boolean, } class FailoverRouter { activeHost: FailoverHost; fallbackHost: FailoverHost; + maxHeight: number = 0; hosts: FailoverHost[]; multihost: boolean; pollInterval: number = 60000; @@ -34,6 +37,7 @@ class FailoverRouter { this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => { return { host: domain, + checked: false, rtts: [], rtt: Infinity, failures: 0, @@ -46,6 +50,7 @@ class FailoverRouter { failures: 0, socket: !!config.ESPLORA.UNIX_SOCKET_PATH, preferred: true, + checked: false, }; this.fallbackHost = this.activeHost; this.hosts.unshift(this.activeHost); @@ -74,66 +79,87 @@ class FailoverRouter { clearTimeout(this.pollTimer); } - const results = await Promise.allSettled(this.hosts.map(async (host) => { - if (host.socket) { - return this.pollConnection.get('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT }); - } else { - return this.pollConnection.get(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT }); - } - })); - const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0); + const start = Date.now(); // update rtts & sync status - for (let i = 0; i < results.length; i++) { - const host = this.hosts[i]; - const result = results[i].status === 'fulfilled' ? (results[i] as PromiseFulfilledResult>).value : null; - if (result) { - const height = result.data; - const rtt = result.config['meta'].rtt; - host.rtts.unshift(rtt); - host.rtts.slice(0, 5); - host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length; - host.latestHeight = height; - if (height == null || isNaN(height) || (maxHeight - height > 2)) { - host.outOfSync = true; + for (const host of this.hosts) { + try { + const result = await (host.socket + ? this.pollConnection.get('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT }) + : this.pollConnection.get(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT }) + ); + if (result) { + const height = result.data; + this.maxHeight = Math.max(height, this.maxHeight); + const rtt = result.config['meta'].rtt; + host.rtts.unshift(rtt); + host.rtts.slice(0, 5); + host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length; + host.latestHeight = height; + if (height == null || isNaN(height) || (this.maxHeight - height > 2)) { + host.outOfSync = true; + } else { + host.outOfSync = false; + } + host.unreachable = false; } else { - host.outOfSync = false; + host.outOfSync = true; + host.unreachable = true; + host.rtts = []; + host.rtt = Infinity; } - host.unreachable = false; - } else { + } catch (e) { host.outOfSync = true; host.unreachable = true; + host.rtts = []; + host.rtt = Infinity; } + host.checked = true; + + + // switch if the current host is out of sync or significantly slower than the next best alternative + const rankOrder = this.sortHosts(); + // switch if the current host is out of sync or significantly slower than the next best alternative + if (this.activeHost.outOfSync || this.activeHost.unreachable || (this.activeHost !== rankOrder[0] && rankOrder[0].preferred) || (!this.activeHost.preferred && this.activeHost.rtt > (rankOrder[0].rtt * 2) + 50)) { + if (this.activeHost.unreachable) { + logger.warn(`🚨🚨🚨 Unable to reach ${this.activeHost.host}, failing over to next best alternative 🚨🚨🚨`); + } else if (this.activeHost.outOfSync) { + logger.warn(`🚨🚨🚨 ${this.activeHost.host} has fallen behind, failing over to next best alternative 🚨🚨🚨`); + } else { + logger.debug(`🛠️ ${this.activeHost.host} is no longer the best esplora host 🛠️`); + } + this.electHost(); + } + await Common.sleep$(50); } - this.sortHosts(); + const rankOrder = this.updateFallback(); + logger.debug(`Tomahawk ranking:\n${rankOrder.map((host, index) => this.formatRanking(index, host, this.activeHost, this.maxHeight)).join('\n')}`); - logger.debug(`Tomahawk ranking:\n${this.hosts.map((host, index) => this.formatRanking(index, host, this.activeHost, maxHeight)).join('\n')}`); + const elapsed = Date.now() - start; - // switch if the current host is out of sync or significantly slower than the next best alternative - if (this.activeHost.outOfSync || this.activeHost.unreachable || (this.activeHost !== this.hosts[0] && this.hosts[0].preferred) || (!this.activeHost.preferred && this.activeHost.rtt > (this.hosts[0].rtt * 2) + 50)) { - if (this.activeHost.unreachable) { - logger.warn(`🚨🚨🚨 Unable to reach ${this.activeHost.host}, failing over to next best alternative 🚨🚨🚨`); - } else if (this.activeHost.outOfSync) { - logger.warn(`🚨🚨🚨 ${this.activeHost.host} has fallen behind, failing over to next best alternative 🚨🚨🚨`); - } else { - logger.debug(`🛠️ ${this.activeHost.host} is no longer the best esplora host 🛠️`); - } - this.electHost(); - } - - this.pollTimer = setTimeout(() => { this.pollHosts(); }, this.pollInterval); + this.pollTimer = setTimeout(() => { this.pollHosts(); }, Math.max(1, this.pollInterval - elapsed)); } private formatRanking(index: number, host: FailoverHost, active: FailoverHost, maxHeight: number): string { - const heightStatus = host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅'); - return `${host === active ? '⭐️' : ' '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : ' - '} ${host.unreachable ? '🔥' : '✅'} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : ' '}`; + const heightStatus = !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅')); + return `${host === active ? '⭐️' : ' '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : ' - '} ${!host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅')} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : ' '}`; + } + + private updateFallback(): FailoverHost[] { + const rankOrder = this.sortHosts(); + if (rankOrder.length > 1 && rankOrder[0] === this.activeHost) { + this.fallbackHost = rankOrder[1]; + } else { + this.fallbackHost = rankOrder[0]; + } + return rankOrder; } // sort hosts by connection quality, and update default fallback - private sortHosts(): void { + private sortHosts(): FailoverHost[] { // sort by connection quality - this.hosts.sort((a, b) => { + return this.hosts.slice().sort((a, b) => { if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) { if (a.preferred === b.preferred) { // lower rtt is best @@ -145,19 +171,14 @@ class FailoverRouter { return (a.unreachable || a.outOfSync) ? 1 : -1; } }); - if (this.hosts.length > 1 && this.hosts[0] === this.activeHost) { - this.fallbackHost = this.hosts[1]; - } else { - this.fallbackHost = this.hosts[0]; - } } // depose the active host and choose the next best replacement private electHost(): void { this.activeHost.outOfSync = true; this.activeHost.failures = 0; - this.sortHosts(); - this.activeHost = this.hosts[0]; + const rankOrder = this.sortHosts(); + this.activeHost = rankOrder[0]; logger.warn(`Switching esplora host to ${this.activeHost.host}`); }