diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 9d8e4af37..190fd7164 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -17,7 +17,7 @@ "BISQ_ENABLED": false, "BISQ_MARKET_ENABLED": false, "BSQ_BLOCKS_DATA_PATH": "/bisq/data", - "BSQ_MARKETS_DATA_PATH": "/bisq-markets/data", + "BSQ_MARKETS_DATA_PATH": "/bisq-folder/Bisq", "SSL": false, "SSL_CERT_FILE_PATH": "/etc/letsencrypt/live/mysite/fullchain.pem", "SSL_KEY_FILE_PATH": "/etc/letsencrypt/live/mysite/privkey.pem" diff --git a/backend/package-lock.json b/backend/package-lock.json index 6052b4260..02bbf97f7 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -86,6 +86,11 @@ "@types/range-parser": "*" } }, + "@types/locutus": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@types/locutus/-/locutus-0.0.6.tgz", + "integrity": "sha512-P+BQds4wrJhqKiIOBWAYpbsE9UOztnnqW9zHk4Bci7kCXjEQAA7FJrD9HX5JU2Z36fhE2WDctuuIpLvqDsciWQ==" + }, "@types/mime": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", @@ -446,6 +451,11 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, + "es6-promise": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", + "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -733,6 +743,14 @@ "verror": "1.10.0" } }, + "locutus": { + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/locutus/-/locutus-2.0.12.tgz", + "integrity": "sha512-wnzhY9xOdDb2djr17kQhTh9oZgEfp78zI27KRRiiV1GnPXWA2xfVODbpH3QgpIuUMLupM02+6X/rJXvktTpnoA==", + "requires": { + "es6-promise": "^4.2.5" + } + }, "long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", diff --git a/backend/package.json b/backend/package.json index 884a9fb90..1de591301 100644 --- a/backend/package.json +++ b/backend/package.json @@ -29,6 +29,7 @@ "express": "^4.17.1", "mysql2": "^1.6.1", "request": "^2.88.2", + "locutus": "^2.0.12", "ws": "^7.3.1" }, "devDependencies": { @@ -36,6 +37,7 @@ "@types/express": "^4.17.2", "@types/request": "^2.48.2", "@types/ws": "^6.0.4", + "@types/locutus": "^0.0.6", "tslint": "~6.1.0", "typescript": "~3.9.7" } diff --git a/backend/src/api/bisq/interfaces.ts b/backend/src/api/bisq/interfaces.ts index c35800e40..1eb890fc5 100644 --- a/backend/src/api/bisq/interfaces.ts +++ b/backend/src/api/bisq/interfaces.ts @@ -222,10 +222,14 @@ export interface TradesData { primaryMarketTradeVolume: number; _market: string; - _tradePrice: string; - _tradeAmount: string; - _tradeVolume: string; - _offerAmount: string; + _tradePriceStr: string; + _tradeAmountStr: string; + _tradeVolumeStr: string; + _offerAmountStr: string; + _tradePrice: number; + _tradeAmount: number; + _tradeVolume: number; + _offerAmount: number; } export interface MarketVolume { diff --git a/backend/src/api/bisq/markets-api.ts b/backend/src/api/bisq/markets-api.ts index c209c4930..910fc2d0b 100644 --- a/backend/src/api/bisq/markets-api.ts +++ b/backend/src/api/bisq/markets-api.ts @@ -1,6 +1,8 @@ import { Currencies, OffsersData, TradesData, Depth, Currency, Interval, HighLowOpenClose, Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers } from './interfaces'; +import * as datetime from 'locutus/php/datetime'; + class BisqMarketsApi { private cryptoCurrencyData: Currency[] = []; private fiatCurrencyData: Currency[] = []; @@ -149,92 +151,18 @@ class BisqMarketsApi { direction?: 'buy' | 'sell', limit: number = 100, sort: 'asc' | 'desc' = 'desc', - ): BisqTrade[] { + ): BisqTrade[] { limit = Math.min(limit, 2000); - let trade_id_from_ts: number | null = null; - let trade_id_to_ts: number | null = null; const _market = market === 'all' ? null : market; if (!timestamp_from) { - timestamp_from = new Date('2016-01-01').getTime(); - } else { - timestamp_from = timestamp_from * 1000; + timestamp_from = new Date('2016-01-01').getTime() / 1000; } if (!timestamp_to) { - timestamp_to = new Date().getTime(); - } else { - timestamp_to = timestamp_to * 1000; + timestamp_to = new Date().getTime() / 1000; } - const allCurrencies = this.getCurrencies(); - - // note: the offer_id_from/to depends on iterating over trades in - // descending chronological order. - const tradesDataSorted = this.tradesData.slice(); - if (sort === 'asc') { - tradesDataSorted.reverse(); - } - - let matches: TradesData[] = []; - for (const trade of tradesDataSorted) { - if (trade_id_from === trade.offerId) { - trade_id_from_ts = trade.tradeDate; - } - if (trade_id_to === trade.offerId) { - trade_id_to_ts = trade.tradeDate; - } - if (trade_id_to && trade_id_to_ts === null) { - continue; - } - if (trade_id_from && trade_id_from_ts != null && trade_id_from_ts !== trade.tradeDate ) { - continue; - } - if (_market && _market !== trade._market) { - continue; - } - if (timestamp_from && timestamp_from > trade.tradeDate) { - continue; - } - if (timestamp_to && timestamp_to < trade.tradeDate) { - continue; - } - if (direction && direction !== trade.direction.toLowerCase() ) { - continue; - } - - // Filter out bogus trades with BTC/BTC or XXX/XXX market. - // See github issue: https://github.com/bitsquare/bitsquare/issues/883 - const currencyPairs = trade.currencyPair.split('/'); - if (currencyPairs[0] === currencyPairs[1]) { - continue; - } - - const currencyLeft = allCurrencies[currencyPairs[0]]; - const currencyRight = allCurrencies[currencyPairs[1]]; - - if (!currencyLeft || !currencyRight) { - continue; - } - - const tradePrice = trade.primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision); - const tradeAmount = trade.primaryMarketTradeAmount * Math.pow(10, 8 - currencyLeft.precision); - const tradeVolume = trade.primaryMarketTradeVolume * Math.pow(10, 8 - currencyRight.precision); - - trade._tradePrice = this.intToBtc(tradePrice); - trade._tradeAmount = this.intToBtc(tradeAmount); - trade._tradeVolume = this.intToBtc(tradeVolume); - trade._offerAmount = this.intToBtc(trade.offerAmount); - - matches.push(trade); - - if (matches.length >= limit) { - break; - } - } - - if ((trade_id_from && !trade_id_from_ts) || (trade_id_to && !trade_id_to_ts) ) { - matches = []; - } + const matches = this.getTradesByCriteria(_market, timestamp_to, timestamp_from, trade_id_to, trade_id_from, direction, sort, limit); if (sort === 'asc') { matches.sort((a, b) => a.tradeDate - b.tradeDate); @@ -245,9 +173,9 @@ class BisqMarketsApi { return matches.map((trade) => { const bsqTrade: BisqTrade = { direction: trade.primaryMarketDirection, - price: trade._tradePrice, - amount: trade._tradeAmount, - volume: trade._tradeVolume, + price: trade._tradePriceStr, + amount: trade._tradeAmountStr, + volume: trade._tradeVolumeStr, payment_method: trade.paymentMethod, trade_id: trade.offerId, trade_date: trade.tradeDate, @@ -279,20 +207,246 @@ class BisqMarketsApi { interval: Interval = 'auto', timestamp_from?: number, timestamp_to?: number, + milliseconds?: boolean, ): HighLowOpenClose[] { + if (milliseconds) { + timestamp_from = timestamp_from ? timestamp_from / 1000 : timestamp_from; + timestamp_to = timestamp_to ? timestamp_to / 1000 : timestamp_to; + } if (!timestamp_from) { - timestamp_from = new Date('2016-01-01').getTime(); - } else { - timestamp_from = timestamp_from * 1000; + timestamp_from = new Date('2016-01-01').getTime() / 1000; } if (!timestamp_to) { - timestamp_to = new Date().getTime(); - } else { - timestamp_to = timestamp_to * 1000; + timestamp_to = new Date().getTime() / 1000; } - return []; + + const range = timestamp_to - timestamp_from; + + const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from, + undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER); + + if (interval === 'auto') { + // two days range loads minute data + if (range <= 3600) { + // up to one hour range loads minutely data + interval = 'minute'; + } else if (range <= 1 * 24 * 3600) { + // up to one day range loads half-hourly data + interval = 'half_hour'; + } else if (range <= 3 * 24 * 3600) { + // up to 3 day range loads hourly data + interval = 'hour'; + } else if (range <= 7 * 24 * 3600) { + // up to 7 day range loads half-daily data + interval = 'half_day'; + } else if (range <= 60 * 24 * 3600) { + // up to 2 month range loads daily data + interval = 'day'; + } else if (range <= 12 * 31 * 24 * 3600) { + // up to one year range loads weekly data + interval = 'week'; + } else if (range <= 12 * 31 * 24 * 3600) { + // up to 5 year range loads monthly data + interval = 'month'; + } else { + // greater range loads yearly data + interval = 'year'; + } + } + + const hlocs = this.getTradesSummarized(trades, timestamp_from, interval); + + return hlocs; } + private getTradesSummarized(trades: TradesData[], timestamp_from, interval: string): HighLowOpenClose[] { + const intervals: any = {}; + const intervals_prices: any = {}; + const one_period = false; + + for (const trade of trades) { + const traded_at = trade.tradeDate / 1000; + const interval_start = one_period ? timestamp_from : this.intervalStart(traded_at, interval); + + if (!intervals[interval_start]) { + intervals[interval_start] = { + 'open': 0, + 'close': 0, + 'high': 0, + 'low': 0, + 'avg': 0, + 'volume_right': 0, + 'volume_left': 0, + }; + intervals_prices[interval_start] = []; + } + const period = intervals[interval_start]; + const price = trade._tradePrice; + + if (!intervals_prices[interval_start]['leftvol']) { + intervals_prices[interval_start]['leftvol'] = []; + } + if (!intervals_prices[interval_start]['rightvol']) { + intervals_prices[interval_start]['rightvol'] = []; + } + + intervals_prices[interval_start]['leftvol'].push(trade._tradeAmount); + intervals_prices[interval_start]['rightvol'].push(trade._tradeVolume); + + if (price) { + const plow = period['low']; + period['period_start'] = interval_start; + period['open'] = period['open'] || price; + period['close'] = price; + period['high'] = price > period['high'] ? price : period['high']; + period['low'] = (plow && price > plow) ? period['low'] : price; + period['avg'] = intervals_prices[interval_start]['rightvol'].reduce((p: number, c: number) => c + p, 0) + / intervals_prices[interval_start]['leftvol'].reduce((c: number, p: number) => c + p, 0) * 100000000; + period['volume_left'] += trade._tradeAmount; + period['volume_right'] += trade._tradeVolume; + } + } + + const hloc: HighLowOpenClose[] = []; + + for (const p in intervals) { + if (intervals.hasOwnProperty(p)) { + const period = intervals[p]; + hloc.push({ + period_start: period['period_start'], + open: this.intToBtc(period['open']), + close: this.intToBtc(period['close']), + high: this.intToBtc(period['high']), + low: this.intToBtc(period['low']), + avg: this.intToBtc(period['avg']), + volume_right: this.intToBtc(period['volume_right']), + volume_left: this.intToBtc(period['volume_left']), + }); + } + } + + return hloc; + } + + private getTradesByCriteria( + market: string | null, + timestamp_to: number, + timestamp_from: number, + trade_id_to: string | undefined, + trade_id_from: string | undefined, + direction: 'buy' | 'sell' | undefined, + sort: string, limit: number, + integerAmounts: boolean = true, + ): TradesData[] { + let trade_id_from_ts: number | null = null; + let trade_id_to_ts: number | null = null; + const allCurrencies = this.getCurrencies(); + + const timestampFromMilli = timestamp_from * 1000; + const timestampToMilli = timestamp_to * 1000; + + // note: the offer_id_from/to depends on iterating over trades in + // descending chronological order. + const tradesDataSorted = this.tradesData.slice(); + if (sort === 'asc') { + tradesDataSorted.reverse(); + } + + let matches: TradesData[] = []; + for (const trade of tradesDataSorted) { + if (trade_id_from === trade.offerId) { + trade_id_from_ts = trade.tradeDate; + } + if (trade_id_to === trade.offerId) { + trade_id_to_ts = trade.tradeDate; + } + if (trade_id_to && trade_id_to_ts === null) { + continue; + } + if (trade_id_from && trade_id_from_ts != null && trade_id_from_ts !== trade.tradeDate) { + continue; + } + if (market && market !== trade._market) { + continue; + } + if (timestampFromMilli && timestampFromMilli > trade.tradeDate) { + continue; + } + if (timestampToMilli && timestampToMilli < trade.tradeDate) { + continue; + } + if (direction && direction !== trade.direction.toLowerCase()) { + continue; + } + + // Filter out bogus trades with BTC/BTC or XXX/XXX market. + // See github issue: https://github.com/bitsquare/bitsquare/issues/883 + const currencyPairs = trade.currencyPair.split('/'); + if (currencyPairs[0] === currencyPairs[1]) { + continue; + } + + const currencyLeft = allCurrencies[currencyPairs[0]]; + const currencyRight = allCurrencies[currencyPairs[1]]; + + if (!currencyLeft || !currencyRight) { + continue; + } + + const tradePrice = trade.primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision); + const tradeAmount = trade.primaryMarketTradeAmount * Math.pow(10, 8 - currencyLeft.precision); + const tradeVolume = trade.primaryMarketTradeVolume * Math.pow(10, 8 - currencyRight.precision); + + if (integerAmounts) { + trade._tradePrice = tradePrice; + trade._tradeAmount = tradeAmount; + trade._tradeVolume = tradeVolume; + trade._offerAmount = trade.offerAmount; + } else { + trade._tradePriceStr = this.intToBtc(tradePrice); + trade._tradeAmountStr = this.intToBtc(tradeAmount); + trade._tradeVolumeStr = this.intToBtc(tradeVolume); + trade._offerAmountStr = this.intToBtc(trade.offerAmount); + } + + matches.push(trade); + + if (matches.length >= limit) { + break; + } + } + + if ((trade_id_from && !trade_id_from_ts) || (trade_id_to && !trade_id_to_ts)) { + matches = []; + } + return matches; + } + + private intervalStart(ts: number, interval: string) { + switch (interval) { + case 'minute': + return (ts - (ts % 60)); + case '10_minute': + return (ts - (ts % 600)); + case 'half_hour': + return (ts - (ts % 1800)); + case 'hour': + return (ts - (ts % 3600)); + case 'half_day': + return (ts - (ts % (3600 * 12))); + case 'day': + return datetime.strtotime('midnight today', ts); + case 'week': + return datetime.strtotime('midnight sunday last week', ts); + case 'month': + return datetime.strtotime('midnight first day of this month', ts); + case 'year': + return datetime.strtotime('midnight first day of january', ts); + default: + throw new Error('Unsupported interval: ' + interval); + } +} + private offerDataToOffer(offer: OffsersData): Offer { return { offer_id: offer.id, diff --git a/backend/src/api/bisq/markets.ts b/backend/src/api/bisq/markets.ts index 9bac9dd45..d9902a00b 100644 --- a/backend/src/api/bisq/markets.ts +++ b/backend/src/api/bisq/markets.ts @@ -19,7 +19,7 @@ class Bisq { startBisqService(): void { this.checkForBisqDataFolder(); this.loadBisqDumpFile(); - this.startSubDirectoryWatcher(); + this.startBisqDirectoryWatcher(); } private checkForBisqDataFolder() { @@ -29,13 +29,13 @@ class Bisq { } } - private startSubDirectoryWatcher() { + private startBisqDirectoryWatcher() { if (this.subdirectoryWatcher) { this.subdirectoryWatcher.close(); } if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) { console.log(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`); - setTimeout(() => this.startSubDirectoryWatcher(), 180000); + setTimeout(() => this.startBisqDirectoryWatcher(), 180000); return; } let fsWait: NodeJS.Timeout | null = null; diff --git a/backend/src/interfaces.ts b/backend/src/interfaces.ts index d652760ac..019168ae5 100644 --- a/backend/src/interfaces.ts +++ b/backend/src/interfaces.ts @@ -235,5 +235,5 @@ export interface RequiredSpec { [name: string]: RequiredParams; } interface RequiredParams { required: boolean; - types: ('@string' | '@number' | string)[]; + types: ('@string' | '@number' | '@boolean' | string)[]; } diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 115354aca..0aaea5212 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -347,6 +347,10 @@ class Routes { required: false, types: ['@number'] }, + 'milliseconds': { + required: false, + types: ['@boolean'] + }, }; const p = this.parseRequestParameters(req, constraints); @@ -355,7 +359,7 @@ class Routes { return; } - const result = bisqMarket.getHloc(p.market, p.interval, p.timestamp_from, p.timestamp_to); + const result = bisqMarket.getHloc(p.market, p.interval, p.timestamp_from, p.timestamp_to, p.milliseconds); if (result) { res.json(result); } else { @@ -399,6 +403,8 @@ class Routes { final[i] = number; } else if (params[i].types.indexOf('@string') > -1) { final[i] = str; + } else if (params[i].types.indexOf('@boolean') > -1) { + final[i] = str === 'true' || str === 'yes'; } else if (params[i].types.indexOf(str) > -1) { final[i] = str; } else {