diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index b544a3f9b..312d9d18d 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -79,8 +79,8 @@ }, "LND": { "TLS_CERT_PATH": "tls.cert", - "MACAROON_PATH": "admin.macaroon", - "SOCKET": "localhost:10009" + "MACAROON_PATH": "readonly.macaroon", + "REST_API_URL": "https://localhost:8080" }, "SOCKS5PROXY": { "ENABLED": false, diff --git a/backend/package-lock.json b/backend/package-lock.json index e724ac35b..968cb953b 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -13,10 +13,8 @@ "@types/node": "^16.11.41", "axios": "~0.27.2", "bitcoinjs-lib": "6.0.1", - "bolt07": "^1.8.1", "crypto-js": "^4.0.0", "express": "^4.18.0", - "lightning": "^5.16.3", "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", @@ -33,6 +31,7 @@ "@typescript-eslint/parser": "^5.30.5", "eslint": "^8.19.0", "eslint-config-prettier": "^8.5.0", + "fast-xml-parser": "^4.0.9", "prettier": "^2.7.1" } }, @@ -97,36 +96,6 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, - "node_modules/@grpc/grpc-js": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.7.tgz", - "integrity": "sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw==", - "dependencies": { - "@grpc/proto-loader": "^0.6.4", - "@types/node": ">=12.12.47" - }, - "engines": { - "node": "^8.13.0 || >=10.10.0" - } - }, - "node_modules/@grpc/proto-loader": { - "version": "0.6.13", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.6.13.tgz", - "integrity": "sha512-FjxPYDRTn6Ec3V0arm1FtSpmP6V50wuph2yILpyvTKzjc76oDdoihXqM1DzOW5ubvCC8GivfCnNtfaRE8myJ7g==", - "dependencies": { - "@types/long": "^4.0.1", - "lodash.camelcase": "^4.3.0", - "long": "^4.0.0", - "protobufjs": "^6.11.3", - "yargs": "^16.2.0" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -213,74 +182,16 @@ "node": ">= 8" } }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" - }, "node_modules/@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", "integrity": "sha512-W98JrE0j2K78swW4ukqMleo8R7h/pFETjM2DQ90MF6XK2i4LO4W3gQ71Lt4w3bfm2EvVSyWHplECvB5sK22yFQ==", + "dev": true, "dependencies": { "@types/connect": "*", "@types/node": "*" } }, - "node_modules/@types/caseless": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", - "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" - }, "node_modules/@types/compression": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.2.tgz", @@ -294,6 +205,7 @@ "version": "3.4.34", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.34.tgz", "integrity": "sha512-ePPA/JuI+X0vb+gSWlPKOY0NdNAie/rPUqX2GUPpbZwiKTkSPhjXWuee47E4MtE54QVzGCQMQkAL6JhV2E1+cQ==", + "dev": true, "dependencies": { "@types/node": "*" } @@ -308,6 +220,7 @@ "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "dev": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.18", @@ -319,6 +232,7 @@ "version": "4.17.28", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", + "dev": true, "dependencies": { "@types/node": "*", "@types/qs": "*", @@ -331,15 +245,11 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, - "node_modules/@types/long": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", - "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" - }, "node_modules/@types/mime": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", - "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==" + "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==", + "dev": true }, "node_modules/@types/node": { "version": "16.11.41", @@ -349,55 +259,30 @@ "node_modules/@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", + "dev": true }, "node_modules/@types/range-parser": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "node_modules/@types/request": { - "version": "2.48.8", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.8.tgz", - "integrity": "sha512-whjk1EDJPcAR2kYHRbFl/lKeeKYTi05A15K9bnLInCVroNDCtXce57xKdI0/rQaA3K+6q0eFyUBPmqfSndUZdQ==", - "dependencies": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - } - }, - "node_modules/@types/request/node_modules/form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", + "dev": true }, "node_modules/@types/serve-static": { "version": "1.13.8", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.8.tgz", "integrity": "sha512-MoJhSQreaVoL+/hurAZzIm8wafFR6ajiTM1m4A0kv6AGeVBl4r4pOV8bGFrjjq1sGxDTnCoF8i22o0/aE5XCyA==", + "dev": true, "dependencies": { "@types/mime": "*", "@types/node": "*" } }, - "node_modules/@types/tough-cookie": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", - "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==" - }, "node_modules/@types/ws": { "version": "8.5.3", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", + "dev": true, "dependencies": { "@types/node": "*" } @@ -838,6 +723,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, "engines": { "node": ">=8" } @@ -846,6 +732,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -870,24 +757,6 @@ "node": ">=8" } }, - "node_modules/async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" - }, - "node_modules/asyncjs-util": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/asyncjs-util/-/asyncjs-util-1.2.9.tgz", - "integrity": "sha512-U9imS8ehJA6DPNdBdvoLcIRDFh7yzI9J93CC8/2obk8gUSIy8KKhmCqYe+3NlISJhxLLi8aWmVL1Gkb3dz1xhg==", - "dependencies": { - "async": "3.2.3" - } - }, - "node_modules/asyncjs-util/node_modules/async": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", - "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -929,19 +798,6 @@ "node": ">=8.0.0" } }, - "node_modules/bip66": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/bip66/-/bip66-1.1.5.tgz", - "integrity": "sha512-nemMHz95EmS38a26XbbdxIYj5csHd3RMP3H5bwQknX0WYHF01qhpufP42mLOwVICuH2JmhIhXiWs89MfUGL7Xw==", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/bitcoin-ops": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/bitcoin-ops/-/bitcoin-ops-1.4.1.tgz", - "integrity": "sha512-pef6gxZFztEhaE9RY9HmWVmiIHqCb2OyS4HPKkpc6CIiiOa3Qmuoylxc5P2EkU3w+5eTSifI9SEZC88idAIGow==" - }, "node_modules/bitcoinjs-lib": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-6.0.1.tgz", @@ -959,11 +815,6 @@ "node": ">=8.0.0" } }, - "node_modules/bn.js": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", - "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" - }, "node_modules/body-parser": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", @@ -987,22 +838,6 @@ "npm": "1.2.8000 || >= 1.4.16" } }, - "node_modules/bolt07": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/bolt07/-/bolt07-1.8.2.tgz", - "integrity": "sha512-jq1b/ZdMambhh+yi+pm+1PJBAnlYvQYljaBgSajvVAINHrHg32ovCBra8d0ADE3BAoj6G/tK7OSV4t/yT9A+/g==", - "dependencies": { - "bn.js": "5.2.1" - } - }, - "node_modules/bolt09": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/bolt09/-/bolt09-0.2.3.tgz", - "integrity": "sha512-xEt5GE6pXB8wMIWHAoyF28k0Yt2rFqIou1LCyIeNadAOQhu/F7GTjZwreFwLl07YYkhOH23avewRt5PD8JnKKg==", - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1072,17 +907,6 @@ "node": ">=6" } }, - "node_modules/cbor": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/cbor/-/cbor-8.1.0.tgz", - "integrity": "sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==", - "dependencies": { - "nofilter": "^3.1.0" - }, - "engines": { - "node": ">=12.19" - } - }, "node_modules/cipher-base": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", @@ -1092,20 +916,11 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -1116,7 +931,8 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "node_modules/combined-stream": { "version": "1.0.8", @@ -1269,29 +1085,11 @@ "node": ">=6.0.0" } }, - "node_modules/ecpair": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/ecpair/-/ecpair-2.0.1.tgz", - "integrity": "sha512-iT3wztQMeE/nDTlfnAg8dAFUfBS7Tq2BXzq3ae6L+pWgFU0fQ3l0woTzdTBrJV3OxBjxbzjq8EQhAbEmJNWFSw==", - "dependencies": { - "randombytes": "^2.1.0", - "typeforce": "^1.18.0", - "wif": "^2.0.6" - }, - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1300,14 +1098,6 @@ "node": ">= 0.8" } }, - "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "engines": { - "node": ">=6" - } - }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -1707,6 +1497,22 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.9.tgz", + "integrity": "sha512-4G8EzDg2Nb1Qurs3f7BpFV4+jpMVsdgLVuG1Uv8O2OHJfVCg7gcA53obuKbmVqzd4Y7YXVBK05oJG7hzGIdyzg==", + "dev": true, + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + }, + "funding": { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + }, "node_modules/fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -1849,14 +1655,6 @@ "is-property": "^1.0.2" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, "node_modules/get-intrinsic": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", @@ -2047,22 +1845,6 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, - "node_modules/invoices": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/invoices/-/invoices-2.0.7.tgz", - "integrity": "sha512-2kpjok/83zOTnb4tbV+RbJz7LuGVzj/GZ+jwsC7FxMqwLAf4Sf6OESNM3uuamX9oeFRo44Vip3wn1aX+9D2m8w==", - "dependencies": { - "bech32": "2.0.0", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "bolt07": "1.8.2", - "bolt09": "0.2.3", - "tiny-secp256k1": "2.2.1" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", @@ -2085,14 +1867,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -2150,57 +1924,6 @@ "node": ">= 0.8.0" } }, - "node_modules/lightning": { - "version": "5.16.3", - "resolved": "https://registry.npmjs.org/lightning/-/lightning-5.16.3.tgz", - "integrity": "sha512-ghban3KbqkbzahwIp4NAtuhc8xIurVcCXAd7tV6qGkFYKZAy9loIvFrhZqoWF4A4jnaKbRnJPCaxzJ8JwPl3EA==", - "dependencies": { - "@grpc/grpc-js": "1.6.7", - "@grpc/proto-loader": "0.6.13", - "@types/express": "4.17.13", - "@types/node": "17.0.41", - "@types/request": "2.48.8", - "@types/ws": "8.5.3", - "async": "3.2.4", - "asyncjs-util": "1.2.9", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "body-parser": "1.20.0", - "bolt07": "1.8.2", - "bolt09": "0.2.3", - "cbor": "8.1.0", - "ecpair": "2.0.1", - "express": "4.18.1", - "invoices": "2.0.7", - "psbt": "2.6.0", - "tiny-secp256k1": "2.2.1", - "type-fest": "2.13.0" - }, - "engines": { - "node": ">=12.20" - } - }, - "node_modules/lightning/node_modules/@types/node": { - "version": "17.0.41", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.41.tgz", - "integrity": "sha512-xA6drNNeqb5YyV5fO3OAEsnXLfO7uF0whiOfPTz5AeDo8KeZFmODKnvwPymMNO8qE/an8pVY/O50tig2SQCrGw==" - }, - "node_modules/lightning/node_modules/type-fest": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.13.0.tgz", - "integrity": "sha512-lPfAm42MxE4/456+QyIaaVBAwgpJb6xZ8PRu09utnhPdWwcyj9vgy6Sq0Z5yNbJ21EdxB5dRU/Qg8bsyAMtlcw==", - "engines": { - "node": ">=12.20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" - }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -2418,14 +2141,6 @@ "resolved": "https://registry.npmjs.org/node-worker-threads-pool/-/node-worker-threads-pool-1.5.1.tgz", "integrity": "sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw==" }, - "node_modules/nofilter": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", - "integrity": "sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==", - "engines": { - "node": ">=12.19" - } - }, "node_modules/object-inspect": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", @@ -2559,31 +2274,6 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, - "node_modules/protobufjs": { - "version": "6.11.3", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz", - "integrity": "sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==", - "hasInstallScript": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": ">=13.7.0", - "long": "^4.0.0" - }, - "bin": { - "pbjs": "bin/pbjs", - "pbts": "bin/pbts" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2596,22 +2286,6 @@ "node": ">= 0.10" } }, - "node_modules/psbt": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/psbt/-/psbt-2.6.0.tgz", - "integrity": "sha512-z2ca00AMwZ6PfVETQNvXRumZdRwGuQzApIH/hKNp2o6Qo8N8TW7Ug2V+aSH2w/eC1b/bOOMZIE57V3jYN+kB4A==", - "dependencies": { - "bip66": "1.1.5", - "bitcoin-ops": "1.4.1", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "pushdata-bitcoin": "1.0.1", - "varuint-bitcoin": "1.1.2" - }, - "engines": { - "node": ">=12.20" - } - }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -2626,14 +2300,6 @@ "node": ">=6" } }, - "node_modules/pushdata-bitcoin": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pushdata-bitcoin/-/pushdata-bitcoin-1.0.1.tgz", - "integrity": "sha512-hw7rcYTJRAl4olM8Owe8x0fBuJJ+WGbMhQuLWOXEMN3PxPCKQHRkhfL+XG0+iXUmSHjkMmb3Ba55Mt21cZc9kQ==", - "dependencies": { - "bitcoin-ops": "^1.3.0" - } - }, "node_modules/qs": { "version": "6.10.3", "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", @@ -2668,14 +2334,6 @@ } ] }, - "node_modules/randombytes": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -2723,14 +2381,6 @@ "url": "https://github.com/sponsors/mysticatea" } }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -3008,23 +2658,11 @@ "safe-buffer": "~5.2.0" } }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -3044,6 +2682,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", + "dev": true + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -3058,17 +2702,6 @@ "node": ">=6" } }, - "node_modules/tiny-secp256k1": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz", - "integrity": "sha512-/U4xfVqnVxJXN4YVsru0E6t5wVncu2uunB8+RVR40fYUxkKYUPS10f+ePQZgFBoE/Jbf9H1NBveupF2VmB58Ng==", - "dependencies": { - "uint8array-tools": "0.0.7" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -3148,14 +2781,6 @@ "node": ">=4.2.0" } }, - "node_modules/uint8array-tools": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/uint8array-tools/-/uint8array-tools-0.0.7.tgz", - "integrity": "sha512-vrrNZJiusLWoFWBqz5Y5KMCgP9W9hnjZHzZiZRT8oNAkq3d5Z5Oe76jAvVVSRh4U8GGR90N2X1dWtrhvx6L8UQ==", - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -3240,22 +2865,6 @@ "node": ">=0.10.0" } }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -3282,43 +2891,10 @@ } } }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "engines": { - "node": ">=10" - } - }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "node_modules/yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", - "engines": { - "node": ">=10" - } } }, "dependencies": { @@ -3371,27 +2947,6 @@ } } }, - "@grpc/grpc-js": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.6.7.tgz", - "integrity": "sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw==", - "requires": { - "@grpc/proto-loader": "^0.6.4", - "@types/node": ">=12.12.47" - } - }, - "@grpc/proto-loader": { - "version": "0.6.13", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.6.13.tgz", - "integrity": "sha512-FjxPYDRTn6Ec3V0arm1FtSpmP6V50wuph2yILpyvTKzjc76oDdoihXqM1DzOW5ubvCC8GivfCnNtfaRE8myJ7g==", - "requires": { - "@types/long": "^4.0.1", - "lodash.camelcase": "^4.3.0", - "long": "^4.0.0", - "protobufjs": "^6.11.3", - "yargs": "^16.2.0" - } - }, "@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -3457,74 +3012,16 @@ "fastq": "^1.6.0" } }, - "@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" - }, - "@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" - }, - "@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "requires": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" - }, - "@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" - }, - "@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" - }, - "@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" - }, - "@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" - }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", "integrity": "sha512-W98JrE0j2K78swW4ukqMleo8R7h/pFETjM2DQ90MF6XK2i4LO4W3gQ71Lt4w3bfm2EvVSyWHplECvB5sK22yFQ==", + "dev": true, "requires": { "@types/connect": "*", "@types/node": "*" } }, - "@types/caseless": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", - "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" - }, "@types/compression": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.2.tgz", @@ -3538,6 +3035,7 @@ "version": "3.4.34", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.34.tgz", "integrity": "sha512-ePPA/JuI+X0vb+gSWlPKOY0NdNAie/rPUqX2GUPpbZwiKTkSPhjXWuee47E4MtE54QVzGCQMQkAL6JhV2E1+cQ==", + "dev": true, "requires": { "@types/node": "*" } @@ -3552,6 +3050,7 @@ "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "dev": true, "requires": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.18", @@ -3563,6 +3062,7 @@ "version": "4.17.28", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", "integrity": "sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig==", + "dev": true, "requires": { "@types/node": "*", "@types/qs": "*", @@ -3575,15 +3075,11 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, - "@types/long": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", - "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" - }, "@types/mime": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.3.tgz", - "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==" + "integrity": "sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q==", + "dev": true }, "@types/node": { "version": "16.11.41", @@ -3593,54 +3089,30 @@ "@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", - "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==" + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", + "dev": true }, "@types/range-parser": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", - "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" - }, - "@types/request": { - "version": "2.48.8", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.8.tgz", - "integrity": "sha512-whjk1EDJPcAR2kYHRbFl/lKeeKYTi05A15K9bnLInCVroNDCtXce57xKdI0/rQaA3K+6q0eFyUBPmqfSndUZdQ==", - "requires": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - }, - "dependencies": { - "form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - } - } + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", + "dev": true }, "@types/serve-static": { "version": "1.13.8", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.8.tgz", "integrity": "sha512-MoJhSQreaVoL+/hurAZzIm8wafFR6ajiTM1m4A0kv6AGeVBl4r4pOV8bGFrjjq1sGxDTnCoF8i22o0/aE5XCyA==", + "dev": true, "requires": { "@types/mime": "*", "@types/node": "*" } }, - "@types/tough-cookie": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", - "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==" - }, "@types/ws": { "version": "8.5.3", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", + "dev": true, "requires": { "@types/node": "*" } @@ -3913,12 +3385,14 @@ "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -3934,26 +3408,6 @@ "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true }, - "async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" - }, - "asyncjs-util": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/asyncjs-util/-/asyncjs-util-1.2.9.tgz", - "integrity": "sha512-U9imS8ehJA6DPNdBdvoLcIRDFh7yzI9J93CC8/2obk8gUSIy8KKhmCqYe+3NlISJhxLLi8aWmVL1Gkb3dz1xhg==", - "requires": { - "async": "3.2.3" - }, - "dependencies": { - "async": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", - "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==" - } - } - }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -3992,19 +3446,6 @@ "resolved": "https://registry.npmjs.org/bip174/-/bip174-2.0.1.tgz", "integrity": "sha512-i3X26uKJOkDTAalYAp0Er+qGMDhrbbh2o93/xiPyAN2s25KrClSpe3VXo/7mNJoqA5qfko8rLS2l3RWZgYmjKQ==" }, - "bip66": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/bip66/-/bip66-1.1.5.tgz", - "integrity": "sha512-nemMHz95EmS38a26XbbdxIYj5csHd3RMP3H5bwQknX0WYHF01qhpufP42mLOwVICuH2JmhIhXiWs89MfUGL7Xw==", - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "bitcoin-ops": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/bitcoin-ops/-/bitcoin-ops-1.4.1.tgz", - "integrity": "sha512-pef6gxZFztEhaE9RY9HmWVmiIHqCb2OyS4HPKkpc6CIiiOa3Qmuoylxc5P2EkU3w+5eTSifI9SEZC88idAIGow==" - }, "bitcoinjs-lib": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-6.0.1.tgz", @@ -4019,11 +3460,6 @@ "wif": "^2.0.1" } }, - "bn.js": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", - "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" - }, "body-parser": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", @@ -4043,19 +3479,6 @@ "unpipe": "1.0.0" } }, - "bolt07": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/bolt07/-/bolt07-1.8.2.tgz", - "integrity": "sha512-jq1b/ZdMambhh+yi+pm+1PJBAnlYvQYljaBgSajvVAINHrHg32ovCBra8d0ADE3BAoj6G/tK7OSV4t/yT9A+/g==", - "requires": { - "bn.js": "5.2.1" - } - }, - "bolt09": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/bolt09/-/bolt09-0.2.3.tgz", - "integrity": "sha512-xEt5GE6pXB8wMIWHAoyF28k0Yt2rFqIou1LCyIeNadAOQhu/F7GTjZwreFwLl07YYkhOH23avewRt5PD8JnKKg==" - }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -4113,14 +3536,6 @@ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true }, - "cbor": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/cbor/-/cbor-8.1.0.tgz", - "integrity": "sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==", - "requires": { - "nofilter": "^3.1.0" - } - }, "cipher-base": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", @@ -4130,20 +3545,11 @@ "safe-buffer": "^5.0.1" } }, - "cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "requires": { "color-name": "~1.1.4" } @@ -4151,7 +3557,8 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true }, "combined-stream": { "version": "1.0.8", @@ -4270,36 +3677,16 @@ "esutils": "^2.0.2" } }, - "ecpair": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/ecpair/-/ecpair-2.0.1.tgz", - "integrity": "sha512-iT3wztQMeE/nDTlfnAg8dAFUfBS7Tq2BXzq3ae6L+pWgFU0fQ3l0woTzdTBrJV3OxBjxbzjq8EQhAbEmJNWFSw==", - "requires": { - "randombytes": "^2.1.0", - "typeforce": "^1.18.0", - "wif": "^2.0.6" - } - }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, - "escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" - }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -4609,6 +3996,15 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-xml-parser": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.9.tgz", + "integrity": "sha512-4G8EzDg2Nb1Qurs3f7BpFV4+jpMVsdgLVuG1Uv8O2OHJfVCg7gcA53obuKbmVqzd4Y7YXVBK05oJG7hzGIdyzg==", + "dev": true, + "requires": { + "strnum": "^1.0.5" + } + }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -4716,11 +4112,6 @@ "is-property": "^1.0.2" } }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" - }, "get-intrinsic": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", @@ -4857,19 +4248,6 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, - "invoices": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/invoices/-/invoices-2.0.7.tgz", - "integrity": "sha512-2kpjok/83zOTnb4tbV+RbJz7LuGVzj/GZ+jwsC7FxMqwLAf4Sf6OESNM3uuamX9oeFRo44Vip3wn1aX+9D2m8w==", - "requires": { - "bech32": "2.0.0", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "bolt07": "1.8.2", - "bolt09": "0.2.3", - "tiny-secp256k1": "2.2.1" - } - }, "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", @@ -4886,11 +4264,6 @@ "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, "is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -4939,50 +4312,6 @@ "type-check": "~0.4.0" } }, - "lightning": { - "version": "5.16.3", - "resolved": "https://registry.npmjs.org/lightning/-/lightning-5.16.3.tgz", - "integrity": "sha512-ghban3KbqkbzahwIp4NAtuhc8xIurVcCXAd7tV6qGkFYKZAy9loIvFrhZqoWF4A4jnaKbRnJPCaxzJ8JwPl3EA==", - "requires": { - "@grpc/grpc-js": "1.6.7", - "@grpc/proto-loader": "0.6.13", - "@types/express": "4.17.13", - "@types/node": "17.0.41", - "@types/request": "2.48.8", - "@types/ws": "8.5.3", - "async": "3.2.4", - "asyncjs-util": "1.2.9", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "body-parser": "1.20.0", - "bolt07": "1.8.2", - "bolt09": "0.2.3", - "cbor": "8.1.0", - "ecpair": "2.0.1", - "express": "4.18.1", - "invoices": "2.0.7", - "psbt": "2.6.0", - "tiny-secp256k1": "2.2.1", - "type-fest": "2.13.0" - }, - "dependencies": { - "@types/node": { - "version": "17.0.41", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.41.tgz", - "integrity": "sha512-xA6drNNeqb5YyV5fO3OAEsnXLfO7uF0whiOfPTz5AeDo8KeZFmODKnvwPymMNO8qE/an8pVY/O50tig2SQCrGw==" - }, - "type-fest": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.13.0.tgz", - "integrity": "sha512-lPfAm42MxE4/456+QyIaaVBAwgpJb6xZ8PRu09utnhPdWwcyj9vgy6Sq0Z5yNbJ21EdxB5dRU/Qg8bsyAMtlcw==" - } - } - }, - "lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" - }, "lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -5154,11 +4483,6 @@ "resolved": "https://registry.npmjs.org/node-worker-threads-pool/-/node-worker-threads-pool-1.5.1.tgz", "integrity": "sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw==" }, - "nofilter": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", - "integrity": "sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==" - }, "object-inspect": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", @@ -5250,26 +4574,6 @@ "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", "dev": true }, - "protobufjs": { - "version": "6.11.3", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.3.tgz", - "integrity": "sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==", - "requires": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": ">=13.7.0", - "long": "^4.0.0" - } - }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -5279,19 +4583,6 @@ "ipaddr.js": "1.9.1" } }, - "psbt": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/psbt/-/psbt-2.6.0.tgz", - "integrity": "sha512-z2ca00AMwZ6PfVETQNvXRumZdRwGuQzApIH/hKNp2o6Qo8N8TW7Ug2V+aSH2w/eC1b/bOOMZIE57V3jYN+kB4A==", - "requires": { - "bip66": "1.1.5", - "bitcoin-ops": "1.4.1", - "bitcoinjs-lib": "6.0.1", - "bn.js": "5.2.1", - "pushdata-bitcoin": "1.0.1", - "varuint-bitcoin": "1.1.2" - } - }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -5303,14 +4594,6 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, - "pushdata-bitcoin": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pushdata-bitcoin/-/pushdata-bitcoin-1.0.1.tgz", - "integrity": "sha512-hw7rcYTJRAl4olM8Owe8x0fBuJJ+WGbMhQuLWOXEMN3PxPCKQHRkhfL+XG0+iXUmSHjkMmb3Ba55Mt21cZc9kQ==", - "requires": { - "bitcoin-ops": "^1.3.0" - } - }, "qs": { "version": "6.10.3", "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", @@ -5325,14 +4608,6 @@ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true }, - "randombytes": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "requires": { - "safe-buffer": "^5.1.0" - } - }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -5365,11 +4640,6 @@ "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", "dev": true }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" - }, "resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -5564,20 +4834,11 @@ "safe-buffer": "~5.2.0" } }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "requires": { "ansi-regex": "^5.0.1" } @@ -5588,6 +4849,12 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", + "dev": true + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -5599,14 +4866,6 @@ "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==" }, - "tiny-secp256k1": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz", - "integrity": "sha512-/U4xfVqnVxJXN4YVsru0E6t5wVncu2uunB8+RVR40fYUxkKYUPS10f+ePQZgFBoE/Jbf9H1NBveupF2VmB58Ng==", - "requires": { - "uint8array-tools": "0.0.7" - } - }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -5661,11 +4920,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==" }, - "uint8array-tools": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/uint8array-tools/-/uint8array-tools-0.0.7.tgz", - "integrity": "sha512-vrrNZJiusLWoFWBqz5Y5KMCgP9W9hnjZHzZiZRT8oNAkq3d5Z5Oe76jAvVVSRh4U8GGR90N2X1dWtrhvx6L8UQ==" - }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -5732,16 +4986,6 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", "dev": true }, - "wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -5754,34 +4998,10 @@ "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==", "requires": {} }, - "y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" - }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "requires": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - } - }, - "yargs-parser": { - "version": "20.2.9", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", - "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==" } } } diff --git a/backend/package.json b/backend/package.json index b8930d6e5..47694ecf8 100644 --- a/backend/package.json +++ b/backend/package.json @@ -16,7 +16,8 @@ "mempool", "blockchain", "explorer", - "liquid" + "liquid", + "lightning" ], "main": "index.ts", "scripts": { @@ -34,10 +35,9 @@ "@types/node": "^16.11.41", "axios": "~0.27.2", "bitcoinjs-lib": "6.0.1", - "bolt07": "^1.8.1", "crypto-js": "^4.0.0", "express": "^4.18.0", - "lightning": "^5.16.3", + "fast-xml-parser": "^4.0.9", "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index e40977c6c..d702b4927 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -22,6 +22,8 @@ import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; +import PricesRepository from '../repositories/PricesRepository'; +import priceUpdater from '../tasks/price-updater'; class Blocks { private blocks: BlockExtended[] = []; @@ -457,6 +459,19 @@ class Blocks { } await blocksRepository.$saveBlockInDatabase(blockExtended); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === true && lastestPriceId !== null) { + await blocksRepository.$saveBlockPrices([{ + height: blockExtended.height, + priceId: lastestPriceId, + }]); + } else { + logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`) + setTimeout(() => { + indexer.runSingleTask('blocksPrices'); + }, 10000); + } + // Save blocks summary for visualization if it's enabled if (Common.blocksSummariesIndexingEnabled() === true) { await this.$getStrippedBlockTransactions(blockExtended.id, true); diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index fe6b858e0..410d34a01 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,5 +1,6 @@ import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces'; import config from '../config'; +import { convertChannelId } from './lightning/clightning/clightning-convert'; export class Common { static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ? '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49' @@ -184,4 +185,37 @@ export class Common { config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true ); } + + static setDateMidnight(date: Date): void { + date.setUTCHours(0); + date.setUTCMinutes(0); + date.setUTCSeconds(0); + date.setUTCMilliseconds(0); + } + + static channelShortIdToIntegerId(id: string): string { + if (config.LIGHTNING.BACKEND === 'lnd') { + return id; + } + return convertChannelId(id); + } + + /** Decodes a channel id returned by lnd as uint64 to a short channel id */ + static channelIntegerIdToShortId(id: string): string { + if (config.LIGHTNING.BACKEND === 'cln') { + return id; + } + + const n = BigInt(id); + return [ + n >> 40n, // nth block + (n >> 16n) & 0xffffffn, // nth tx of the block + n & 0xffffn // nth output of the tx + ].join('x'); + } + + static utcDateToMysql(date?: number): string { + const d = new Date((date || 0) * 1000); + return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; + } } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index d26bfd6cc..cfc0092d8 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 33; + private static currentVersion = 36; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -311,6 +311,19 @@ class DatabaseMigration { if (databaseSchemaVersion < 33 && isBitcoin == true) { await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL'); } + + if (databaseSchemaVersion < 34 && isBitcoin == true) { + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"'); + } + + if (databaseSchemaVersion < 35 && isBitcoin == true) { + await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);'); + } + + if (databaseSchemaVersion < 36 && isBitcoin == true) { + await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"'); + } } /** diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 79aeebb97..55043197d 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -1,5 +1,9 @@ import logger from '../../logger'; import DB from '../../database'; +import nodesApi from './nodes.api'; +import { ResultSetHeader } from 'mysql2'; +import { ILightningApi } from '../lightning/lightning-api.interface'; +import { Common } from '../common'; class ChannelsApi { public async $getAllChannels(): Promise { @@ -181,15 +185,57 @@ class ChannelsApi { public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise { try { - // Default active and inactive channels - let statusQuery = '< 2'; - // Closed channels only - if (status === 'closed') { - statusQuery = '= 2'; + let channelStatusFilter; + if (status === 'open') { + channelStatusFilter = '< 2'; + } else if (status === 'closed') { + channelStatusFilter = '= 2'; } - const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery} ORDER BY channels.capacity DESC LIMIT ?, ?`; - const [rows]: any = await DB.query(query, [public_key, public_key, index, length]); - const channels = rows.map((row) => this.convertChannel(row)); + + // Channels originating from node + let query = ` + SELECT node2.alias, node2.public_key, channels.status, channels.node1_fee_rate, + channels.capacity, channels.short_id, channels.id + FROM channels + JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key + WHERE node1_public_key = ? AND channels.status ${channelStatusFilter} + `; + const [channelsFromNode]: any = await DB.query(query, [public_key, index, length]); + + // Channels incoming to node + query = ` + SELECT node1.alias, node1.public_key, channels.status, channels.node2_fee_rate, + channels.capacity, channels.short_id, channels.id + FROM channels + JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key + WHERE node2_public_key = ? AND channels.status ${channelStatusFilter} + `; + const [channelsToNode]: any = await DB.query(query, [public_key, index, length]); + + let allChannels = channelsFromNode.concat(channelsToNode); + allChannels.sort((a, b) => { + return b.capacity - a.capacity; + }); + allChannels = allChannels.slice(index, index + length); + + const channels: any[] = [] + for (const row of allChannels) { + const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key); + channels.push({ + status: row.status, + capacity: row.capacity ?? 0, + short_id: row.short_id, + id: row.id, + fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0, + node: { + alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), + public_key: row.public_key, + channels: activeChannelsStats.active_channel_count ?? 0, + capacity: activeChannelsStats.capacity ?? 0, + } + }); + } + return channels; } catch (e) { logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e)); @@ -205,7 +251,12 @@ class ChannelsApi { if (status === 'closed') { statusQuery = '= 2'; } - const query = `SELECT COUNT(*) AS count FROM channels WHERE (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery}`; + const query = ` + SELECT COUNT(*) AS count + FROM channels + WHERE (node1_public_key = ? OR node2_public_key = ?) + AND status ${statusQuery} + `; const [rows]: any = await DB.query(query, [public_key, public_key]); return rows[0]['count']; } catch (e) { @@ -254,6 +305,135 @@ class ChannelsApi { }, }; } + + /** + * Save or update a channel present in the graph + */ + public async $saveChannel(channel: ILightningApi.Channel): Promise { + const [ txid, vout ] = channel.chan_point.split(':'); + + const policy1: Partial = channel.node1_policy || {}; + const policy2: Partial = channel.node2_policy || {}; + + const query = `INSERT INTO channels + ( + id, + short_id, + capacity, + transaction_id, + transaction_vout, + updated_at, + status, + node1_public_key, + node1_base_fee_mtokens, + node1_cltv_delta, + node1_fee_rate, + node1_is_disabled, + node1_max_htlc_mtokens, + node1_min_htlc_mtokens, + node1_updated_at, + node2_public_key, + node2_base_fee_mtokens, + node2_cltv_delta, + node2_fee_rate, + node2_is_disabled, + node2_max_htlc_mtokens, + node2_min_htlc_mtokens, + node2_updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + capacity = ?, + updated_at = ?, + status = 1, + node1_public_key = ?, + node1_base_fee_mtokens = ?, + node1_cltv_delta = ?, + node1_fee_rate = ?, + node1_is_disabled = ?, + node1_max_htlc_mtokens = ?, + node1_min_htlc_mtokens = ?, + node1_updated_at = ?, + node2_public_key = ?, + node2_base_fee_mtokens = ?, + node2_cltv_delta = ?, + node2_fee_rate = ?, + node2_is_disabled = ?, + node2_max_htlc_mtokens = ?, + node2_min_htlc_mtokens = ?, + node2_updated_at = ? + ;`; + + await DB.query(query, [ + Common.channelShortIdToIntegerId(channel.channel_id), + Common.channelIntegerIdToShortId(channel.channel_id), + channel.capacity, + txid, + vout, + Common.utcDateToMysql(channel.last_update), + channel.node1_pub, + policy1.fee_base_msat, + policy1.time_lock_delta, + policy1.fee_rate_milli_msat, + policy1.disabled, + policy1.max_htlc_msat, + policy1.min_htlc, + Common.utcDateToMysql(policy1.last_update), + channel.node2_pub, + policy2.fee_base_msat, + policy2.time_lock_delta, + policy2.fee_rate_milli_msat, + policy2.disabled, + policy2.max_htlc_msat, + policy2.min_htlc, + Common.utcDateToMysql(policy2.last_update), + channel.capacity, + Common.utcDateToMysql(channel.last_update), + channel.node1_pub, + policy1.fee_base_msat, + policy1.time_lock_delta, + policy1.fee_rate_milli_msat, + policy1.disabled, + policy1.max_htlc_msat, + policy1.min_htlc, + Common.utcDateToMysql(policy1.last_update), + channel.node2_pub, + policy2.fee_base_msat, + policy2.time_lock_delta, + policy2.fee_rate_milli_msat, + policy2.disabled, + policy2.max_htlc_msat, + policy2.min_htlc, + Common.utcDateToMysql(policy2.last_update) + ]); + } + + /** + * Set all channels not in `graphChannelsIds` as inactive (status = 0) + */ + public async $setChannelsInactive(graphChannelsIds: string[]): Promise { + if (graphChannelsIds.length === 0) { + return; + } + + try { + const result = await DB.query(` + UPDATE channels + SET status = 0 + WHERE short_id NOT IN ( + ${graphChannelsIds.map(id => `"${id}"`).join(',')} + ) + AND status != 2 + `); + if (result[0].changedRows ?? 0 > 0) { + logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`); + } else { + logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`); + } + } catch (e) { + logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e)); + } + } } export default new ChannelsApi(); diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index 495eec789..bbb075aa6 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -46,9 +46,11 @@ class ChannelsRoutes { } const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0; const status: string = typeof req.query.status === 'string' ? req.query.status : ''; - const length = 25; - const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, length, status); + const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status); const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.header('X-Total-Count', channelsCount.toString()); res.json(channels); } catch (e) { diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 55b0ba5cb..d4857a3a4 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -1,24 +1,18 @@ import logger from '../../logger'; import DB from '../../database'; +import { ResultSetHeader } from 'mysql2'; +import { ILightningApi } from '../lightning/lightning-api.interface'; class NodesApi { public async $getNode(public_key: string): Promise { try { - const query = ` - SELECT nodes.*, geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city, - geo_names_country.names as country, geo_names_subdivision.names as subdivision, - (SELECT Count(*) - FROM channels - WHERE channels.status = 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_closed_count, - (SELECT Count(*) - FROM channels - WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_active_count, - (SELECT Sum(capacity) - FROM channels - WHERE channels.status = 1 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS capacity, - (SELECT Avg(capacity) - FROM channels - WHERE status = 1 AND ( node1_public_key = ? OR node2_public_key = ? )) AS channels_capacity_avg + // General info + let query = ` + SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen, + UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets, + as_number, city_id, country_id, subdivision_id, longitude, latitude, + geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city, + geo_names_country.names as country, geo_names_subdivision.names as subdivision FROM nodes LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id @@ -27,21 +21,70 @@ class NodesApi { LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' WHERE public_key = ? `; - const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key, public_key]); - if (rows.length > 0) { - rows[0].as_organization = JSON.parse(rows[0].as_organization); - rows[0].subdivision = JSON.parse(rows[0].subdivision); - rows[0].city = JSON.parse(rows[0].city); - rows[0].country = JSON.parse(rows[0].country); - return rows[0]; + let [rows]: any[] = await DB.query(query, [public_key]); + if (rows.length === 0) { + throw new Error(`This node does not exist, or our node is not seeing it yet`); } - return null; + + const node = rows[0]; + node.as_organization = JSON.parse(node.as_organization); + node.subdivision = JSON.parse(node.subdivision); + node.city = JSON.parse(node.city); + node.country = JSON.parse(node.country); + + // Active channels and capacity + const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key); + node.active_channel_count = activeChannelsStats.active_channel_count ?? 0; + node.capacity = activeChannelsStats.capacity ?? 0; + + // Opened channels count + query = ` + SELECT count(short_id) as opened_channel_count + FROM channels + WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?) + `; + [rows] = await DB.query(query, [public_key, public_key]); + node.opened_channel_count = 0; + if (rows.length > 0) { + node.opened_channel_count = rows[0].opened_channel_count; + } + + // Closed channels count + query = ` + SELECT count(short_id) as closed_channel_count + FROM channels + WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?) + `; + [rows] = await DB.query(query, [public_key, public_key]); + node.closed_channel_count = 0; + if (rows.length > 0) { + node.closed_channel_count = rows[0].closed_channel_count; + } + + return node; } catch (e) { - logger.err('$getNode error: ' + (e instanceof Error ? e.message : e)); + logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`); throw e; } } + public async $getActiveChannelsStats(node_public_key: string): Promise { + const query = ` + SELECT count(short_id) as active_channel_count, sum(capacity) as capacity + FROM channels + WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?) + `; + const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]); + if (rows.length > 0) { + return { + active_channel_count: rows[0].active_channel_count, + capacity: rows[0].capacity + }; + } else { + return null; + } + } + public async $getAllNodes(): Promise { try { const query = `SELECT * FROM nodes`; @@ -55,7 +98,12 @@ class NodesApi { public async $getNodeStats(public_key: string): Promise { try { - const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`; + const query = ` + SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels + FROM node_stats + WHERE public_key = ? + ORDER BY added DESC + `; const [rows]: any = await DB.query(query, [public_key]); return rows; } catch (e) { @@ -66,8 +114,19 @@ class NodesApi { public async $getTopCapacityNodes(): Promise { try { - const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.capacity DESC LIMIT 10`; - const [rows]: any = await DB.query(query); + let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); + const latestDate = rows[0].maxAdded; + + const query = ` + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels + FROM node_stats + JOIN nodes ON nodes.public_key = node_stats.public_key + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY capacity DESC + LIMIT 10; + `; + [rows] = await DB.query(query); + return rows; } catch (e) { logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e)); @@ -77,8 +136,19 @@ class NodesApi { public async $getTopChannelsNodes(): Promise { try { - const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.channels DESC LIMIT 10`; - const [rows]: any = await DB.query(query); + let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); + const latestDate = rows[0].maxAdded; + + const query = ` + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels + FROM node_stats + JOIN nodes ON nodes.public_key = node_stats.public_key + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY channels DESC + LIMIT 10; + `; + [rows] = await DB.query(query); + return rows; } catch (e) { logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e)); @@ -163,8 +233,8 @@ class NodesApi { public async $getNodesPerCountry(countryId: string) { try { const query = ` - SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias, - UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, + SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city FROM node_stats JOIN ( @@ -172,7 +242,7 @@ class NodesApi { FROM node_stats GROUP BY public_key ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added - JOIN nodes ON nodes.public_key = node_stats.public_key + RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' WHERE geo_names_country.id = ? @@ -193,8 +263,8 @@ class NodesApi { public async $getNodesPerISP(ISPId: string) { try { const query = ` - SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias, - UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, + SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at, geo_names_city.names as city, geo_names_country.names as country FROM node_stats JOIN ( @@ -253,6 +323,66 @@ class NodesApi { throw e; } } + + /** + * Save or update a node present in the graph + */ + public async $saveNode(node: ILightningApi.Node): Promise { + try { + const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? ''; + const query = `INSERT INTO nodes( + public_key, + first_seen, + updated_at, + alias, + color, + sockets, + status + ) + VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, 1) + ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?, status = 1`; + + await DB.query(query, [ + node.pub_key, + node.last_update, + node.alias, + node.color, + sockets, + node.last_update, + node.alias, + node.color, + sockets, + ]); + } catch (e) { + logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e)); + } + } + + /** + * Set all nodes not in `nodesPubkeys` as inactive (status = 0) + */ + public async $setNodesInactive(graphNodesPubkeys: string[]): Promise { + if (graphNodesPubkeys.length === 0) { + return; + } + + try { + const result = await DB.query(` + UPDATE nodes + SET status = 0 + WHERE public_key NOT IN ( + ${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')} + ) + `); + if (result[0].changedRows ?? 0 > 0) { + logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`); + } else { + logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`); + } + } catch (e) { + logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e)); + } + } } export default new NodesApi(); diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index 83e3c393e..a850b6a09 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -35,6 +35,9 @@ class NodesRoutes { res.status(404).send('Node not found'); return; } + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.json(node); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); @@ -44,6 +47,9 @@ class NodesRoutes { private async $getHistoricalNodeStats(req: Request, res: Response) { try { const statistics = await nodesApi.$getNodeStats(req.params.public_key); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.json(statistics); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts new file mode 100644 index 000000000..0535e0881 --- /dev/null +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -0,0 +1,272 @@ +// Imported from https://github.com/shesek/lightning-client-js + +'use strict'; + +const methods = [ + 'addgossip', + 'autocleaninvoice', + 'check', + 'checkmessage', + 'close', + 'connect', + 'createinvoice', + 'createinvoicerequest', + 'createoffer', + 'createonion', + 'decode', + 'decodepay', + 'delexpiredinvoice', + 'delinvoice', + 'delpay', + 'dev-listaddrs', + 'dev-rescan-outputs', + 'disableoffer', + 'disconnect', + 'estimatefees', + 'feerates', + 'fetchinvoice', + 'fundchannel', + 'fundchannel_cancel', + 'fundchannel_complete', + 'fundchannel_start', + 'fundpsbt', + 'getchaininfo', + 'getinfo', + 'getlog', + 'getrawblockbyheight', + 'getroute', + 'getsharedsecret', + 'getutxout', + 'help', + 'invoice', + 'keysend', + 'legacypay', + 'listchannels', + 'listconfigs', + 'listforwards', + 'listfunds', + 'listinvoices', + 'listnodes', + 'listoffers', + 'listpays', + 'listpeers', + 'listsendpays', + 'listtransactions', + 'multifundchannel', + 'multiwithdraw', + 'newaddr', + 'notifications', + 'offer', + 'offerout', + 'openchannel_abort', + 'openchannel_bump', + 'openchannel_init', + 'openchannel_signed', + 'openchannel_update', + 'pay', + 'payersign', + 'paystatus', + 'ping', + 'plugin', + 'reserveinputs', + 'sendinvoice', + 'sendonion', + 'sendonionmessage', + 'sendpay', + 'sendpsbt', + 'sendrawtransaction', + 'setchannelfee', + 'signmessage', + 'signpsbt', + 'stop', + 'txdiscard', + 'txprepare', + 'txsend', + 'unreserveinputs', + 'utxopsbt', + 'waitanyinvoice', + 'waitblockheight', + 'waitinvoice', + 'waitsendpay', + 'withdraw' +]; + + +import EventEmitter from 'events'; +import { existsSync, statSync } from 'fs'; +import { createConnection, Socket } from 'net'; +import { homedir } from 'os'; +import path from 'path'; +import { createInterface, Interface } from 'readline'; +import logger from '../../../logger'; +import { AbstractLightningApi } from '../lightning-api-abstract-factory'; +import { ILightningApi } from '../lightning-api.interface'; +import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert'; + +class LightningError extends Error { + type: string = 'lightning'; + message: string = 'lightning-client error'; + + constructor(error) { + super(); + this.type = error.type; + this.message = error.message; + } +} + +const defaultRpcPath = path.join(homedir(), '.lightning') + , fStat = (...p) => statSync(path.join(...p)) + , fExists = (...p) => existsSync(path.join(...p)) + +export default class CLightningClient extends EventEmitter implements AbstractLightningApi { + private rpcPath: string; + private reconnectWait: number; + private reconnectTimeout; + private reqcount: number; + private client: Socket; + private rl: Interface; + private clientConnectionPromise: Promise; + + constructor(rpcPath = defaultRpcPath) { + if (!path.isAbsolute(rpcPath)) { + throw new Error('The rpcPath must be an absolute path'); + } + + if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) { + // network directory provided, use the lightning-rpc within in + if (fExists(rpcPath, 'lightning-rpc')) { + rpcPath = path.join(rpcPath, 'lightning-rpc'); + } + + // main data directory provided, default to using the bitcoin mainnet subdirectory + // to be removed in v0.2.0 + else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) { + logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`) + logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`) + rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc') + } + } + + logger.debug(`[CLightningClient] Connecting to ${rpcPath}`); + + super(); + this.rpcPath = rpcPath; + this.reconnectWait = 0.5; + this.reconnectTimeout = null; + this.reqcount = 0; + + const _self = this; + + this.client = createConnection(rpcPath).on( + 'error', () => { + _self.increaseWaitTime(); + _self.reconnect(); + } + ); + this.rl = createInterface({ input: this.client }).on( + 'error', () => { + _self.increaseWaitTime(); + _self.reconnect(); + } + ); + + this.clientConnectionPromise = new Promise(resolve => { + _self.client.on('connect', () => { + logger.info(`[CLightningClient] Lightning client connected`); + _self.reconnectWait = 1; + resolve(); + }); + + _self.client.on('end', () => { + logger.err('[CLightningClient] Lightning client connection closed, reconnecting'); + _self.increaseWaitTime(); + _self.reconnect(); + }); + + _self.client.on('error', error => { + logger.err(`[CLightningClient] Lightning client connection error: ${error}`); + _self.increaseWaitTime(); + _self.reconnect(); + }); + }); + + this.rl.on('line', line => { + line = line.trim(); + if (!line) { + return; + } + const data = JSON.parse(line); + // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); + _self.emit('res:' + data.id, data); + }); + } + + increaseWaitTime(): void { + if (this.reconnectWait >= 16) { + this.reconnectWait = 16; + } else { + this.reconnectWait *= 2; + } + } + + reconnect(): void { + const _self = this; + + if (this.reconnectTimeout) { + return; + } + + this.reconnectTimeout = setTimeout(() => { + logger.debug('[CLightningClient] Trying to reconnect...'); + + _self.client.connect(_self.rpcPath); + _self.reconnectTimeout = null; + }, this.reconnectWait * 1000); + } + + call(method, args = []): Promise { + const _self = this; + + const callInt = ++this.reqcount; + const sendObj = { + jsonrpc: '2.0', + method, + params: args, + id: '' + callInt + }; + + // logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`); + + // Wait for the client to connect + return this.clientConnectionPromise + .then(() => new Promise((resolve, reject) => { + // Wait for a response + this.once('res:' + callInt, res => res.error == null + ? resolve(res.result) + : reject(new LightningError(res.error)) + ); + + // Send the command + _self.client.write(JSON.stringify(sendObj)); + })); + } + + async $getNetworkGraph(): Promise { + const listnodes: any[] = await this.call('listnodes'); + const listchannels: any[] = await this.call('listchannels'); + const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']); + + return { + nodes: listnodes['nodes'].map(node => convertNode(node)), + edges: channelsList, + }; + } +} + +const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase()); + +methods.forEach(k => { + CLightningClient.prototype[protify(k)] = function (...args: any) { + return this.call(k, args); + }; +}); diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts new file mode 100644 index 000000000..5df51aadc --- /dev/null +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -0,0 +1,138 @@ +import { ILightningApi } from '../lightning-api.interface'; +import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher'; +import logger from '../../../logger'; + +/** + * Convert a clightning "listnode" entry to a lnd node entry + */ +export function convertNode(clNode: any): ILightningApi.Node { + return { + alias: clNode.alias ?? '', + color: `#${clNode.color ?? ''}`, + features: [], // TODO parse and return clNode.feature + pub_key: clNode.nodeid, + addresses: clNode.addresses?.map((addr) => { + return { + network: addr.type, + addr: `${addr.address}:${addr.port}` + }; + }), + last_update: clNode?.last_timestamp ?? 0, + }; +} + +/** + * Convert clightning "listchannels" response to lnd "describegraph.edges" format + */ +export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise { + logger.info('Converting clightning nodes and channels to lnd graph format'); + + let loggerTimer = new Date().getTime() / 1000; + let channelProcessed = 0; + + const consolidatedChannelList: ILightningApi.Channel[] = []; + const clChannelsDict = {}; + const clChannelsDictCount = {}; + + for (const clChannel of clChannels) { + if (!clChannelsDict[clChannel.short_channel_id]) { + clChannelsDict[clChannel.short_channel_id] = clChannel; + clChannelsDictCount[clChannel.short_channel_id] = 1; + } else { + consolidatedChannelList.push( + await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) + ); + delete clChannelsDict[clChannel.short_channel_id]; + clChannelsDictCount[clChannel.short_channel_id]++; + } + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`); + loggerTimer = new Date().getTime() / 1000; + } + + ++channelProcessed; + } + + channelProcessed = 0; + const keys = Object.keys(clChannelsDict); + for (const short_channel_id of keys) { + consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id])); + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`); + loggerTimer = new Date().getTime() / 1000; + } + } + + return consolidatedChannelList; +} + +export function convertChannelId(channelId): string { + if (channelId.indexOf('/') !== -1) { + channelId = channelId.slice(0, -2); + } + const s = channelId.split('x').map(part => BigInt(part)); + return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString(); +} + +/** + * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format + * In this case, clightning knows the channel policy for both nodes + */ +async function buildFullChannel(clChannelA: any, clChannelB: any): Promise { + const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0); + + const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id); + const parts = clChannelA.short_channel_id.split('x'); + const outputIdx = parts[2]; + + return { + channel_id: clChannelA.short_channel_id, + capacity: clChannelA.satoshis, + last_update: lastUpdate, + node1_policy: convertPolicy(clChannelA), + node2_policy: convertPolicy(clChannelB), + chan_point: `${tx.txid}:${outputIdx}`, + node1_pub: clChannelA.source, + node2_pub: clChannelB.source, + }; +} + +/** + * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format + * In this case, clightning knows the channel policy of only one node + */ +async function buildIncompleteChannel(clChannel: any): Promise { + const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id); + const parts = clChannel.short_channel_id.split('x'); + const outputIdx = parts[2]; + + return { + channel_id: clChannel.short_channel_id, + capacity: clChannel.satoshis, + last_update: clChannel.last_update ?? 0, + node1_policy: convertPolicy(clChannel), + node2_policy: null, + chan_point: `${tx.txid}:${outputIdx}`, + node1_pub: clChannel.source, + node2_pub: clChannel.destination, + }; +} + +/** + * Convert a clightning "listnode" response to a lnd channel policy format + */ +function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy { + return { + time_lock_delta: 0, // TODO + min_htlc: clChannel.htlc_minimum_msat.slice(0, -4), + max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4), + fee_base_msat: clChannel.base_fee_millisatoshi, + fee_rate_milli_msat: clChannel.fee_per_millionth, + disabled: !clChannel.active, + last_update: clChannel.last_update ?? 0, + }; +} diff --git a/backend/src/api/lightning/lightning-api-abstract-factory.ts b/backend/src/api/lightning/lightning-api-abstract-factory.ts index 026568c6d..e6691b0a4 100644 --- a/backend/src/api/lightning/lightning-api-abstract-factory.ts +++ b/backend/src/api/lightning/lightning-api-abstract-factory.ts @@ -1,7 +1,5 @@ import { ILightningApi } from './lightning-api.interface'; export interface AbstractLightningApi { - $getNetworkInfo(): Promise; $getNetworkGraph(): Promise; - $getInfo(): Promise; } diff --git a/backend/src/api/lightning/lightning-api-factory.ts b/backend/src/api/lightning/lightning-api-factory.ts index ab551095c..fdadd8230 100644 --- a/backend/src/api/lightning/lightning-api-factory.ts +++ b/backend/src/api/lightning/lightning-api-factory.ts @@ -1,9 +1,12 @@ import config from '../../config'; +import CLightningClient from './clightning/clightning-client'; import { AbstractLightningApi } from './lightning-api-abstract-factory'; import LndApi from './lnd/lnd-api'; function lightningApiFactory(): AbstractLightningApi { - switch (config.LIGHTNING.BACKEND) { + switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) { + case 'cln': + return new CLightningClient(config.CLIGHTNING.SOCKET); case 'lnd': default: return new LndApi(); diff --git a/backend/src/api/lightning/lightning-api.interface.ts b/backend/src/api/lightning/lightning-api.interface.ts index 9b83b5473..283f34a5a 100644 --- a/backend/src/api/lightning/lightning-api.interface.ts +++ b/backend/src/api/lightning/lightning-api.interface.ts @@ -1,71 +1,85 @@ export namespace ILightningApi { export interface NetworkInfo { - average_channel_size: number; - channel_count: number; - max_channel_size: number; - median_channel_size: number; - min_channel_size: number; - node_count: number; - not_recently_updated_policy_count: number; - total_capacity: number; + graph_diameter: number; + avg_out_degree: number; + max_out_degree: number; + num_nodes: number; + num_channels: number; + total_network_capacity: string; + avg_channel_size: number; + min_channel_size: string; + max_channel_size: string; + median_channel_size_sat: string; + num_zombie_chans: string; } export interface NetworkGraph { - channels: Channel[]; nodes: Node[]; + edges: Channel[]; } export interface Channel { - id: string; - capacity: number; - policies: Policy[]; - transaction_id: string; - transaction_vout: number; - updated_at?: string; + channel_id: string; + chan_point: string; + last_update: number; + node1_pub: string; + node2_pub: string; + capacity: string; + node1_policy: RoutingPolicy | null; + node2_policy: RoutingPolicy | null; } - interface Policy { - public_key: string; - base_fee_mtokens?: string; - cltv_delta?: number; - fee_rate?: number; - is_disabled?: boolean; - max_htlc_mtokens?: string; - min_htlc_mtokens?: string; - updated_at?: string; + export interface RoutingPolicy { + time_lock_delta: number; + min_htlc: string; + fee_base_msat: string; + fee_rate_milli_msat: string; + disabled: boolean; + max_htlc_msat: string; + last_update: number; } export interface Node { + last_update: number; + pub_key: string; alias: string; + addresses: { + network: string; + addr: string; + }[]; color: string; - features: Feature[]; - public_key: string; - sockets: string[]; - updated_at?: string; + features: { [key: number]: Feature }; } export interface Info { - chains: string[]; - color: string; - active_channels_count: number; + identity_pubkey: string; alias: string; - current_block_hash: string; - current_block_height: number; - features: Feature[]; - is_synced_to_chain: boolean; - is_synced_to_graph: boolean; - latest_block_at: string; - peers_count: number; - pending_channels_count: number; - public_key: string; - uris: any[]; + num_pending_channels: number; + num_active_channels: number; + num_peers: number; + block_height: number; + block_hash: string; + synced_to_chain: boolean; + testnet: boolean; + uris: string[]; + best_header_timestamp: string; version: string; + num_inactive_channels: number; + chains: { + chain: string; + network: string; + }[]; + color: string; + synced_to_graph: boolean; + features: { [key: number]: Feature }; + commit_hash: string; + /** Available on LND since v0.15.0-beta */ + require_htlc_interceptor?: boolean; } - + export interface Feature { - bit: number; - is_known: boolean; + name: string; is_required: boolean; - type?: string; + is_known: boolean; } } diff --git a/backend/src/api/lightning/lnd/lnd-api.ts b/backend/src/api/lightning/lnd/lnd-api.ts index 19d98744d..1480f9b8f 100644 --- a/backend/src/api/lightning/lnd/lnd-api.ts +++ b/backend/src/api/lightning/lnd/lnd-api.ts @@ -1,44 +1,40 @@ +import axios, { AxiosRequestConfig } from 'axios'; +import { Agent } from 'https'; +import * as fs from 'fs'; import { AbstractLightningApi } from '../lightning-api-abstract-factory'; import { ILightningApi } from '../lightning-api.interface'; -import * as fs from 'fs'; -import { authenticatedLndGrpc, getWalletInfo, getNetworkGraph, getNetworkInfo } from 'lightning'; import config from '../../../config'; -import logger from '../../../logger'; class LndApi implements AbstractLightningApi { - private lnd: any; + axiosConfig: AxiosRequestConfig = {}; + constructor() { - if (!config.LIGHTNING.ENABLED) { - return; - } - try { - const tls = fs.readFileSync(config.LND.TLS_CERT_PATH).toString('base64'); - const macaroon = fs.readFileSync(config.LND.MACAROON_PATH).toString('base64'); - - const { lnd } = authenticatedLndGrpc({ - cert: tls, - macaroon: macaroon, - socket: config.LND.SOCKET, - }); - - this.lnd = lnd; - } catch (e) { - logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e)); - process.exit(1); + if (config.LIGHTNING.ENABLED) { + this.axiosConfig = { + headers: { + 'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex') + }, + httpsAgent: new Agent({ + ca: fs.readFileSync(config.LND.TLS_CERT_PATH) + }), + timeout: 10000 + }; } } async $getNetworkInfo(): Promise { - return await getNetworkInfo({ lnd: this.lnd }); + return axios.get(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig) + .then((response) => response.data); } async $getInfo(): Promise { - // @ts-ignore - return await getWalletInfo({ lnd: this.lnd }); + return axios.get(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig) + .then((response) => response.data); } async $getNetworkGraph(): Promise { - return await getNetworkGraph({ lnd: this.lnd }); + return axios.get(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig) + .then((response) => response.data); } } diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 55e749596..55cd33bd3 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -473,7 +473,7 @@ class Mining { for (const block of blocksWithoutPrices) { // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks - if (block.height < 68951) { + if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) { blocksPrices.push({ height: block.height, priceId: prices[0].id, @@ -492,11 +492,11 @@ class Mining { if (blocksPrices.length >= 100000) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); blocksPrices.length = 0; } @@ -504,11 +504,11 @@ class Mining { if (blocksPrices.length > 0) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); } } catch (e) { diff --git a/backend/src/config.ts b/backend/src/config.ts index 0e3382517..ddf1fd3d4 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -31,10 +31,16 @@ interface IConfig { LIGHTNING: { ENABLED: boolean; BACKEND: 'lnd' | 'cln' | 'ldk'; + TOPOLOGY_FOLDER: string; + STATS_REFRESH_INTERVAL: number; + GRAPH_REFRESH_INTERVAL: number; }; LND: { TLS_CERT_PATH: string; MACAROON_PATH: string; + REST_API_URL: string; + }; + CLIGHTNING: { SOCKET: string; }; ELECTRUM: { @@ -177,12 +183,18 @@ const defaults: IConfig = { }, 'LIGHTNING': { 'ENABLED': false, - 'BACKEND': 'lnd' + 'BACKEND': 'lnd', + 'TOPOLOGY_FOLDER': '', + 'STATS_REFRESH_INTERVAL': 600, + 'GRAPH_REFRESH_INTERVAL': 600, }, 'LND': { 'TLS_CERT_PATH': '', 'MACAROON_PATH': '', - 'SOCKET': 'localhost:10009', + 'REST_API_URL': 'https://localhost:8080', + }, + 'CLIGHTNING': { + 'SOCKET': '', }, 'SOCKS5PROXY': { 'ENABLED': false, @@ -224,6 +236,7 @@ class Config implements IConfig { BISQ: IConfig['BISQ']; LIGHTNING: IConfig['LIGHTNING']; LND: IConfig['LND']; + CLIGHTNING: IConfig['CLIGHTNING']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; @@ -242,6 +255,7 @@ class Config implements IConfig { this.BISQ = configs.BISQ; this.LIGHTNING = configs.LIGHTNING; this.LND = configs.LND; + this.CLIGHTNING = configs.CLIGHTNING; this.SOCKS5PROXY = configs.SOCKS5PROXY; this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; diff --git a/backend/src/database.ts b/backend/src/database.ts index 66c876378..c2fb0980b 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -1,7 +1,7 @@ import config from './config'; import { createPool, Pool, PoolConnection } from 'mysql2/promise'; import logger from './logger'; -import { PoolOptions } from 'mysql2/typings/mysql'; +import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql'; class DB { constructor() { @@ -28,7 +28,9 @@ import { PoolOptions } from 'mysql2/typings/mysql'; } } - public async query(query, params?) { + public async query(query, params?): Promise<[T, FieldPacket[]]> + { this.checkDBFlag(); const pool = await this.getPool(); return pool.query(query, params); diff --git a/backend/src/index.ts b/backend/src/index.ts index b7159afaf..683f964f0 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -28,12 +28,13 @@ import nodesRoutes from './api/explorer/nodes.routes'; import channelsRoutes from './api/explorer/channels.routes'; import generalLightningRoutes from './api/explorer/general.routes'; import lightningStatsUpdater from './tasks/lightning/stats-updater.service'; -import nodeSyncService from './tasks/lightning/node-sync.service'; -import statisticsRoutes from "./api/statistics/statistics.routes"; -import miningRoutes from "./api/mining/mining-routes"; -import bisqRoutes from "./api/bisq/bisq.routes"; -import liquidRoutes from "./api/liquid/liquid.routes"; -import bitcoinRoutes from "./api/bitcoin/bitcoin.routes"; +import networkSyncService from './tasks/lightning/network-sync.service'; +import statisticsRoutes from './api/statistics/statistics.routes'; +import miningRoutes from './api/mining/mining-routes'; +import bisqRoutes from './api/bisq/bisq.routes'; +import liquidRoutes from './api/liquid/liquid.routes'; +import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; +import fundingTxFetcher from "./tasks/lightning/sync-tasks/funding-tx-fetcher"; class Server { private wss: WebSocket.Server | undefined; @@ -136,8 +137,7 @@ class Server { } if (config.LIGHTNING.ENABLED) { - nodeSyncService.$startService() - .then(() => lightningStatsUpdater.$startService()); + this.$runLightningBackend(); } this.server.listen(config.MEMPOOL.HTTP_PORT, () => { @@ -183,6 +183,18 @@ class Server { } } + async $runLightningBackend() { + try { + await fundingTxFetcher.$init(); + await networkSyncService.$startService(); + await lightningStatsUpdater.$startService(); + } catch(e) { + logger.err(`Lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`); + await Common.sleep$(1000 * 60); + this.$runLightningBackend(); + }; +} + setUpWebsocketHandling() { if (this.wss) { websocketHandler.setWebsocketServer(this.wss); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index e452a42f4..26a407291 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -6,13 +6,12 @@ import logger from './logger'; import HashratesRepository from './repositories/HashratesRepository'; import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; +import PricesRepository from './repositories/PricesRepository'; class Indexer { runIndexer = true; indexerRunning = false; - - constructor() { - } + tasksRunning: string[] = []; public reindex() { if (Common.indexingEnabled()) { @@ -20,6 +19,28 @@ class Indexer { } } + public async runSingleTask(task: 'blocksPrices') { + if (!Common.indexingEnabled()) { + return; + } + + if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) { + this.tasksRunning.push(task); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === false || lastestPriceId === null) { + logger.debug(`Blocks prices indexer is waiting for the price updater to complete`) + setTimeout(() => { + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.runSingleTask('blocksPrices'); + }, 10000); + } else { + logger.debug(`Blocks prices indexer will run now`) + await mining.$indexBlockPrices(); + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + } + } + } + public async $run() { if (!Common.indexingEnabled() || this.runIndexer === false || this.indexerRunning === true || mempool.hasPriority() @@ -50,7 +71,7 @@ class Indexer { return; } - await mining.$indexBlockPrices(); + this.runSingleTask('blocksPrices'); await mining.$indexDifficultyAdjustments(); await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient await mining.$generateNetworkHashrateHistory(); diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 92fb4860f..cc79ff2a6 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -27,6 +27,11 @@ class PricesRepository { return oldestRow[0] ? oldestRow[0].time : 0; } + public async $getLatestPriceId(): Promise { + const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); + return oldestRow[0] ? oldestRow[0].id : null; + } + public async $getLatestPriceTime(): Promise { const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); return oldestRow[0] ? oldestRow[0].time : 0; diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts similarity index 51% rename from backend/src/tasks/lightning/node-sync.service.ts rename to backend/src/tasks/lightning/network-sync.service.ts index f45473aba..b87c63031 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -1,53 +1,43 @@ -import { chanNumber } from 'bolt07'; import DB from '../../database'; import logger from '../../logger'; import channelsApi from '../../api/explorer/channels.api'; -import bitcoinClient from '../../api/bitcoin/bitcoin-client'; import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory'; import config from '../../config'; import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; -import lightningApi from '../../api/lightning/lightning-api-factory'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { $lookupNodeLocation } from './sync-tasks/node-locations'; +import lightningApi from '../../api/lightning/lightning-api-factory'; +import nodesApi from '../../api/explorer/nodes.api'; +import { ResultSetHeader } from 'mysql2'; +import fundingTxFetcher from './sync-tasks/funding-tx-fetcher'; + +class NetworkSyncService { + loggerTimer = 0; -class NodeSyncService { constructor() {} - public async $startService() { - logger.info('Starting node sync service'); + public async $startService(): Promise { + logger.info('Starting lightning network sync service'); - await this.$runUpdater(); + this.loggerTimer = new Date().getTime() / 1000; - setInterval(async () => { - await this.$runUpdater(); - }, 1000 * 60 * 60); + await this.$runTasks(); } - private async $runUpdater() { + private async $runTasks(): Promise { try { - logger.info(`Updating nodes and channels...`); + logger.info(`Updating nodes and channels`); const networkGraph = await lightningApi.$getNetworkGraph(); - - for (const node of networkGraph.nodes) { - await this.$saveNode(node); - } - logger.info(`Nodes updated.`); - - if (config.MAXMIND.ENABLED) { - await $lookupNodeLocation(); + if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) { + logger.info(`LN Network graph is empty, retrying in 10 seconds`); + setTimeout(() => { this.$runTasks(); }, 10000); + return; } - const graphChannelsIds: string[] = []; - for (const channel of networkGraph.channels) { - await this.$saveChannel(channel); - graphChannelsIds.push(channel.id); - } - await this.$setChannelsInactive(graphChannelsIds); - - logger.info(`Channels updated.`); - - await this.$findInactiveNodesAndChannels(); + await this.$updateNodesList(networkGraph.nodes); + await this.$updateChannelsList(networkGraph.edges); + await this.$deactivateChannelsWithoutActiveNodes(); await this.$lookUpCreationDateFromChain(); await this.$updateNodeFirstSeen(); await this.$scanForClosedChannels(); @@ -56,85 +46,183 @@ class NodeSyncService { } } catch (e) { - logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e)); + logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e)); } + + setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL); + } + + /** + * Update the `nodes` table to reflect the current network graph state + */ + private async $updateNodesList(nodes: ILightningApi.Node[]): Promise { + let progress = 0; + + const graphNodesPubkeys: string[] = []; + for (const node of nodes) { + await nodesApi.$saveNode(node); + graphNodesPubkeys.push(node.pub_key); + ++progress; + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating node ${progress}/${nodes.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } + } + logger.info(`${progress} nodes updated`); + + // If a channel if not present in the graph, mark it as inactive + nodesApi.$setNodesInactive(graphNodesPubkeys); + + if (config.MAXMIND.ENABLED) { + $lookupNodeLocation(); + } + } + + /** + * Update the `channels` table to reflect the current network graph state + */ + private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise { + try { + let progress = 0; + + const graphChannelsIds: string[] = []; + for (const channel of channels) { + await channelsApi.$saveChannel(channel); + graphChannelsIds.push(channel.channel_id); + ++progress; + + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } + } + + logger.info(`${progress} channels updated`); + + // If a channel if not present in the graph, mark it as inactive + channelsApi.$setChannelsInactive(graphChannelsIds); + } catch (e) { + logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`); + } + + setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL); } // This method look up the creation date of the earliest channel of the node // and update the node to that date in order to get the earliest first seen date - private async $updateNodeFirstSeen() { + private async $updateNodeFirstSeen(): Promise { + let progress = 0; + let updated = 0; + try { - const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`); + const [nodes]: any[] = await DB.query(` + SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, + ( + SELECT MIN(UNIX_TIMESTAMP(created)) + FROM channels + WHERE channels.node1_public_key = nodes.public_key + ) AS created1, + ( + SELECT MIN(UNIX_TIMESTAMP(created)) + FROM channels + WHERE channels.node2_public_key = nodes.public_key + ) AS created2 + FROM nodes + `); + for (const node of nodes) { - let lowest = 0; - if (node.created1) { - if (node.created2 && node.created2 < node.created1) { - lowest = node.created2; - } else { - lowest = node.created1; - } - } else if (node.created2) { - lowest = node.created2; - } - if (lowest && lowest < node.first_seen) { + const lowest = Math.min( + node.created1 ?? Number.MAX_SAFE_INTEGER, + node.created2 ?? Number.MAX_SAFE_INTEGER, + node.first_seen ?? Number.MAX_SAFE_INTEGER + ); + if (lowest < node.first_seen) { const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`; const params = [lowest, node.public_key]; await DB.query(query, params); } + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating node first seen date ${progress}/${nodes.length}`); + this.loggerTimer = new Date().getTime() / 1000; + ++updated; + } } - logger.info(`Node first seen dates scan complete.`); + logger.info(`Updated ${updated} node first seen dates`); } catch (e) { logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e)); } } - private async $lookUpCreationDateFromChain() { - logger.info(`Running channel creation date lookup...`); + private async $lookUpCreationDateFromChain(): Promise { + let progress = 0; + + logger.info(`Running channel creation date lookup`); try { const channels = await channelsApi.$getChannelsWithoutCreatedDate(); for (const channel of channels) { - const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1); - await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]); + const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id); + await DB.query(` + UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, + [transaction.timestamp, channel.id] + ); + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel creation date ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } } - logger.info(`Channel creation dates scan complete.`); + logger.info(`Updated ${channels.length} channels' creation date`); } catch (e) { - logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e)); + logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e)); } } - // Looking for channels whos nodes are inactive - private async $findInactiveNodesAndChannels(): Promise { - logger.info(`Running inactive channels scan...`); + /** + * If a channel does not have any active node linked to it, then also + * mark that channel as inactive + */ + private async $deactivateChannelsWithoutActiveNodes(): Promise { + logger.info(`Find channels which nodes are offline`); try { - // @ts-ignore - const [channels]: [ILightningApi.Channel[]] = await DB.query(` - SELECT channels.id - FROM channels + const result = await DB.query(` + UPDATE channels + SET status = 0 WHERE channels.status = 1 AND ( ( SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node1_public_key + AND nodes.status = 1 ) = 0 OR ( SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node2_public_key + AND nodes.status = 1 ) = 0) `); - for (const channel of channels) { - await this.$updateChannelStatus(channel.id, 0); + if (result[0].changedRows ?? 0 > 0) { + logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`); + } else { + logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`); } - logger.info(`Inactive channels scan complete.`); } catch (e) { - logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e)); + logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e)); } } private async $scanForClosedChannels(): Promise { + let progress = 0; + try { logger.info(`Starting closed channels scan...`); const channels = await channelsApi.$getChannelsByStatus(0); @@ -148,6 +236,13 @@ class NodeSyncService { await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]); } } + + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Checking if channel has been closed ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } } logger.info(`Closed channels scan complete.`); } catch (e) { @@ -165,6 +260,9 @@ class NodeSyncService { if (!config.ESPLORA.REST_API_URL) { return; } + + let progress = 0; + try { logger.info(`Started running closed channel forensics...`); const channels = await channelsApi.$getClosedChannelsWithoutReason(); @@ -210,6 +308,13 @@ class NodeSyncService { logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.'); await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]); } + + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } } logger.info(`Closed channels forensics scan complete.`); } catch (e) { @@ -264,164 +369,6 @@ class NodeSyncService { } return 1; } - - private async $saveChannel(channel: ILightningApi.Channel): Promise { - const fromChannel = chanNumber({ channel: channel.id }).number; - - try { - const query = `INSERT INTO channels - ( - id, - short_id, - capacity, - transaction_id, - transaction_vout, - updated_at, - status, - node1_public_key, - node1_base_fee_mtokens, - node1_cltv_delta, - node1_fee_rate, - node1_is_disabled, - node1_max_htlc_mtokens, - node1_min_htlc_mtokens, - node1_updated_at, - node2_public_key, - node2_base_fee_mtokens, - node2_cltv_delta, - node2_fee_rate, - node2_is_disabled, - node2_max_htlc_mtokens, - node2_min_htlc_mtokens, - node2_updated_at - ) - VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - capacity = ?, - updated_at = ?, - status = 1, - node1_public_key = ?, - node1_base_fee_mtokens = ?, - node1_cltv_delta = ?, - node1_fee_rate = ?, - node1_is_disabled = ?, - node1_max_htlc_mtokens = ?, - node1_min_htlc_mtokens = ?, - node1_updated_at = ?, - node2_public_key = ?, - node2_base_fee_mtokens = ?, - node2_cltv_delta = ?, - node2_fee_rate = ?, - node2_is_disabled = ?, - node2_max_htlc_mtokens = ?, - node2_min_htlc_mtokens = ?, - node2_updated_at = ? - ;`; - - await DB.query(query, [ - fromChannel, - channel.id, - channel.capacity, - channel.transaction_id, - channel.transaction_vout, - channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0, - channel.policies[0].public_key, - channel.policies[0].base_fee_mtokens, - channel.policies[0].cltv_delta, - channel.policies[0].fee_rate, - channel.policies[0].is_disabled, - channel.policies[0].max_htlc_mtokens, - channel.policies[0].min_htlc_mtokens, - channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0, - channel.policies[1].public_key, - channel.policies[1].base_fee_mtokens, - channel.policies[1].cltv_delta, - channel.policies[1].fee_rate, - channel.policies[1].is_disabled, - channel.policies[1].max_htlc_mtokens, - channel.policies[1].min_htlc_mtokens, - channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0, - channel.capacity, - channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0, - channel.policies[0].public_key, - channel.policies[0].base_fee_mtokens, - channel.policies[0].cltv_delta, - channel.policies[0].fee_rate, - channel.policies[0].is_disabled, - channel.policies[0].max_htlc_mtokens, - channel.policies[0].min_htlc_mtokens, - channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0, - channel.policies[1].public_key, - channel.policies[1].base_fee_mtokens, - channel.policies[1].cltv_delta, - channel.policies[1].fee_rate, - channel.policies[1].is_disabled, - channel.policies[1].max_htlc_mtokens, - channel.policies[1].min_htlc_mtokens, - channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0, - ]); - } catch (e) { - logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $updateChannelStatus(channelShortId: string, status: number): Promise { - try { - await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]); - } catch (e) { - logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $setChannelsInactive(graphChannelsIds: string[]): Promise { - try { - await DB.query(` - UPDATE channels - SET status = 0 - WHERE short_id NOT IN ( - ${graphChannelsIds.map(id => `"${id}"`).join(',')} - ) - AND status != 2 - `); - } catch (e) { - logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $saveNode(node: ILightningApi.Node): Promise { - try { - const updatedAt = node.updated_at ? this.utcDateToMysql(node.updated_at) : '0000-00-00 00:00:00'; - const sockets = node.sockets.join(','); - const query = `INSERT INTO nodes( - public_key, - first_seen, - updated_at, - alias, - color, - sockets - ) - VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`; - - await DB.query(query, [ - node.public_key, - updatedAt, - node.alias, - node.color, - sockets, - updatedAt, - node.alias, - node.color, - sockets, - ]); - } catch (e) { - logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private utcDateToMysql(dateString: string): string { - const d = new Date(Date.parse(dateString)); - return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; - } } -export default new NodeSyncService(); +export default new NetworkSyncService(); diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index f30da9e96..ecb056859 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -1,350 +1,33 @@ - -import DB from '../../database'; import logger from '../../logger'; import lightningApi from '../../api/lightning/lightning-api-factory'; -import channelsApi from '../../api/explorer/channels.api'; -import * as net from 'net'; +import LightningStatsImporter from './sync-tasks/stats-importer'; +import config from '../../config'; +import { Common } from '../../api/common'; class LightningStatsUpdater { - hardCodedStartTime = '2018-01-12'; - - public async $startService() { + public async $startService(): Promise { logger.info('Starting Lightning Stats service'); - let isInSync = false; - let error: any; - try { - error = null; - isInSync = await this.$lightningIsSynced(); - } catch (e) { - error = e; - } - if (!isInSync) { - if (error) { - logger.warn('Was not able to fetch Lightning Node status: ' + (error instanceof Error ? error.message : error) + '. Retrying in 1 minute...'); - } else { - logger.notice('The Lightning graph is not yet in sync. Retrying in 1 minute...'); - } - setTimeout(() => this.$startService(), 60 * 1000); - return; - } - await this.$populateHistoricalStatistics(); - await this.$populateHistoricalNodeStatistics(); - - setTimeout(() => { - this.$runTasks(); - }, this.timeUntilMidnight()); - } - - private timeUntilMidnight(): number { - const date = new Date(); - this.setDateMidnight(date); - date.setUTCHours(24); - return date.getTime() - new Date().getTime(); - } - - private setDateMidnight(date: Date): void { - date.setUTCHours(0); - date.setUTCMinutes(0); - date.setUTCSeconds(0); - date.setUTCMilliseconds(0); - } - - private async $lightningIsSynced(): Promise { - const nodeInfo = await lightningApi.$getInfo(); - return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph; + await this.$runTasks(); + LightningStatsImporter.$run(); } private async $runTasks(): Promise { - await this.$logLightningStatsDaily(); - await this.$logNodeStatsDaily(); + await this.$logStatsDaily(); - setTimeout(() => { - this.$runTasks(); - }, this.timeUntilMidnight()); + setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL); } - private async $logLightningStatsDaily() { - try { - logger.info(`Running lightning daily stats log...`); - - const networkGraph = await lightningApi.$getNetworkGraph(); - let total_capacity = 0; - for (const channel of networkGraph.channels) { - if (channel.capacity) { - total_capacity += channel.capacity; - } - } - - let clearnetNodes = 0; - let torNodes = 0; - let unannouncedNodes = 0; - for (const node of networkGraph.nodes) { - let isUnnanounced = true; - for (const socket of node.sockets) { - const hasOnion = socket.indexOf('.onion') !== -1; - if (hasOnion) { - torNodes++; - isUnnanounced = false; - } - const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0])); - if (hasClearnet) { - clearnetNodes++; - isUnnanounced = false; - } - } - if (isUnnanounced) { - unannouncedNodes++; - } - } - - const channelStats = await channelsApi.$getChannelsStats(); - - const query = `INSERT INTO lightning_stats( - added, - channel_count, - node_count, - total_capacity, - tor_nodes, - clearnet_nodes, - unannounced_nodes, - avg_capacity, - avg_fee_rate, - avg_base_fee_mtokens, - med_capacity, - med_fee_rate, - med_base_fee_mtokens - ) - VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; - - await DB.query(query, [ - networkGraph.channels.length, - networkGraph.nodes.length, - total_capacity, - torNodes, - clearnetNodes, - unannouncedNodes, - channelStats.avgCapacity, - channelStats.avgFeeRate, - channelStats.avgBaseFee, - channelStats.medianCapacity, - channelStats.medianFeeRate, - channelStats.medianBaseFee, - ]); - logger.info(`Lightning daily stats done.`); - } catch (e) { - logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $logNodeStatsDaily() { - try { - logger.info(`Running daily node stats update...`); - - const query = ` - SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left, - c2.channels_capacity_right - FROM nodes - LEFT JOIN ( - SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left - FROM channels - WHERE channels.status = 1 - GROUP BY node1_public_key - ) c1 ON c1.node1_public_key = nodes.public_key - LEFT JOIN ( - SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right - FROM channels WHERE channels.status = 1 GROUP BY node2_public_key - ) c2 ON c2.node2_public_key = nodes.public_key - `; - - const [nodes]: any = await DB.query(query); - - for (const node of nodes) { - await DB.query( - `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW() - INTERVAL 1 DAY, ?, ?)`, - [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)), - node.channels_count_left + node.channels_count_right]); - } - logger.info('Daily node stats has updated.'); - } catch (e) { - logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e)); - } - } - - // We only run this on first launch - private async $populateHistoricalStatistics() { - try { - const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`); - // Only run if table is empty - if (rows[0]['COUNT(*)'] > 0) { - return; - } - logger.info(`Running historical stats population...`); - - const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`); - const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`); - - const date: Date = new Date(this.hardCodedStartTime); - const currentDate = new Date(); - this.setDateMidnight(currentDate); - - while (date < currentDate) { - let totalCapacity = 0; - let channelsCount = 0; - - for (const channel of channels) { - if (new Date(channel.created) > date) { - break; - } - if (channel.closing_date === null || new Date(channel.closing_date) > date) { - totalCapacity += channel.capacity; - channelsCount++; - } - } - - let nodeCount = 0; - let clearnetNodes = 0; - let torNodes = 0; - let unannouncedNodes = 0; - - for (const node of nodes) { - if (new Date(node.first_seen) > date) { - break; - } - nodeCount++; - - const sockets = node.sockets.split(','); - let isUnnanounced = true; - for (const socket of sockets) { - const hasOnion = socket.indexOf('.onion') !== -1; - if (hasOnion) { - torNodes++; - isUnnanounced = false; - } - const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':')))); - if (hasClearnet) { - clearnetNodes++; - isUnnanounced = false; - } - } - if (isUnnanounced) { - unannouncedNodes++; - } - } - - const query = `INSERT INTO lightning_stats( - added, - channel_count, - node_count, - total_capacity, - tor_nodes, - clearnet_nodes, - unannounced_nodes, - avg_capacity, - avg_fee_rate, - avg_base_fee_mtokens, - med_capacity, - med_fee_rate, - med_base_fee_mtokens - ) - VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; - - const rowTimestamp = date.getTime() / 1000; // Save timestamp for the row insertion down below - - date.setUTCDate(date.getUTCDate() + 1); - - // Last iteration, save channels stats - const channelStats = (date >= currentDate ? await channelsApi.$getChannelsStats() : undefined); - - await DB.query(query, [ - rowTimestamp, - channelsCount, - nodeCount, - totalCapacity, - torNodes, - clearnetNodes, - unannouncedNodes, - channelStats?.avgCapacity ?? 0, - channelStats?.avgFeeRate ?? 0, - channelStats?.avgBaseFee ?? 0, - channelStats?.medianCapacity ?? 0, - channelStats?.medianFeeRate ?? 0, - channelStats?.medianBaseFee ?? 0, - ]); - } - - logger.info('Historical stats populated.'); - } catch (e) { - logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e)); - } - } - - private async $populateHistoricalNodeStatistics() { - try { - const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`); - // Only run if table is empty - if (rows[0]['COUNT(*)'] > 0) { - return; - } - logger.info(`Running historical node stats population...`); - - const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`); - - for (const node of nodes) { - const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); - - const date: Date = new Date(this.hardCodedStartTime); - const currentDate = new Date(); - this.setDateMidnight(currentDate); - - let lastTotalCapacity = 0; - let lastChannelsCount = 0; - - while (date < currentDate) { - let totalCapacity = 0; - let channelsCount = 0; - for (const channel of channels) { - if (new Date(channel.created) > date) { - break; - } - if (channel.closing_date !== null && new Date(channel.closing_date) < date) { - date.setUTCDate(date.getUTCDate() + 1); - continue; - } - totalCapacity += channel.capacity; - channelsCount++; - } - - if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) { - date.setUTCDate(date.getUTCDate() + 1); - continue; - } - - lastTotalCapacity = totalCapacity; - lastChannelsCount = channelsCount; - - const query = `INSERT INTO node_stats( - public_key, - added, - capacity, - channels - ) - VALUES (?, FROM_UNIXTIME(?), ?, ?)`; - - await DB.query(query, [ - node.public_key, - date.getTime() / 1000, - totalCapacity, - channelsCount, - ]); - date.setUTCDate(date.getUTCDate() + 1); - } - logger.debug('Updated node_stats for: ' + node.alias); - } - logger.info('Historical stats populated.'); - } catch (e) { - logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e)); - } + /** + * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds + */ + private async $logStatsDaily(): Promise { + const date = new Date(); + Common.setDateMidnight(date); + const networkGraph = await lightningApi.$getNetworkGraph(); + LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph); + + logger.info(`Updated latest network stats`); } } diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts new file mode 100644 index 000000000..9dbc21c72 --- /dev/null +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -0,0 +1,118 @@ +import { existsSync, promises } from 'fs'; +import bitcoinClient from '../../../api/bitcoin/bitcoin-client'; +import { Common } from '../../../api/common'; +import config from '../../../config'; +import logger from '../../../logger'; + +const fsPromises = promises; + +const BLOCKS_CACHE_MAX_SIZE = 100; +const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json'; + +class FundingTxFetcher { + private running = false; + private blocksCache = {}; + private channelNewlyProcessed = 0; + public fundingTxCache = {}; + + async $init(): Promise { + // Load funding tx disk cache + if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) { + try { + this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8')); + } catch (e) { + logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`); + this.fundingTxCache = {}; + } + logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`); + } + } + + async $fetchChannelsFundingTxs(channelIds: string[]): Promise { + if (this.running) { + return; + } + this.running = true; + + const globalTimer = new Date().getTime() / 1000; + let cacheTimer = new Date().getTime() / 1000; + let loggerTimer = new Date().getTime() / 1000; + let channelProcessed = 0; + this.channelNewlyProcessed = 0; + for (const channelId of channelIds) { + await this.$fetchChannelOpenTx(channelId); + ++channelProcessed; + + let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer); + if (elapsedSeconds > 10) { + elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer); + logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` + + `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` + + `elapsed: ${elapsedSeconds} seconds` + ); + loggerTimer = new Date().getTime() / 1000; + } + + elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer); + if (elapsedSeconds > 60) { + logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`); + fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)); + cacheTimer = new Date().getTime() / 1000; + } + } + + if (this.channelNewlyProcessed > 0) { + logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`); + logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`); + fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache)); + } + + this.running = false; + } + + public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> { + if (channelId.indexOf('x') === -1) { + channelId = Common.channelIntegerIdToShortId(channelId); + } + + if (this.fundingTxCache[channelId]) { + return this.fundingTxCache[channelId]; + } + + const parts = channelId.split('x'); + const blockHeight = parts[0]; + const txIdx = parts[1]; + const outputIdx = parts[2]; + + let block = this.blocksCache[blockHeight]; + // Fetch it from core + if (!block) { + const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10)); + block = await bitcoinClient.getBlock(blockHash, 1); + } + this.blocksCache[block.height] = block; + + const blocksCacheHashes = Object.keys(this.blocksCache).sort((a, b) => parseInt(b) - parseInt(a)).reverse(); + if (blocksCacheHashes.length > BLOCKS_CACHE_MAX_SIZE) { + for (let i = 0; i < 10; ++i) { + delete this.blocksCache[blocksCacheHashes[i]]; + } + } + + const txid = block.tx[txIdx]; + const rawTx = await bitcoinClient.getRawTransaction(txid); + const tx = await bitcoinClient.decodeRawTransaction(rawTx); + + this.fundingTxCache[channelId] = { + timestamp: block.time, + txid: txid, + value: tx.vout[outputIdx].value, + }; + + ++this.channelNewlyProcessed; + + return this.fundingTxCache[channelId]; + } +} + +export default new FundingTxFetcher; diff --git a/backend/src/tasks/lightning/sync-tasks/node-locations.ts b/backend/src/tasks/lightning/sync-tasks/node-locations.ts index 483131b26..30a6bfc2a 100644 --- a/backend/src/tasks/lightning/sync-tasks/node-locations.ts +++ b/backend/src/tasks/lightning/sync-tasks/node-locations.ts @@ -6,7 +6,10 @@ import DB from '../../../database'; import logger from '../../../logger'; export async function $lookupNodeLocation(): Promise { - logger.info(`Running node location updater using Maxmind...`); + let loggerTimer = new Date().getTime() / 1000; + let progress = 0; + + logger.info(`Running node location updater using Maxmind`); try { const nodes = await nodesApi.$getAllNodes(); const lookupCity = await maxmind.open(config.MAXMIND.GEOLITE2_CITY); @@ -18,21 +21,24 @@ export async function $lookupNodeLocation(): Promise { for (const socket of sockets) { const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', ''); const hasClearnet = [4, 6].includes(net.isIP(ip)); + if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') { const city = lookupCity.get(ip); const asn = lookupAsn.get(ip); const isp = lookupIsp.get(ip); if (city && (asn || isp)) { - const query = `UPDATE nodes SET - as_number = ?, - city_id = ?, - country_id = ?, - subdivision_id = ?, - longitude = ?, - latitude = ?, - accuracy_radius = ? - WHERE public_key = ?`; + const query = ` + UPDATE nodes SET + as_number = ?, + city_id = ?, + country_id = ?, + subdivision_id = ?, + longitude = ?, + latitude = ?, + accuracy_radius = ? + WHERE public_key = ? + `; const params = [ isp?.autonomous_system_number ?? asn?.autonomous_system_number, @@ -46,25 +52,25 @@ export async function $lookupNodeLocation(): Promise { ]; await DB.query(query, params); - // Store Continent - if (city.continent?.geoname_id) { - await DB.query( + // Store Continent + if (city.continent?.geoname_id) { + await DB.query( `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`, [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]); - } + } - // Store Country - if (city.country?.geoname_id) { - await DB.query( + // Store Country + if (city.country?.geoname_id) { + await DB.query( `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`, [city.country?.geoname_id, JSON.stringify(city.country?.names)]); - } + } // Store Country ISO code if (city.country?.iso_code) { await DB.query( - `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`, - [city.country?.geoname_id, city.country?.iso_code]); + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country_iso_code', ?)`, + [city.country?.geoname_id, city.country?.iso_code]); } // Store Division @@ -88,10 +94,17 @@ export async function $lookupNodeLocation(): Promise { [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]); } } + + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating node location data ${progress}/${nodes.length}`); + loggerTimer = new Date().getTime() / 1000; + } } } } - logger.info(`Node location data updated.`); + logger.info(`${progress} nodes location data updated`); } catch (e) { logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts new file mode 100644 index 000000000..8c823e2ef --- /dev/null +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -0,0 +1,411 @@ +import DB from '../../../database'; +import { promises } from 'fs'; +import { XMLParser } from 'fast-xml-parser'; +import logger from '../../../logger'; +import fundingTxFetcher from './funding-tx-fetcher'; +import config from '../../../config'; + +const fsPromises = promises; + +interface Node { + id: string; + timestamp: number; + features: string; + rgb_color: string; + alias: string; + addresses: unknown[]; + out_degree: number; + in_degree: number; +} + +interface Channel { + channel_id: string; + node1_pub: string; + node2_pub: string; + timestamp: number; + features: string; + fee_base_msat: number; + fee_rate_milli_msat: number; + htlc_minimim_msat: number; + cltv_expiry_delta: number; + htlc_maximum_msat: number; +} + +class LightningStatsImporter { + topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER; + parser = new XMLParser(); + + async $run(): Promise { + logger.info(`Importing historical lightning stats`); + + const [channels]: any[] = await DB.query('SELECT short_id from channels;'); + logger.info('Caching funding txs for currently existing channels'); + await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); + + await this.$importHistoricalLightningStats(); + } + + /** + * Generate LN network stats for one day + */ + public async computeNetworkStats(timestamp: number, networkGraph): Promise { + // Node counts and network shares + let clearnetNodes = 0; + let torNodes = 0; + let clearnetTorNodes = 0; + let unannouncedNodes = 0; + + for (const node of networkGraph.nodes) { + let hasOnion = false; + let hasClearnet = false; + let isUnnanounced = true; + + for (const socket of (node.addresses ?? [])) { + hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network); + hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network); + } + if (hasOnion && hasClearnet) { + clearnetTorNodes++; + isUnnanounced = false; + } else if (hasOnion) { + torNodes++; + isUnnanounced = false; + } else if (hasClearnet) { + clearnetNodes++; + isUnnanounced = false; + } + if (isUnnanounced) { + unannouncedNodes++; + } + } + + // Channels and node historical stats + const nodeStats = {}; + let capacity = 0; + let avgFeeRate = 0; + let avgBaseFee = 0; + const capacities: number[] = []; + const feeRates: number[] = []; + const baseFees: number[] = []; + const alreadyCountedChannels = {}; + + for (const channel of networkGraph.edges) { + let short_id = channel.channel_id; + if (short_id.indexOf('/') !== -1) { + short_id = short_id.slice(0, -2); + } + + const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id); + if (!tx) { + logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`); + continue; + } + + if (!nodeStats[channel.node1_pub]) { + nodeStats[channel.node1_pub] = { + capacity: 0, + channels: 0, + }; + } + if (!nodeStats[channel.node2_pub]) { + nodeStats[channel.node2_pub] = { + capacity: 0, + channels: 0, + }; + } + + if (!alreadyCountedChannels[short_id]) { + capacity += Math.round(tx.value * 100000000); + capacities.push(Math.round(tx.value * 100000000)); + alreadyCountedChannels[short_id] = true; + + nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.node1_pub].channels++; + nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000); + nodeStats[channel.node2_pub].channels++; + } + + if (channel.node1_policy !== undefined) { // Coming from the node + for (const policy of [channel.node1_policy, channel.node2_policy]) { + if (policy && policy.fee_rate_milli_msat < 5000) { + avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10); + feeRates.push(parseInt(policy.fee_rate_milli_msat, 10)); + } + if (policy && policy.fee_base_msat < 5000) { + avgBaseFee += parseInt(policy.fee_base_msat, 10); + baseFees.push(parseInt(policy.fee_base_msat, 10)); + } + } + } else { // Coming from the historical import + if (channel.fee_rate_milli_msat < 5000) { + avgFeeRate += parseInt(channel.fee_rate_milli_msat, 10); + feeRates.push(parseInt(channel.fee_rate_milli_msat), 10); + } + if (channel.fee_base_msat < 5000) { + avgBaseFee += parseInt(channel.fee_base_msat, 10); + baseFees.push(parseInt(channel.fee_base_msat), 10); + } + } + } + + avgFeeRate /= Math.max(networkGraph.edges.length, 1); + avgBaseFee /= Math.max(networkGraph.edges.length, 1); + const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; + const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; + const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; + const avgCapacity = Math.round(capacity / Math.max(capacities.length, 1)); + + let query = `INSERT INTO lightning_stats( + added, + channel_count, + node_count, + total_capacity, + tor_nodes, + clearnet_nodes, + unannounced_nodes, + clearnet_tor_nodes, + avg_capacity, + avg_fee_rate, + avg_base_fee_mtokens, + med_capacity, + med_fee_rate, + med_base_fee_mtokens + ) + VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + added = FROM_UNIXTIME(?), + channel_count = ?, + node_count = ?, + total_capacity = ?, + tor_nodes = ?, + clearnet_nodes = ?, + unannounced_nodes = ?, + clearnet_tor_nodes = ?, + avg_capacity = ?, + avg_fee_rate = ?, + avg_base_fee_mtokens = ?, + med_capacity = ?, + med_fee_rate = ?, + med_base_fee_mtokens = ? + `; + + await DB.query(query, [ + timestamp, + capacities.length, + networkGraph.nodes.length, + capacity, + torNodes, + clearnetNodes, + unannouncedNodes, + clearnetTorNodes, + avgCapacity, + avgFeeRate, + avgBaseFee, + medCapacity, + medFeeRate, + medBaseFee, + timestamp, + capacities.length, + networkGraph.nodes.length, + capacity, + torNodes, + clearnetNodes, + unannouncedNodes, + clearnetTorNodes, + avgCapacity, + avgFeeRate, + avgBaseFee, + medCapacity, + medFeeRate, + medBaseFee, + ]); + + for (const public_key of Object.keys(nodeStats)) { + query = `INSERT INTO node_stats( + public_key, + added, + capacity, + channels + ) + VALUES (?, FROM_UNIXTIME(?), ?, ?) + ON DUPLICATE KEY UPDATE + added = FROM_UNIXTIME(?), + capacity = ?, + channels = ? + `; + + await DB.query(query, [ + public_key, + timestamp, + nodeStats[public_key].capacity, + nodeStats[public_key].channels, + timestamp, + nodeStats[public_key].capacity, + nodeStats[public_key].channels, + ]); + } + + return { + added: timestamp, + node_count: networkGraph.nodes.length + }; + } + + /** + * Import topology files LN historical data into the database + */ + async $importHistoricalLightningStats(): Promise { + let latestNodeCount = 1; + + const fileList = await fsPromises.readdir(this.topologiesFolder); + // Insert history from the most recent to the oldest + // This also put the .json cached files first + fileList.sort().reverse(); + + const [rows]: any[] = await DB.query(` + SELECT UNIX_TIMESTAMP(added) AS added, node_count + FROM lightning_stats + ORDER BY added DESC + `); + const existingStatsTimestamps = {}; + for (const row of rows) { + existingStatsTimestamps[row.added] = row; + } + + // For logging purpose + let processed = 10; + let totalProcessed = -1; + + for (const filename of fileList) { + processed++; + totalProcessed++; + + const timestamp = parseInt(filename.split('_')[1], 10); + + // Stats exist already, don't calculate/insert them + if (existingStatsTimestamps[timestamp] !== undefined) { + latestNodeCount = existingStatsTimestamps[timestamp].node_count; + continue; + } + + logger.debug(`Reading ${this.topologiesFolder}/${filename}`); + const fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8'); + + let graph; + if (filename.indexOf('.json') !== -1) { + try { + graph = JSON.parse(fileContent); + } catch (e) { + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); + continue; + } + } else { + graph = this.parseFile(fileContent); + if (!graph) { + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); + continue; + } + await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph)); + } + + if (timestamp > 1556316000) { + // "No, the reason most likely is just that I started collection in 2019, + // so what I had before that is just the survivors from before, which weren't that many" + const diffRatio = graph.nodes.length / latestNodeCount; + if (diffRatio < 0.9) { + // Ignore drop of more than 90% of the node count as it's probably a missing data point + logger.debug(`Nodes count diff ratio threshold reached, ignore the data for this day ${graph.nodes.length} nodes vs ${latestNodeCount}`); + continue; + } + } + latestNodeCount = graph.nodes.length; + + const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; + logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); + + if (processed > 10) { + logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); + processed = 0; + } else { + logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); + } + await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2))); + const stat = await this.computeNetworkStats(timestamp, graph); + + existingStatsTimestamps[timestamp] = stat; + } + + logger.info(`Lightning network stats historical import completed`); + } + + /** + * Parse the file content into XML, and return a list of nodes and channels + */ + private parseFile(fileContent): any { + const graph = this.parser.parse(fileContent); + if (Object.keys(graph).length === 0) { + return null; + } + + const nodes: Node[] = []; + const channels: Channel[] = []; + + // If there is only one entry, the parser does not return an array, so we override this + if (!Array.isArray(graph.graphml.graph.node)) { + graph.graphml.graph.node = [graph.graphml.graph.node]; + } + if (!Array.isArray(graph.graphml.graph.edge)) { + graph.graphml.graph.edge = [graph.graphml.graph.edge]; + } + + for (const node of graph.graphml.graph.node) { + if (!node.data) { + continue; + } + const addresses: unknown[] = []; + const sockets = node.data[5].split(','); + for (const socket of sockets) { + const parts = socket.split('://'); + addresses.push({ + network: parts[0], + addr: parts[1], + }); + } + nodes.push({ + id: node.data[0], + timestamp: node.data[1], + features: node.data[2], + rgb_color: node.data[3], + alias: node.data[4], + addresses: addresses, + out_degree: node.data[6], + in_degree: node.data[7], + }); + } + + for (const channel of graph.graphml.graph.edge) { + if (!channel.data) { + continue; + } + channels.push({ + channel_id: channel.data[0], + node1_pub: channel.data[1], + node2_pub: channel.data[2], + timestamp: channel.data[3], + features: channel.data[4], + fee_base_msat: channel.data[5], + fee_rate_milli_msat: channel.data[6], + htlc_minimim_msat: channel.data[7], + cltv_expiry_delta: channel.data[8], + htlc_maximum_msat: channel.data[9], + }); + } + + return { + nodes: nodes, + edges: channels, + }; + } +} + +export default new LightningStatsImporter; diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index a5901d7f7..81066efb2 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import { Common } from '../api/common'; import config from '../config'; import logger from '../logger'; import PricesRepository from '../repositories/PricesRepository'; @@ -34,10 +35,10 @@ export interface Prices { } class PriceUpdater { - historyInserted: boolean = false; - lastRun: number = 0; - lastHistoricalRun: number = 0; - running: boolean = false; + public historyInserted = false; + lastRun = 0; + lastHistoricalRun = 0; + running = false; feeds: PriceFeed[] = []; currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; latestPrices: Prices; diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index f152cb7b3..39acf122d 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -1,7 +1,7 @@