From 0f97f8f18b8c80deea411ff515fa42f9363d571c Mon Sep 17 00:00:00 2001 From: calle <93376500+callebtc@users.noreply.github.com> Date: Wed, 16 Mar 2022 07:20:15 +0100 Subject: [PATCH] Fee reserve for lightning backends (#557) * preparing fees * fee_limit_msat * await resp result * clightning * fix tests * fix test * add fee to test * mypy * invoice_status * checking id fix * fee reserve error message * only for external payments --- .github/workflows/mypy.yml | 1 + lnbits/core/services.py | 47 ++++++++-------- lnbits/wallets/base.py | 4 +- lnbits/wallets/clightning.py | 17 ++++-- lnbits/wallets/fake.py | 31 +++++++---- lnbits/wallets/lnbits.py | 2 +- lnbits/wallets/lndgrpc.py | 15 +++--- lnbits/wallets/lndrest.py | 15 +++--- lnbits/wallets/lnpay.py | 2 +- lnbits/wallets/lntxbot.py | 2 +- lnbits/wallets/opennode.py | 2 +- lnbits/wallets/spark.py | 2 +- lnbits/wallets/void.py | 2 +- tests/extensions/bleskomat/test_lnurl_api.py | 15 ++++-- tests/mocks.py | 56 ++++++++++++-------- 15 files changed, 129 insertions(+), 84 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 77d340c19..bf90a8e31 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -9,4 +9,5 @@ jobs: - uses: actions/checkout@v1 - uses: jpetrucciani/mypy-check@master with: + mypy_flags: '--install-types --non-interactive' path: lnbits diff --git a/lnbits/core/services.py b/lnbits/core/services.py index be21a84ef..875f7644c 100644 --- a/lnbits/core/services.py +++ b/lnbits/core/services.py @@ -85,18 +85,17 @@ async def pay_invoice( description: str = "", conn: Optional[Connection] = None, ) -> str: + invoice = bolt11.decode(payment_request) + fee_reserve_msat = fee_reserve(invoice.amount_msat) async with (db.reuse_conn(conn) if conn else db.connect()) as conn: temp_id = f"temp_{urlsafe_short_hash()}" internal_id = f"internal_{urlsafe_short_hash()}" - invoice = bolt11.decode(payment_request) if invoice.amount_msat == 0: raise ValueError("Amountless invoices not supported.") if max_sat and invoice.amount_msat > max_sat * 1000: raise ValueError("Amount in invoice is too high.") - wallet = await get_wallet(wallet_id, conn=conn) - # put all parameters that don't change here PaymentKwargs = TypedDict( "PaymentKwargs", @@ -134,26 +133,20 @@ async def pay_invoice( # the balance is enough in the next step await create_payment( checking_id=temp_id, - fee=-fee_reserve(invoice.amount_msat), + fee=-fee_reserve_msat, conn=conn, **payment_kwargs, ) - # do the balance check if internal payment - if internal_checking_id: - wallet = await get_wallet(wallet_id, conn=conn) - assert wallet - if wallet.balance_msat < 0: - raise PermissionError("Insufficient balance.") - - # do the balance check if external payment - else: - if invoice.amount_msat > wallet.balance_msat - ( - wallet.balance_msat / 100 * 2 - ): - raise PermissionError( - "LNbits requires you keep at least 2% reserve to cover potential routing fees." + # do the balance check + wallet = await get_wallet(wallet_id, conn=conn) + assert wallet + if wallet.balance_msat < 0: + if not internal_checking_id and wallet.balance_msat > -fee_reserve_msat: + raise PaymentFailure( + f"You must reserve at least 1% ({round(fee_reserve_msat/1000)} sat) to cover potential routing fees." ) + raise PermissionError("Insufficient balance.") if internal_checking_id: # mark the invoice from the other side as not pending anymore @@ -171,7 +164,9 @@ async def pay_invoice( await internal_invoice_queue.put(internal_checking_id) else: # actually pay the external invoice - payment: PaymentResponse = await WALLET.pay_invoice(payment_request) + payment: PaymentResponse = await WALLET.pay_invoice( + payment_request, fee_reserve_msat + ) if payment.checking_id: async with db.connect() as conn: await create_payment( @@ -286,12 +281,12 @@ async def perform_lnurlauth( sign_len = 6 + r_len + s_len signature = BytesIO() - signature.write(0x30 .to_bytes(1, "big", signed=False)) + signature.write(0x30.to_bytes(1, "big", signed=False)) signature.write((sign_len - 2).to_bytes(1, "big", signed=False)) - signature.write(0x02 .to_bytes(1, "big", signed=False)) + signature.write(0x02.to_bytes(1, "big", signed=False)) signature.write(r_len.to_bytes(1, "big", signed=False)) signature.write(r) - signature.write(0x02 .to_bytes(1, "big", signed=False)) + signature.write(0x02.to_bytes(1, "big", signed=False)) signature.write(s_len.to_bytes(1, "big", signed=False)) signature.write(s) @@ -326,7 +321,10 @@ async def check_invoice_status( payment = await get_wallet_payment(wallet_id, payment_hash, conn=conn) if not payment: return PaymentStatus(None) - status = await WALLET.get_invoice_status(payment.checking_id) + if payment.is_out: + status = await WALLET.get_payment_status(payment.checking_id) + else: + status = await WALLET.get_invoice_status(payment.checking_id) if not payment.pending: return status if payment.is_out and status.failed: @@ -340,5 +338,6 @@ async def check_invoice_status( return status +# WARN: this same value must be used for balance check and passed to WALLET.pay_invoice(), it may cause a vulnerability if the values differ def fee_reserve(amount_msat: int) -> int: - return max(1000, int(amount_msat * 0.01)) + return max(2000, int(amount_msat * 0.01)) diff --git a/lnbits/wallets/base.py b/lnbits/wallets/base.py index 973c18085..39c687594 100644 --- a/lnbits/wallets/base.py +++ b/lnbits/wallets/base.py @@ -60,7 +60,9 @@ class Wallet(ABC): pass @abstractmethod - def pay_invoice(self, bolt11: str) -> Coroutine[None, None, PaymentResponse]: + def pay_invoice( + self, bolt11: str, fee_limit_msat: int + ) -> Coroutine[None, None, PaymentResponse]: pass @abstractmethod diff --git a/lnbits/wallets/clightning.py b/lnbits/wallets/clightning.py index 639d0b384..f8c2b16cf 100644 --- a/lnbits/wallets/clightning.py +++ b/lnbits/wallets/clightning.py @@ -18,6 +18,7 @@ from .base import ( Unsupported, Wallet, ) +from lnbits import bolt11 as lnbits_bolt11 def async_wrap(func): @@ -31,8 +32,8 @@ def async_wrap(func): return run -def _pay_invoice(ln, bolt11): - return ln.pay(bolt11) +def _pay_invoice(ln, payload): + return ln.call("pay", payload) def _paid_invoices_stream(ln, last_pay_index): @@ -102,10 +103,18 @@ class CLightningWallet(Wallet): error_message = f"lightningd '{exc.method}' failed with '{exc.error}'." return InvoiceResponse(False, label, None, error_message) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: + invoice = lnbits_bolt11.decode(bolt11) + fee_limit_percent = fee_limit_msat / invoice.amount_msat * 100 + + payload = { + "bolt11": bolt11, + "maxfeepercent": "{:.11}".format(fee_limit_percent), + "exemptfee": 0, # so fee_limit_percent is applied even on payments with fee under 5000 millisatoshi (which is default value of exemptfee) + } try: wrapped = async_wrap(_pay_invoice) - r = await wrapped(self.ln, bolt11) + r = await wrapped(self.ln, payload) except RpcError as exc: return PaymentResponse(False, None, 0, None, str(exc)) diff --git a/lnbits/wallets/fake.py b/lnbits/wallets/fake.py index 93e50abd9..331d5285b 100644 --- a/lnbits/wallets/fake.py +++ b/lnbits/wallets/fake.py @@ -36,7 +36,13 @@ class FakeWallet(Wallet): "out": False, "amount": amount, "currency": "bc", - "privkey": hashlib.pbkdf2_hmac('sha256', secret.encode("utf-8"), ("FakeWallet").encode("utf-8"), 2048, 32).hex(), + "privkey": hashlib.pbkdf2_hmac( + "sha256", + secret.encode("utf-8"), + ("FakeWallet").encode("utf-8"), + 2048, + 32, + ).hex(), "memo": None, "description_hash": None, "description": "", @@ -53,22 +59,29 @@ class FakeWallet(Wallet): data["tags_set"] = ["d"] data["memo"] = memo data["description"] = memo - randomHash = data["privkey"][:6] + hashlib.sha256( - str(random.getrandbits(256)).encode("utf-8") - ).hexdigest()[6:] + randomHash = ( + data["privkey"][:6] + + hashlib.sha256(str(random.getrandbits(256)).encode("utf-8")).hexdigest()[ + 6: + ] + ) data["paymenthash"] = randomHash payment_request = encode(data) checking_id = randomHash return InvoiceResponse(True, checking_id, payment_request) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: - invoice = decode(bolt11) - if hasattr(invoice, 'checking_id') and invoice.checking_id[6:] == data["privkey"][:6]: + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: + invoice = decode(bolt11) + if ( + hasattr(invoice, "checking_id") + and invoice.checking_id[6:] == data["privkey"][:6] + ): return PaymentResponse(True, invoice.payment_hash, 0) else: - return PaymentResponse(ok = False, error_message="Only internal invoices can be used!") - + return PaymentResponse( + ok=False, error_message="Only internal invoices can be used!" + ) async def get_invoice_status(self, checking_id: str) -> PaymentStatus: return PaymentStatus(False) diff --git a/lnbits/wallets/lnbits.py b/lnbits/wallets/lnbits.py index 5e34a2d77..414e987b9 100644 --- a/lnbits/wallets/lnbits.py +++ b/lnbits/wallets/lnbits.py @@ -80,7 +80,7 @@ class LNbitsWallet(Wallet): return InvoiceResponse(ok, checking_id, payment_request, error_message) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: async with httpx.AsyncClient() as client: r = await client.post( url=f"{self.endpoint}/api/v1/payments", diff --git a/lnbits/wallets/lndgrpc.py b/lnbits/wallets/lndgrpc.py index 85c6dd098..f9a0496a3 100644 --- a/lnbits/wallets/lndgrpc.py +++ b/lnbits/wallets/lndgrpc.py @@ -92,11 +92,12 @@ class LndWallet(Wallet): or getenv("LND_GRPC_INVOICE_MACAROON") or getenv("LND_INVOICE_MACAROON") ) - - + encrypted_macaroon = getenv("LND_GRPC_MACAROON_ENCRYPTED") if encrypted_macaroon: - macaroon = AESCipher(description="macaroon decryption").decrypt(encrypted_macaroon) + macaroon = AESCipher(description="macaroon decryption").decrypt( + encrypted_macaroon + ) self.macaroon = load_macaroon(macaroon) cert = open(self.cert_path, "rb").read() @@ -143,10 +144,10 @@ class LndWallet(Wallet): payment_request = str(resp.payment_request) return InvoiceResponse(True, checking_id, payment_request, None) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: - resp = await self.rpc.SendPayment( - lnrpc.SendPaymentRequest(payment_request=bolt11) - ) + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: + fee_limit_fixed = ln.FeeLimit(fixed=fee_limit_msat // 1000) + req = ln.SendRequest(payment_request=bolt11, fee_limit=fee_limit_fixed) + resp = await self.rpc.SendPaymentSync(req) if resp.payment_error: return PaymentResponse(False, "", 0, None, resp.payment_error) diff --git a/lnbits/wallets/lndrest.py b/lnbits/wallets/lndrest.py index aa9b7b0fc..a107f7493 100644 --- a/lnbits/wallets/lndrest.py +++ b/lnbits/wallets/lndrest.py @@ -39,9 +39,11 @@ class LndRestWallet(Wallet): encrypted_macaroon = getenv("LND_REST_MACAROON_ENCRYPTED") if encrypted_macaroon: - macaroon = AESCipher(description="macaroon decryption").decrypt(encrypted_macaroon) + macaroon = AESCipher(description="macaroon decryption").decrypt( + encrypted_macaroon + ) self.macaroon = load_macaroon(macaroon) - + self.auth = {"Grpc-Metadata-macaroon": self.macaroon} self.cert = getenv("LND_REST_CERT", True) @@ -97,15 +99,11 @@ class LndRestWallet(Wallet): return InvoiceResponse(True, checking_id, payment_request, None) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: async with httpx.AsyncClient(verify=self.cert) as client: # set the fee limit for the payment - invoice = lnbits_bolt11.decode(bolt11) lnrpcFeeLimit = dict() - if invoice.amount_msat > 1000_000: - lnrpcFeeLimit["percent"] = "1" # in percent - else: - lnrpcFeeLimit["fixed"] = "10" # in sat + lnrpcFeeLimit["fixed_msat"] = "{}".format(fee_limit_msat) r = await client.post( url=f"{self.endpoint}/v1/channels/transactions", @@ -162,6 +160,7 @@ class LndRestWallet(Wallet): # for some reason our checking_ids are in base64 but the payment hashes # returned here are in hex, lnd is weird + checking_id = checking_id.replace("_", "/") checking_id = base64.b64decode(checking_id).hex() for p in r.json()["payments"]: diff --git a/lnbits/wallets/lnpay.py b/lnbits/wallets/lnpay.py index 98610a799..88e6a3a2c 100644 --- a/lnbits/wallets/lnpay.py +++ b/lnbits/wallets/lnpay.py @@ -76,7 +76,7 @@ class LNPayWallet(Wallet): return InvoiceResponse(ok, checking_id, payment_request, error_message) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: async with httpx.AsyncClient() as client: r = await client.post( f"{self.endpoint}/wallet/{self.wallet_key}/withdraw", diff --git a/lnbits/wallets/lntxbot.py b/lnbits/wallets/lntxbot.py index 6d706959c..bbd87a73d 100644 --- a/lnbits/wallets/lntxbot.py +++ b/lnbits/wallets/lntxbot.py @@ -74,7 +74,7 @@ class LntxbotWallet(Wallet): data = r.json() return InvoiceResponse(True, data["payment_hash"], data["pay_req"], None) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: async with httpx.AsyncClient() as client: r = await client.post( f"{self.endpoint}/payinvoice", diff --git a/lnbits/wallets/opennode.py b/lnbits/wallets/opennode.py index 965b6d661..0ac205e25 100644 --- a/lnbits/wallets/opennode.py +++ b/lnbits/wallets/opennode.py @@ -77,7 +77,7 @@ class OpenNodeWallet(Wallet): payment_request = data["lightning_invoice"]["payreq"] return InvoiceResponse(True, checking_id, payment_request, None) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: async with httpx.AsyncClient() as client: r = await client.post( f"{self.endpoint}/v2/withdrawals", diff --git a/lnbits/wallets/spark.py b/lnbits/wallets/spark.py index 93f37cceb..9404de6fb 100644 --- a/lnbits/wallets/spark.py +++ b/lnbits/wallets/spark.py @@ -107,7 +107,7 @@ class SparkWallet(Wallet): return InvoiceResponse(ok, checking_id, payment_request, error_message) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: try: r = await self.pay(bolt11) except (SparkError, UnknownError) as exc: diff --git a/lnbits/wallets/void.py b/lnbits/wallets/void.py index 03b6db447..c5cc08b55 100644 --- a/lnbits/wallets/void.py +++ b/lnbits/wallets/void.py @@ -25,7 +25,7 @@ class VoidWallet(Wallet): ) return StatusResponse(None, 0) - async def pay_invoice(self, bolt11: str) -> PaymentResponse: + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: raise Unsupported("") async def get_invoice_status(self, checking_id: str) -> PaymentStatus: diff --git a/tests/extensions/bleskomat/test_lnurl_api.py b/tests/extensions/bleskomat/test_lnurl_api.py index 2ee59117e..003584705 100644 --- a/tests/extensions/bleskomat/test_lnurl_api.py +++ b/tests/extensions/bleskomat/test_lnurl_api.py @@ -3,7 +3,10 @@ import secrets from lnbits.core.crud import get_wallet from lnbits.settings import HOST, PORT from lnbits.extensions.bleskomat.crud import get_bleskomat_lnurl -from lnbits.extensions.bleskomat.helpers import generate_bleskomat_lnurl_signature, query_to_signing_payload +from lnbits.extensions.bleskomat.helpers import ( + generate_bleskomat_lnurl_signature, + query_to_signing_payload, +) from tests.conftest import client from tests.helpers import credit_wallet from tests.extensions.bleskomat.conftest import bleskomat, lnurl @@ -73,7 +76,9 @@ async def test_bleskomat_lnurl_api_valid_signature(client, bleskomat): } payload = query_to_signing_payload(query) signature = generate_bleskomat_lnurl_signature( - payload=payload, api_key_secret=bleskomat.api_key_secret, api_key_encoding=bleskomat.api_key_encoding + payload=payload, + api_key_secret=bleskomat.api_key_secret, + api_key_encoding=bleskomat.api_key_encoding, ) response = await client.get(f"/bleskomat/u?{payload}&signature={signature}") assert response.status_code == 200 @@ -97,7 +102,9 @@ async def test_bleskomat_lnurl_api_action_insufficient_balance(client, lnurl): response = await client.get(f"/bleskomat/u?k1={secret}&pr={pr}") assert response.status_code == 200 assert response.json()["status"] == "ERROR" - assert ("Insufficient balance" in response.json()["reason"]) or ("fee" in response.json()["reason"]) + assert ("Insufficient balance" in response.json()["reason"]) or ( + "fee" in response.json()["reason"] + ) wallet = await get_wallet(bleskomat.wallet) assert wallet.balance_msat == 0 bleskomat_lnurl = await get_bleskomat_lnurl(secret) @@ -123,4 +130,4 @@ async def test_bleskomat_lnurl_api_action_success(client, lnurl): assert wallet.balance_msat == 50000 bleskomat_lnurl = await get_bleskomat_lnurl(secret) assert bleskomat_lnurl.has_uses_remaining() == False - WALLET.pay_invoice.assert_called_once_with(pr) + WALLET.pay_invoice.assert_called_once_with(pr, 2000) diff --git a/tests/mocks.py b/tests/mocks.py index 5b20824cd..a3b5308d5 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -9,28 +9,42 @@ from lnbits.wallets.base import ( ) from lnbits.settings import WALLET -WALLET.status = AsyncMock(return_value=StatusResponse( - "",# no error - 1000000,# msats -)) -WALLET.create_invoice = AsyncMock(return_value=InvoiceResponse( - True,# ok - "6621aafbdd7709ca6eea6203f362d64bd7cb2911baa91311a176b3ecaf2274bd",# checking_id (i.e. payment_hash) - "lntb1u1psezhgspp5vcs6477awuyu5mh2vgplxckkf0tuk2g3h253xydpw6e7etezwj7sdqqcqzpgxqyz5vqsp5dxpw8zs77hw5pla4wz4mfujllyxtlpu443auur2uxqdrs8q2h56q9qyyssq65zk30ylmygvv5y4tuwalnf3ttnqjn57ef6rmcqg0s53akem560jh8ptemjcmytn3lrlatw4hv9smg88exv3v4f4lqnp96s0psdrhxsp6pp75q",# payment_request - "",# no error -)) -def pay_invoice_side_effect(payment_request: str): +WALLET.status = AsyncMock( + return_value=StatusResponse( + "", # no error + 1000000, # msats + ) +) +WALLET.create_invoice = AsyncMock( + return_value=InvoiceResponse( + True, # ok + "6621aafbdd7709ca6eea6203f362d64bd7cb2911baa91311a176b3ecaf2274bd", # checking_id (i.e. payment_hash) + "lntb1u1psezhgspp5vcs6477awuyu5mh2vgplxckkf0tuk2g3h253xydpw6e7etezwj7sdqqcqzpgxqyz5vqsp5dxpw8zs77hw5pla4wz4mfujllyxtlpu443auur2uxqdrs8q2h56q9qyyssq65zk30ylmygvv5y4tuwalnf3ttnqjn57ef6rmcqg0s53akem560jh8ptemjcmytn3lrlatw4hv9smg88exv3v4f4lqnp96s0psdrhxsp6pp75q", # payment_request + "", # no error + ) +) + + +def pay_invoice_side_effect( + payment_request: str, fee_limit_msat: int +) -> PaymentResponse: invoice = bolt11.decode(payment_request) return PaymentResponse( - True,# ok - invoice.payment_hash,# checking_id (i.e. payment_hash) - 0,# fee_msat - "",# no error + True, # ok + invoice.payment_hash, # checking_id (i.e. payment_hash) + 0, # fee_msat + "", # no error ) + + WALLET.pay_invoice = AsyncMock(side_effect=pay_invoice_side_effect) -WALLET.get_invoice_status = AsyncMock(return_value=PaymentStatus( - True,# paid -)) -WALLET.get_payment_status = AsyncMock(return_value=PaymentStatus( - True,# paid -)) +WALLET.get_invoice_status = AsyncMock( + return_value=PaymentStatus( + True, # paid + ) +) +WALLET.get_payment_status = AsyncMock( + return_value=PaymentStatus( + True, # paid + ) +)