Fee reserve for lightning backends (#557)

* preparing fees

* fee_limit_msat

* await resp result

* clightning

* fix tests

* fix test

* add fee to test

* mypy

* invoice_status

* checking id fix

* fee reserve error message

* only for external payments

.github/workflows/mypy.yml

@@ -9,4 +9,5 @@ jobs:
       - uses: actions/checkout@v1
       - uses: jpetrucciani/mypy-check@master
         with:
+          mypy_flags: '--install-types --non-interactive'
           path: lnbits

lnbits/core/services.py

@@ -85,18 +85,17 @@ async def pay_invoice(
     description: str = "",
     conn: Optional[Connection] = None,
 ) -> str:
+    invoice = bolt11.decode(payment_request)
+    fee_reserve_msat = fee_reserve(invoice.amount_msat)
     async with (db.reuse_conn(conn) if conn else db.connect()) as conn:
         temp_id = f"temp_{urlsafe_short_hash()}"
         internal_id = f"internal_{urlsafe_short_hash()}"
 
-        invoice = bolt11.decode(payment_request)
         if invoice.amount_msat == 0:
             raise ValueError("Amountless invoices not supported.")
         if max_sat and invoice.amount_msat > max_sat * 1000:
             raise ValueError("Amount in invoice is too high.")
 
-        wallet = await get_wallet(wallet_id, conn=conn)
-
         # put all parameters that don't change here
         PaymentKwargs = TypedDict(
             "PaymentKwargs",
@@ -134,26 +133,20 @@ async def pay_invoice(
             # the balance is enough in the next step
             await create_payment(
                 checking_id=temp_id,
-                fee=-fee_reserve(invoice.amount_msat),
+                fee=-fee_reserve_msat,
                 conn=conn,
                 **payment_kwargs,
             )
 
-        # do the balance check if internal payment
+        # do the balance check
-        if internal_checking_id:
+        wallet = await get_wallet(wallet_id, conn=conn)
-            wallet = await get_wallet(wallet_id, conn=conn)
+        assert wallet
-            assert wallet
+        if wallet.balance_msat < 0:
-            if wallet.balance_msat < 0:
+            if not internal_checking_id and wallet.balance_msat > -fee_reserve_msat:
-                raise PermissionError("Insufficient balance.")
+                raise PaymentFailure(
-
+                    f"You must reserve at least 1% ({round(fee_reserve_msat/1000)} sat) to cover potential routing fees."
-        # do the balance check if external payment
-        else:
-            if invoice.amount_msat > wallet.balance_msat - (
-                wallet.balance_msat / 100 * 2
-            ):
-                raise PermissionError(
-                    "LNbits requires you keep at least 2% reserve to cover potential routing fees."
                 )
+            raise PermissionError("Insufficient balance.")
 
     if internal_checking_id:
         # mark the invoice from the other side as not pending anymore
@@ -171,7 +164,9 @@ async def pay_invoice(
         await internal_invoice_queue.put(internal_checking_id)
     else:
         # actually pay the external invoice
-        payment: PaymentResponse = await WALLET.pay_invoice(payment_request)
+        payment: PaymentResponse = await WALLET.pay_invoice(
+            payment_request, fee_reserve_msat
+        )
         if payment.checking_id:
             async with db.connect() as conn:
                 await create_payment(
@@ -286,12 +281,12 @@ async def perform_lnurlauth(
         sign_len = 6 + r_len + s_len
 
         signature = BytesIO()
-        signature.write(0x30 .to_bytes(1, "big", signed=False))
+        signature.write(0x30.to_bytes(1, "big", signed=False))
         signature.write((sign_len - 2).to_bytes(1, "big", signed=False))
-        signature.write(0x02 .to_bytes(1, "big", signed=False))
+        signature.write(0x02.to_bytes(1, "big", signed=False))
         signature.write(r_len.to_bytes(1, "big", signed=False))
         signature.write(r)
-        signature.write(0x02 .to_bytes(1, "big", signed=False))
+        signature.write(0x02.to_bytes(1, "big", signed=False))
         signature.write(s_len.to_bytes(1, "big", signed=False))
         signature.write(s)
 
@@ -326,7 +321,10 @@ async def check_invoice_status(
     payment = await get_wallet_payment(wallet_id, payment_hash, conn=conn)
     if not payment:
         return PaymentStatus(None)
-    status = await WALLET.get_invoice_status(payment.checking_id)
+    if payment.is_out:
+        status = await WALLET.get_payment_status(payment.checking_id)
+    else:
+        status = await WALLET.get_invoice_status(payment.checking_id)
     if not payment.pending:
         return status
     if payment.is_out and status.failed:
@@ -340,5 +338,6 @@ async def check_invoice_status(
     return status
 
 
+# WARN: this same value must be used for balance check and passed to WALLET.pay_invoice(), it may cause a vulnerability if the values differ
 def fee_reserve(amount_msat: int) -> int:
-    return max(1000, int(amount_msat * 0.01))
+    return max(2000, int(amount_msat * 0.01))

lnbits/wallets/base.py

@@ -60,7 +60,9 @@ class Wallet(ABC):
         pass
 
     @abstractmethod
-    def pay_invoice(self, bolt11: str) -> Coroutine[None, None, PaymentResponse]:
+    def pay_invoice(
+        self, bolt11: str, fee_limit_msat: int
+    ) -> Coroutine[None, None, PaymentResponse]:
         pass
 
     @abstractmethod

lnbits/wallets/clightning.py

@@ -18,6 +18,7 @@ from .base import (
     Unsupported,
     Wallet,
 )
+from lnbits import bolt11 as lnbits_bolt11
 
 
 def async_wrap(func):
@@ -31,8 +32,8 @@ def async_wrap(func):
     return run
 
 
-def _pay_invoice(ln, bolt11):
+def _pay_invoice(ln, payload):
-    return ln.pay(bolt11)
+    return ln.call("pay", payload)
 
 
 def _paid_invoices_stream(ln, last_pay_index):
@@ -102,10 +103,18 @@ class CLightningWallet(Wallet):
             error_message = f"lightningd '{exc.method}' failed with '{exc.error}'."
             return InvoiceResponse(False, label, None, error_message)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
+        invoice = lnbits_bolt11.decode(bolt11)
+        fee_limit_percent = fee_limit_msat / invoice.amount_msat * 100
+
+        payload = {
+            "bolt11": bolt11,
+            "maxfeepercent": "{:.11}".format(fee_limit_percent),
+            "exemptfee": 0,  # so fee_limit_percent is applied even on payments with fee under 5000 millisatoshi (which is default value of exemptfee)
+        }
         try:
             wrapped = async_wrap(_pay_invoice)
-            r = await wrapped(self.ln, bolt11)
+            r = await wrapped(self.ln, payload)
         except RpcError as exc:
             return PaymentResponse(False, None, 0, None, str(exc))
 

lnbits/wallets/fake.py

@@ -36,7 +36,13 @@ class FakeWallet(Wallet):
             "out": False,
             "amount": amount,
             "currency": "bc",
-            "privkey": hashlib.pbkdf2_hmac('sha256', secret.encode("utf-8"), ("FakeWallet").encode("utf-8"), 2048, 32).hex(),
+            "privkey": hashlib.pbkdf2_hmac(
+                "sha256",
+                secret.encode("utf-8"),
+                ("FakeWallet").encode("utf-8"),
+                2048,
+                32,
+            ).hex(),
             "memo": None,
             "description_hash": None,
             "description": "",
@@ -53,22 +59,29 @@ class FakeWallet(Wallet):
             data["tags_set"] = ["d"]
             data["memo"] = memo
             data["description"] = memo
-        randomHash = data["privkey"][:6] + hashlib.sha256(
+        randomHash = (
-            str(random.getrandbits(256)).encode("utf-8")
+            data["privkey"][:6]
-        ).hexdigest()[6:]
+            + hashlib.sha256(str(random.getrandbits(256)).encode("utf-8")).hexdigest()[
+                6:
+            ]
+        )
         data["paymenthash"] = randomHash
         payment_request = encode(data)
         checking_id = randomHash
 
         return InvoiceResponse(True, checking_id, payment_request)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
         invoice = decode(bolt11)
-        if hasattr(invoice, 'checking_id') and invoice.checking_id[6:] == data["privkey"][:6]:
+        if (
+            hasattr(invoice, "checking_id")
+            and invoice.checking_id[6:] == data["privkey"][:6]
+        ):
             return PaymentResponse(True, invoice.payment_hash, 0)
         else:
-            return PaymentResponse(ok = False, error_message="Only internal invoices can be used!")
+            return PaymentResponse(
-
+                ok=False, error_message="Only internal invoices can be used!"
+            )
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
         return PaymentStatus(False)

lnbits/wallets/lnbits.py

@@ -80,7 +80,7 @@ class LNbitsWallet(Wallet):
 
         return InvoiceResponse(ok, checking_id, payment_request, error_message)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
         async with httpx.AsyncClient() as client:
             r = await client.post(
                 url=f"{self.endpoint}/api/v1/payments",

lnbits/wallets/lndgrpc.py

@@ -93,10 +93,11 @@ class LndWallet(Wallet):
             or getenv("LND_INVOICE_MACAROON")
         )
 
-        
         encrypted_macaroon = getenv("LND_GRPC_MACAROON_ENCRYPTED")
         if encrypted_macaroon:
-            macaroon = AESCipher(description="macaroon decryption").decrypt(encrypted_macaroon)    
+            macaroon = AESCipher(description="macaroon decryption").decrypt(
+                encrypted_macaroon
+            )
         self.macaroon = load_macaroon(macaroon)
 
         cert = open(self.cert_path, "rb").read()
@@ -143,10 +144,10 @@ class LndWallet(Wallet):
         payment_request = str(resp.payment_request)
         return InvoiceResponse(True, checking_id, payment_request, None)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
-        resp = await self.rpc.SendPayment(
+        fee_limit_fixed = ln.FeeLimit(fixed=fee_limit_msat // 1000)
-            lnrpc.SendPaymentRequest(payment_request=bolt11)
+        req = ln.SendRequest(payment_request=bolt11, fee_limit=fee_limit_fixed)
-        )
+        resp = await self.rpc.SendPaymentSync(req)
 
         if resp.payment_error:
             return PaymentResponse(False, "", 0, None, resp.payment_error)

lnbits/wallets/lndrest.py

@@ -39,7 +39,9 @@ class LndRestWallet(Wallet):
 
         encrypted_macaroon = getenv("LND_REST_MACAROON_ENCRYPTED")
         if encrypted_macaroon:
-            macaroon = AESCipher(description="macaroon decryption").decrypt(encrypted_macaroon)    
+            macaroon = AESCipher(description="macaroon decryption").decrypt(
+                encrypted_macaroon
+            )
         self.macaroon = load_macaroon(macaroon)
 
         self.auth = {"Grpc-Metadata-macaroon": self.macaroon}
@@ -97,15 +99,11 @@ class LndRestWallet(Wallet):
 
         return InvoiceResponse(True, checking_id, payment_request, None)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
         async with httpx.AsyncClient(verify=self.cert) as client:
             # set the fee limit for the payment
-            invoice = lnbits_bolt11.decode(bolt11)
             lnrpcFeeLimit = dict()
-            if invoice.amount_msat > 1000_000:
+            lnrpcFeeLimit["fixed_msat"] = "{}".format(fee_limit_msat)
-                lnrpcFeeLimit["percent"] = "1"  # in percent
-            else:
-                lnrpcFeeLimit["fixed"] = "10"  # in sat
 
             r = await client.post(
                 url=f"{self.endpoint}/v1/channels/transactions",
@@ -162,6 +160,7 @@ class LndRestWallet(Wallet):
 
         # for some reason our checking_ids are in base64 but the payment hashes
         # returned here are in hex, lnd is weird
+        checking_id = checking_id.replace("_", "/")
         checking_id = base64.b64decode(checking_id).hex()
 
         for p in r.json()["payments"]:

lnbits/wallets/lnpay.py

@@ -76,7 +76,7 @@ class LNPayWallet(Wallet):
 
         return InvoiceResponse(ok, checking_id, payment_request, error_message)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
         async with httpx.AsyncClient() as client:
             r = await client.post(
                 f"{self.endpoint}/wallet/{self.wallet_key}/withdraw",

lnbits/wallets/lntxbot.py

@@ -74,7 +74,7 @@ class LntxbotWallet(Wallet):
         data = r.json()
         return InvoiceResponse(True, data["payment_hash"], data["pay_req"], None)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
         async with httpx.AsyncClient() as client:
             r = await client.post(
                 f"{self.endpoint}/payinvoice",

lnbits/wallets/opennode.py

@@ -77,7 +77,7 @@ class OpenNodeWallet(Wallet):
         payment_request = data["lightning_invoice"]["payreq"]
         return InvoiceResponse(True, checking_id, payment_request, None)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
         async with httpx.AsyncClient() as client:
             r = await client.post(
                 f"{self.endpoint}/v2/withdrawals",

lnbits/wallets/spark.py

@@ -107,7 +107,7 @@ class SparkWallet(Wallet):
 
         return InvoiceResponse(ok, checking_id, payment_request, error_message)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
         try:
             r = await self.pay(bolt11)
         except (SparkError, UnknownError) as exc:

lnbits/wallets/void.py

@@ -25,7 +25,7 @@ class VoidWallet(Wallet):
         )
         return StatusResponse(None, 0)
 
-    async def pay_invoice(self, bolt11: str) -> PaymentResponse:
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
         raise Unsupported("")
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:

tests/extensions/bleskomat/test_lnurl_api.py

@@ -3,7 +3,10 @@ import secrets
 from lnbits.core.crud import get_wallet
 from lnbits.settings import HOST, PORT
 from lnbits.extensions.bleskomat.crud import get_bleskomat_lnurl
-from lnbits.extensions.bleskomat.helpers import generate_bleskomat_lnurl_signature, query_to_signing_payload
+from lnbits.extensions.bleskomat.helpers import (
+    generate_bleskomat_lnurl_signature,
+    query_to_signing_payload,
+)
 from tests.conftest import client
 from tests.helpers import credit_wallet
 from tests.extensions.bleskomat.conftest import bleskomat, lnurl
@@ -73,7 +76,9 @@ async def test_bleskomat_lnurl_api_valid_signature(client, bleskomat):
     }
     payload = query_to_signing_payload(query)
     signature = generate_bleskomat_lnurl_signature(
-        payload=payload, api_key_secret=bleskomat.api_key_secret, api_key_encoding=bleskomat.api_key_encoding
+        payload=payload,
+        api_key_secret=bleskomat.api_key_secret,
+        api_key_encoding=bleskomat.api_key_encoding,
     )
     response = await client.get(f"/bleskomat/u?{payload}&signature={signature}")
     assert response.status_code == 200
@@ -97,7 +102,9 @@ async def test_bleskomat_lnurl_api_action_insufficient_balance(client, lnurl):
     response = await client.get(f"/bleskomat/u?k1={secret}&pr={pr}")
     assert response.status_code == 200
     assert response.json()["status"] == "ERROR"
-    assert ("Insufficient balance" in response.json()["reason"]) or ("fee" in response.json()["reason"])
+    assert ("Insufficient balance" in response.json()["reason"]) or (
+        "fee" in response.json()["reason"]
+    )
     wallet = await get_wallet(bleskomat.wallet)
     assert wallet.balance_msat == 0
     bleskomat_lnurl = await get_bleskomat_lnurl(secret)
@@ -123,4 +130,4 @@ async def test_bleskomat_lnurl_api_action_success(client, lnurl):
     assert wallet.balance_msat == 50000
     bleskomat_lnurl = await get_bleskomat_lnurl(secret)
     assert bleskomat_lnurl.has_uses_remaining() == False
-    WALLET.pay_invoice.assert_called_once_with(pr)
+    WALLET.pay_invoice.assert_called_once_with(pr, 2000)

tests/mocks.py

@@ -9,28 +9,42 @@ from lnbits.wallets.base import (
 )
 from lnbits.settings import WALLET
 
-WALLET.status = AsyncMock(return_value=StatusResponse(
+WALLET.status = AsyncMock(
-    "",# no error
+    return_value=StatusResponse(
-    1000000,# msats
+        "",  # no error
-))
+        1000000,  # msats
-WALLET.create_invoice = AsyncMock(return_value=InvoiceResponse(
+    )
-    True,# ok
+)
-    "6621aafbdd7709ca6eea6203f362d64bd7cb2911baa91311a176b3ecaf2274bd",# checking_id (i.e. payment_hash)
+WALLET.create_invoice = AsyncMock(
-    "lntb1u1psezhgspp5vcs6477awuyu5mh2vgplxckkf0tuk2g3h253xydpw6e7etezwj7sdqqcqzpgxqyz5vqsp5dxpw8zs77hw5pla4wz4mfujllyxtlpu443auur2uxqdrs8q2h56q9qyyssq65zk30ylmygvv5y4tuwalnf3ttnqjn57ef6rmcqg0s53akem560jh8ptemjcmytn3lrlatw4hv9smg88exv3v4f4lqnp96s0psdrhxsp6pp75q",# payment_request
+    return_value=InvoiceResponse(
-    "",# no error
+        True,  # ok
-))
+        "6621aafbdd7709ca6eea6203f362d64bd7cb2911baa91311a176b3ecaf2274bd",  # checking_id (i.e. payment_hash)
-def pay_invoice_side_effect(payment_request: str):
+        "lntb1u1psezhgspp5vcs6477awuyu5mh2vgplxckkf0tuk2g3h253xydpw6e7etezwj7sdqqcqzpgxqyz5vqsp5dxpw8zs77hw5pla4wz4mfujllyxtlpu443auur2uxqdrs8q2h56q9qyyssq65zk30ylmygvv5y4tuwalnf3ttnqjn57ef6rmcqg0s53akem560jh8ptemjcmytn3lrlatw4hv9smg88exv3v4f4lqnp96s0psdrhxsp6pp75q",  # payment_request
+        "",  # no error
+    )
+)
+
+
+def pay_invoice_side_effect(
+    payment_request: str, fee_limit_msat: int
+) -> PaymentResponse:
     invoice = bolt11.decode(payment_request)
     return PaymentResponse(
-        True,# ok
+        True,  # ok
-        invoice.payment_hash,# checking_id (i.e. payment_hash)
+        invoice.payment_hash,  # checking_id (i.e. payment_hash)
-        0,# fee_msat
+        0,  # fee_msat
-        "",# no error
+        "",  # no error
     )
+
+
 WALLET.pay_invoice = AsyncMock(side_effect=pay_invoice_side_effect)
-WALLET.get_invoice_status = AsyncMock(return_value=PaymentStatus(
+WALLET.get_invoice_status = AsyncMock(
-    True,# paid
+    return_value=PaymentStatus(
-))
+        True,  # paid
-WALLET.get_payment_status = AsyncMock(return_value=PaymentStatus(
+    )
-    True,# paid
+)
-))
+WALLET.get_payment_status = AsyncMock(
+    return_value=PaymentStatus(
+        True,  # paid
+    )
+)