refactor services

- add PaymentFiatAmount
- return Payment on api endpoints

lnbits/core/crud.py

@@ -36,6 +36,10 @@ from .models import (
 )
 
 
+def update_payment_extra():
+    pass
+
+
 async def create_account(
     account: Optional[Account] = None,
     conn: Optional[Connection] = None,
@@ -689,39 +693,24 @@ async def create_payment(
     previous_payment = await get_standalone_payment(checking_id, conn=conn)
     assert previous_payment is None, "Payment already exists"
 
-    expiry_ph = db.timestamp_placeholder("expiry")
-    await (conn or db).execute(
-        f"""
-        INSERT INTO apipayments
-          (wallet_id, checking_id, bolt11, payment_hash, preimage,
-           amount, status, memo, fee, extra, webhook, expiry)
-          VALUES (:wallet_id, :checking_id, :bolt11, :hash, :preimage,
-           :amount, :status, :memo, :fee, :extra, :webhook, {expiry_ph})
-        """,
-        {
-            "wallet_id": data.wallet_id,
-            "checking_id": checking_id,
-            "bolt11": data.payment_request,
-            "hash": data.payment_hash,
-            "preimage": data.preimage,
-            "amount": data.amount,
-            "status": status.value,
-            "memo": data.memo,
-            "fee": data.fee,
-            "extra": (
-                json.dumps(data.extra)
-                if data.extra and data.extra != {} and isinstance(data.extra, dict)
-                else None
-            ),
-            "webhook": data.webhook,
-            "expiry": data.expiry if data.expiry else None,
-        },
+    payment = Payment(
+        checking_id=checking_id,
+        status=status,
+        wallet_id=data.wallet_id,
+        payment_hash=data.payment_hash,
+        bolt11=data.bolt11,
+        amount=data.amount_msat,
+        memo=data.memo,
+        preimage=data.preimage,
+        expiry=data.expiry,
+        webhook=data.webhook,
+        fee=data.fee,
+        extra=data.extra or {},
     )
 
-    new_payment = await get_wallet_payment(data.wallet_id, data.payment_hash, conn=conn)
-    assert new_payment, "Newly created payment couldn't be retrieved"
+    await (conn or db).insert("apipayments", payment)
 
-    return new_payment
+    return payment
 
 
 async def update_payment(
@@ -830,7 +819,7 @@ async def is_internal_status_success(
     payment_hash: str, conn: Optional[Connection] = None
 ) -> bool:
     """
-    Returns True if the internal payment was found and has the given status,
+    Returns True if the internal payment was found and is successful,
     """
     payment = await (conn or db).fetchone(
         """
@@ -841,7 +830,7 @@ async def is_internal_status_success(
         Payment,
     )
     if not payment:
-        return True
+        return False
     return payment.status == PaymentState.SUCCESS.value
 
 

lnbits/core/migrations.py

@@ -616,3 +616,7 @@ async def m026_update_payment_table(db):
                 "checking_id": payment.get("checking_id"),
             },
         )
+
+
+async def m027_update_payment_table(db):
+    await db.execute("ALTER TABLE apipayments ADD COLUMN fiat_amounts TEXT")

lnbits/core/models.py

@@ -257,34 +257,62 @@ class PaymentState(str, Enum):
         return self.value
 
 
+class PaymentExtra(BaseModel):
+    comment: Optional[str] = None
+    success_action: Optional[str] = None
+    lnurl_response: Optional[str] = None
+
+
+class PayInvoice(BaseModel):
+    payment_request: str
+    description: Optional[str] = None
+    max_sat: Optional[int] = None
+    extra: Optional[dict] = {}
+
+
+class PaymentFiatAmounts(BaseModel):
+    wallet_fiat_currency: Optional[str] = None
+    wallet_fiat_amount: Optional[float] = None
+    wallet_fiat_rate: Optional[float] = None
+    fiat_currency: Optional[str] = None
+    fiat_amount: Optional[float] = None
+    fiat_rate: Optional[float] = None
+
+
 class CreatePayment(BaseModel):
     wallet_id: str
-    payment_request: str
     payment_hash: str
-    amount: int
+    bolt11: str
+    amount_msat: int
     memo: str
+    extra: Optional[dict] = {}
     preimage: Optional[str] = None
     expiry: Optional[datetime] = None
-    extra: Optional[dict] = None
     webhook: Optional[str] = None
     fee: int = 0
+    fiat_amounts: PaymentFiatAmounts = PaymentFiatAmounts()
 
 
 class Payment(BaseModel):
-    status: str
     checking_id: str
     payment_hash: str
     wallet_id: str
     amount: int
     fee: int
-    memo: Optional[str]
-    time: datetime
     bolt11: str
-    expiry: Optional[datetime]
-    extra: Optional[dict]
-    webhook: Optional[str]
+    status: str = PaymentState.PENDING
+    memo: Optional[str] = None
+    expiry: Optional[datetime] = None
+    webhook: Optional[str] = None
     webhook_status: Optional[int] = None
     preimage: Optional[str] = "0" * 64
+    tag: Optional[str] = None
+    extension: Optional[str] = None
+    time: datetime = datetime.now(timezone.utc)
+    created_at: datetime = datetime.now(timezone.utc)
+    updated_at: datetime = datetime.now(timezone.utc)
+    fiat_amounts: PaymentFiatAmounts = PaymentFiatAmounts()
+    extra: dict = {}
 
     @property
     def pending(self) -> bool:
@@ -298,12 +326,6 @@ class Payment(BaseModel):
     def failed(self) -> bool:
         return self.status == PaymentState.FAILED.value
 
-    @property
-    def tag(self) -> Optional[str]:
-        if self.extra is None:
-            return ""
-        return self.extra.get("tag")
-
     @property
     def msat(self) -> int:
         return self.amount
@@ -428,7 +450,6 @@ class CreateInvoice(BaseModel):
     def unit_is_from_allowed_currencies(cls, v):
         if v != "sat" and v not in allowed_currencies():
             raise ValueError("The provided unit is not supported")
-
         return v
 
 

lnbits/core/services.py

@@ -8,7 +8,6 @@ from urllib.parse import parse_qs, urlparse
 from uuid import UUID, uuid4
 
 import httpx
-from bolt11 import MilliSatoshi
 from bolt11 import decode as bolt11_decode
 from cryptography.hazmat.primitives import serialization
 from fastapi import Depends, WebSocket
@@ -54,7 +53,6 @@ from .crud import (
     get_account_by_email,
     get_account_by_pubkey,
     get_account_by_username,
-    get_payment,
     get_payments,
     get_standalone_payment,
     get_super_settings,
@@ -74,6 +72,7 @@ from .models import (
     BalanceDelta,
     CreatePayment,
     Payment,
+    PaymentFiatAmounts,
     PaymentState,
     User,
     UserExtra,
@@ -83,22 +82,17 @@ from .models import (
 
 async def calculate_fiat_amounts(
     amount: float,
-    wallet_id: str,
+    wallet: Wallet,
     currency: Optional[str] = None,
-    extra: Optional[dict] = None,
-    conn: Optional[Connection] = None,
-) -> tuple[int, Optional[dict]]:
-    wallet = await get_wallet(wallet_id, conn=conn)
-    assert wallet, "invalid wallet_id"
+) -> tuple[int, PaymentFiatAmounts]:
     wallet_currency = wallet.currency or settings.lnbits_default_accounting_currency
-
+    fiat_amounts = PaymentFiatAmounts()
     if currency and currency != "sat":
         amount_sat = await fiat_amount_as_satoshis(amount, currency)
-        extra = extra or {}
         if currency != wallet_currency:
-            extra["fiat_currency"] = currency
-            extra["fiat_amount"] = round(amount, ndigits=3)
-            extra["fiat_rate"] = amount_sat / amount
+            fiat_amounts.fiat_currency = currency
+            fiat_amounts.fiat_amount = round(amount, ndigits=3)
+            fiat_amounts.fiat_rate = amount_sat / amount
     else:
         amount_sat = int(amount)
 
@@ -107,16 +101,15 @@ async def calculate_fiat_amounts(
             fiat_amount = amount
         else:
             fiat_amount = await satoshis_amount_as_fiat(amount_sat, wallet_currency)
-        extra = extra or {}
-        extra["wallet_fiat_currency"] = wallet_currency
-        extra["wallet_fiat_amount"] = round(fiat_amount, ndigits=3)
-        extra["wallet_fiat_rate"] = amount_sat / fiat_amount
+        fiat_amounts.wallet_fiat_currency = wallet_currency
+        fiat_amounts.wallet_fiat_amount = round(fiat_amount, ndigits=3)
+        fiat_amounts.wallet_fiat_rate = amount_sat / fiat_amount
 
     logger.debug(
-        f"Calculated fiat amounts {wallet.id=} {amount=} {currency=}: {extra=}"
+        f"Calculated fiat amounts {wallet.id=} {amount=} {currency=}: {fiat_amounts=}"
     )
 
-    return amount_sat, extra
+    return amount_sat, fiat_amounts
 
 
 async def create_invoice(
@@ -132,7 +125,7 @@ async def create_invoice(
     webhook: Optional[str] = None,
     internal: Optional[bool] = False,
     conn: Optional[Connection] = None,
-) -> tuple[str, str]:
+) -> Payment:
     if not amount > 0:
         raise InvoiceError("Amountless invoices not supported.", status="failed")
 
@@ -145,8 +138,8 @@ async def create_invoice(
     # use the fake wallet if the invoice is for internal use only
     funding_source = fake_wallet if internal else get_funding_source()
 
-    amount_sat, extra = await calculate_fiat_amounts(
-        amount, wallet_id, currency=currency, extra=extra, conn=conn
+    amount_sat, fiat_amounts = await calculate_fiat_amounts(
+        amount, user_wallet, currency
     )
 
     if settings.is_wallet_max_balance_exceeded(
@@ -179,22 +172,202 @@ async def create_invoice(
 
     create_payment_model = CreatePayment(
         wallet_id=wallet_id,
-        payment_request=payment_request,
+        bolt11=payment_request,
         payment_hash=invoice.payment_hash,
-        amount=amount_sat * 1000,
+        amount_msat=amount_sat * 1000,
         expiry=invoice.expiry_date,
         memo=memo,
         extra=extra,
+        fiat_amounts=fiat_amounts,
         webhook=webhook,
     )
 
-    await create_payment(
+    payment = await create_payment(
         checking_id=checking_id,
         data=create_payment_model,
         conn=conn,
     )
 
-    return invoice.payment_hash, payment_request
+    return payment
+
+
+async def _pay_internal_invoice(
+    wallet: Wallet,
+    create_payment_model: CreatePayment,
+    conn: Optional[Connection] = None,
+) -> Optional[Payment]:
+    """
+    Pay an internal payment.
+    returns None if the payment is not internal.
+    """
+    # check_internal() returns the payment of the invoice we're waiting for
+    # (pending only)
+    internal_payment = await check_internal(
+        create_payment_model.payment_hash, conn=conn
+    )
+    if not internal_payment:
+        return None
+
+    # perform additional checks on the internal payment
+    # the payment hash is not enough to make sure that this is the same invoice
+    internal_invoice = await get_standalone_payment(
+        internal_payment.checking_id, incoming=True, conn=conn
+    )
+    if not internal_invoice:
+        raise PaymentError("Internal payment not found.", status="failed")
+
+    if (
+        internal_invoice.amount != abs(create_payment_model.amount_msat)
+        or internal_invoice.bolt11 != create_payment_model.bolt11.lower()
+    ):
+        raise PaymentError(
+            "Invalid invoice. Bolt11 or amount is not correct", status="failed"
+        )
+
+    fee_reserve_total_msat = fee_reserve_total(
+        abs(create_payment_model.amount_msat), internal=True
+    )
+    create_payment_model.fee = abs(fee_reserve_total_msat)
+
+    if (
+        wallet.balance_msat
+        < abs(create_payment_model.amount_msat) + fee_reserve_total_msat
+    ):
+        raise PaymentError("Insufficient balance.", status="failed")
+
+    internal_id = f"internal_{create_payment_model.payment_hash}"
+    logger.debug(f"creating temporary internal payment with id {internal_id}")
+    payment = await create_payment(
+        checking_id=internal_id,
+        data=create_payment_model,
+        status=PaymentState.SUCCESS,
+        conn=conn,
+    )
+
+    # mark the invoice from the other side as not pending anymore
+    # so the other side only has access to his new money when we are sure
+    # the payer has enough to deduct from
+    internal_payment.status = PaymentState.SUCCESS
+    await update_payment(internal_payment, conn=conn)
+
+    await send_payment_notification(wallet, payment)
+
+    # notify receiver asynchronously
+    from lnbits.tasks import internal_invoice_queue
+
+    logger.debug(f"enqueuing internal invoice {internal_payment.checking_id}")
+    await internal_invoice_queue.put(internal_payment.checking_id)
+
+    return payment
+
+
+async def _verify_external_payment(
+    payment: Payment, conn: Optional[Connection] = None
+) -> Payment:
+    # fail on pending payments
+    if payment.pending:
+        raise PaymentError("Payment is still pending.", status="pending")
+    if payment.success:
+        raise PaymentError("Payment already paid.", status="success")
+    if payment.failed:
+        status = await payment.check_status()
+        if status.success:
+            # payment was successful on the fundingsource
+            payment.status = PaymentState.SUCCESS
+            await update_payment(payment, conn=conn)
+            raise PaymentError(
+                "Failed payment was already paid on the fundingsource.",
+                status="success",
+            )
+        if status.failed:
+            raise PaymentError(
+                "Payment is failed node, retrying is not possible.", status="failed"
+            )
+        # status.pending fall through and try again
+    return payment
+
+
+async def _pay_external_invoice(
+    wallet: Wallet,
+    create_payment_model: CreatePayment,
+    conn: Optional[Connection] = None,
+) -> Payment:
+    fee_reserve_total_msat = fee_reserve_total(
+        create_payment_model.amount_msat, internal=False
+    )
+    create_payment_model.fee = -abs(fee_reserve_total_msat)
+
+    checking_id = create_payment_model.payment_hash
+
+    # check if there is already a payment with the same checking_id
+    old_payment = await get_standalone_payment(checking_id, conn=conn)
+    if old_payment:
+        return await _verify_external_payment(old_payment, conn)
+
+    if (
+        wallet.balance_msat
+        < abs(create_payment_model.amount_msat) + fee_reserve_total_msat
+    ):
+        raise PaymentError(
+            f"You must reserve at least ({round(fee_reserve_total_msat/1000)}"
+            "  sat) to cover potential routing fees.",
+            status="failed",
+        )
+
+    payment = await create_payment(
+        checking_id=checking_id,
+        data=create_payment_model,
+        conn=conn,
+    )
+
+    fee_reserve_msat = fee_reserve(create_payment_model.amount_msat, internal=False)
+    service_fee_msat = service_fee(create_payment_model.amount_msat, internal=False)
+
+    funding_source = get_funding_source()
+
+    logger.debug(f"fundingsource: sending payment {checking_id}")
+    payment_response: PaymentResponse = await funding_source.pay_invoice(
+        create_payment_model.bolt11, fee_reserve_msat
+    )
+    logger.debug(f"backend: pay_invoice finished {checking_id}, {payment_response}")
+    if payment_response.checking_id and payment_response.checking_id != checking_id:
+        logger.warning(
+            f"backend sent unexpected checking_id (expected: {checking_id} got:"
+            f" {payment_response.checking_id})"
+        )
+    if payment_response.checking_id and payment_response.ok is not False:
+        # payment.ok can be True (paid) or None (pending)!
+        logger.debug(f"updating payment {checking_id}")
+        # TODO: why? we log a warning on an unexpected checking_id
+        # should that be handled?
+        # new checking id
+        payment.checking_id = payment_response.checking_id
+        payment.status = (
+            PaymentState.SUCCESS
+            if payment_response.ok is True
+            else PaymentState.PENDING
+        )
+        payment.fee = -(abs(payment_response.fee_msat or 0) + abs(service_fee_msat))
+        payment.preimage = payment_response.preimage
+        await update_payment(payment, conn=conn)
+        await send_payment_notification(wallet, payment)
+        logger.success(f"payment successful {payment_response.checking_id}")
+    elif payment_response.checking_id is None and payment_response.ok is False:
+        # payment failed
+        logger.debug(f"payment failed {checking_id}, {payment_response.error_message}")
+        payment.status = PaymentState.FAILED
+        await update_payment(payment, conn=conn)
+        raise PaymentError(
+            f"Payment failed: {payment_response.error_message}"
+            or "Payment failed, but backend didn't give us an error message.",
+            status="failed",
+        )
+    else:
+        logger.warning(
+            "didn't receive checking_id from backend, payment may be stuck in"
+            f" database: {checking_id}"
+        )
+    return payment
 
 
 async def pay_invoice(
@@ -204,18 +377,9 @@ async def pay_invoice(
     max_sat: Optional[int] = None,
     extra: Optional[dict] = None,
     description: str = "",
+    tag: str = "",
     conn: Optional[Connection] = None,
-) -> str:
-    """
-    Pay a Lightning invoice.
-    First, we create a temporary payment in the database with fees set to the reserve
-    fee. We then check whether the balance of the payer would go negative.
-    We then attempt to pay the invoice through the backend. If the payment is
-    successful, we update the payment in the database with the payment details.
-    If the payment is unsuccessful, we delete the temporary payment.
-    If the payment is still in flight, we hope that some other process
-    will regularly check for the payment.
-    """
+) -> Payment:
     try:
         invoice = bolt11_decode(payment_request)
     except Exception as exc:
@@ -226,262 +390,96 @@ async def pay_invoice(
     if max_sat and invoice.amount_msat > max_sat * 1000:
         raise PaymentError("Amount in invoice is too high.", status="failed")
 
-    await check_wallet_limits(wallet_id, conn, invoice.amount_msat)
-
     async with db.reuse_conn(conn) if conn else db.connect() as conn:
-        temp_id = invoice.payment_hash
-        internal_id = f"internal_{invoice.payment_hash}"
 
-        _, extra = await calculate_fiat_amounts(
-            invoice.amount_msat / 1000, wallet_id, extra=extra, conn=conn
+        wallet = await get_wallet(wallet_id, conn=conn)
+        if not wallet:
+            raise PaymentError(
+                f"Could not fetch wallet '{wallet_id}'.", status="failed"
+            )
+
+        # check if the payment is made for an extension that the user disabled
+        if tag:
+            status = await check_user_extension_access(wallet.user, tag)
+            if not status.success:
+                raise PaymentError(status.message)
+
+        await check_wallet_limits(wallet_id, invoice.amount_msat, conn)
+
+        if await is_internal_status_success(invoice.payment_hash, conn):
+            raise PaymentError("Internal invoice already paid.", status="failed")
+
+        _, fiat_amounts = await calculate_fiat_amounts(
+            invoice.amount_msat / 1000, wallet
         )
 
         create_payment_model = CreatePayment(
             wallet_id=wallet_id,
-            payment_request=payment_request,
+            bolt11=payment_request,
             payment_hash=invoice.payment_hash,
-            amount=-invoice.amount_msat,
+            amount_msat=invoice.amount_msat * -1,
             expiry=invoice.expiry_date,
             memo=description or invoice.description or "",
             extra=extra,
+            fiat_amounts=fiat_amounts,
         )
 
-        if await is_internal_status_success(invoice.payment_hash, conn=conn):
-            raise PaymentError("Internal invoice already paid.", status="failed")
+        payment = await _pay_internal_invoice(wallet, create_payment_model, conn)
+        if not payment:
+            payment = await _pay_external_invoice(wallet, create_payment_model, conn)
 
-        # check_internal() returns the checking_id of the invoice we're waiting for
-        # (pending only)
-        internal_payment = await check_internal(invoice.payment_hash, conn=conn)
-        if internal_payment:
-            # perform additional checks on the internal payment
-            # the payment hash is not enough to make sure that this is the same invoice
-            internal_invoice = await get_standalone_payment(
-                internal_payment.checking_id, incoming=True, conn=conn
-            )
-            assert internal_invoice is not None
-            if (
-                internal_invoice.amount != invoice.amount_msat
-                or internal_invoice.bolt11 != payment_request.lower()
-            ):
-                raise PaymentError("Invalid invoice.", status="failed")
+        await _credit_service_fee_wallet(payment)
 
-            logger.debug(f"creating temporary internal payment with id {internal_id}")
-            # create a new payment from this wallet
+        return payment
 
-            fee_reserve_total_msat = fee_reserve_total(
-                invoice.amount_msat, internal=True
-            )
-            create_payment_model.fee = abs(fee_reserve_total_msat)
-            new_payment = await create_payment(
-                checking_id=internal_id,
-                data=create_payment_model,
-                status=PaymentState.SUCCESS,
-                conn=conn,
-            )
-        else:
-            new_payment = await _create_external_payment(
-                temp_id=temp_id,
-                amount_msat=invoice.amount_msat,
-                data=create_payment_model,
-                conn=conn,
-            )
 
-        # do the balance check
-        wallet = await get_wallet(wallet_id, conn=conn)
-        assert wallet, "Wallet for balancecheck could not be fetched"
-        fee_reserve_total_msat = fee_reserve_total(invoice.amount_msat, internal=False)
-        _check_wallet_balance(wallet, fee_reserve_total_msat, internal_payment)
-
-    if extra and "tag" in extra:
-        # check if the payment is made for an extension that the user disabled
-        status = await check_user_extension_access(wallet.user, extra["tag"])
-        if not status.success:
-            raise PaymentError(status.message)
-
-    if internal_payment:
-        service_fee_msat = service_fee(invoice.amount_msat, internal=True)
-        logger.debug(
-            f"marking temporary payment as not pending {internal_payment.checking_id}"
-        )
-        # mark the invoice from the other side as not pending anymore
-        # so the other side only has access to his new money when we are sure
-        # the payer has enough to deduct from
-        async with db.connect() as conn:
-            internal_payment.status = PaymentState.SUCCESS
-            await update_payment(internal_payment, conn=conn)
-        await send_payment_notification(wallet, new_payment)
-
-        # notify receiver asynchronously
-        from lnbits.tasks import internal_invoice_queue
-
-        logger.debug(f"enqueuing internal invoice {internal_payment.checking_id}")
-        await internal_invoice_queue.put(internal_payment.checking_id)
-    else:
-        fee_reserve_msat = fee_reserve(invoice.amount_msat, internal=False)
-        service_fee_msat = service_fee(invoice.amount_msat, internal=False)
-        logger.debug(f"backend: sending payment {temp_id}")
-        # actually pay the external invoice
-        funding_source = get_funding_source()
-        payment_response: PaymentResponse = await funding_source.pay_invoice(
-            payment_request, fee_reserve_msat
-        )
-
-        if payment_response.checking_id and payment_response.checking_id != temp_id:
-            logger.warning(
-                f"backend sent unexpected checking_id (expected: {temp_id} got:"
-                f" {payment_response.checking_id})"
-            )
-
-        logger.debug(f"backend: pay_invoice finished {temp_id}, {payment_response}")
-        if payment_response.checking_id and payment_response.ok is not False:
-            # payment.ok can be True (paid) or None (pending)!
-            logger.debug(f"updating payment {temp_id}")
-            async with db.connect() as conn:
-                payment = await get_payment(temp_id, conn=conn)
-                # new checking id
-                payment.checking_id = payment_response.checking_id
-                payment.status = (
-                    PaymentState.SUCCESS
-                    if payment_response.ok is True
-                    else PaymentState.PENDING
-                )
-                payment.fee = -(
-                    abs(payment_response.fee_msat or 0) + abs(service_fee_msat)
-                )
-                payment.preimage = payment_response.preimage
-                await update_payment(payment, conn=conn)
-                wallet = await get_wallet(wallet_id, conn=conn)
-                if wallet:
-                    await send_payment_notification(wallet, payment)
-                logger.success(f"payment successful {payment_response.checking_id}")
-        elif payment_response.checking_id is None and payment_response.ok is False:
-            # payment failed
-            logger.debug(f"payment failed {temp_id}, {payment_response.error_message}")
-            async with db.connect() as conn:
-                payment = await get_payment(temp_id, conn=conn)
-                payment.status = PaymentState.FAILED
-                await update_payment(payment, conn=conn)
-            raise PaymentError(
-                f"Payment failed: {payment_response.error_message}"
-                or "Payment failed, but backend didn't give us an error message.",
-                status="failed",
-            )
-        else:
-            logger.warning(
-                "didn't receive checking_id from backend, payment may be stuck in"
-                f" database: {temp_id}"
-            )
-
-    # credit service fee wallet
+async def _credit_service_fee_wallet(payment: Payment):
+    service_fee_msat = service_fee(payment.amount, internal=payment.is_internal)
     if settings.lnbits_service_fee_wallet and service_fee_msat:
         create_payment_model = CreatePayment(
             wallet_id=settings.lnbits_service_fee_wallet,
-            payment_request=payment_request,
-            payment_hash=invoice.payment_hash,
-            amount=abs(service_fee_msat),
+            bolt11=payment.bolt11,
+            payment_hash=payment.payment_hash,
+            amount_msat=abs(service_fee_msat),
             memo="Service fee",
         )
-        new_payment = await create_payment(
-            checking_id=f"service_fee_{temp_id}",
+        await create_payment(
+            checking_id=f"service_fee_{payment.payment_hash}",
             data=create_payment_model,
             status=PaymentState.SUCCESS,
         )
-    return invoice.payment_hash
 
 
-async def _create_external_payment(
-    temp_id: str,
-    amount_msat: MilliSatoshi,
-    data: CreatePayment,
-    conn: Optional[Connection],
-) -> Payment:
-    fee_reserve_total_msat = fee_reserve_total(amount_msat, internal=False)
-
-    # check if there is already a payment with the same checking_id
-    old_payment = await get_standalone_payment(temp_id, conn=conn)
-    if old_payment:
-        # fail on pending payments
-        if old_payment.pending:
-            raise PaymentError("Payment is still pending.", status="pending")
-        if old_payment.success:
-            raise PaymentError("Payment already paid.", status="success")
-        if old_payment.failed:
-            status = await old_payment.check_status()
-            if status.success:
-                # payment was successful on the fundingsource
-                old_payment.status = PaymentState.SUCCESS
-                await update_payment(old_payment, conn=conn)
-                raise PaymentError(
-                    "Failed payment was already paid on the fundingsource.",
-                    status="success",
-                )
-            if status.failed:
-                raise PaymentError(
-                    "Payment is failed node, retrying is not possible.", status="failed"
-                )
-            # status.pending fall through and try again
-        return old_payment
-
-    logger.debug(f"creating temporary payment with id {temp_id}")
-    # create a temporary payment here so we can check if
-    # the balance is enough in the next step
-    try:
-        data.fee = -abs(fee_reserve_total_msat)
-        new_payment = await create_payment(
-            checking_id=temp_id,
-            data=data,
-            conn=conn,
-        )
-        return new_payment
-    except Exception as exc:
-        logger.error(f"could not create temporary payment: {exc}")
-        # happens if the same wallet tries to pay an invoice twice
-        raise PaymentError("Could not make payment", status="failed") from exc
-
-
-def _check_wallet_balance(
-    wallet: Wallet,
-    fee_reserve_total_msat: int,
-    internal_payment: Optional[Payment] = None,
+async def check_wallet_limits(
+    wallet_id: str, amount_msat: int, conn: Optional[Connection] = None
 ):
-    if wallet.balance_msat < 0:
-        logger.debug("balance is too low, deleting temporary payment")
-        if not internal_payment and wallet.balance_msat > -fee_reserve_total_msat:
-            raise PaymentError(
-                f"You must reserve at least ({round(fee_reserve_total_msat/1000)}"
-                "  sat) to cover potential routing fees.",
-                status="failed",
-            )
-        raise PaymentError("Insufficient balance.", status="failed")
+    await check_time_limit_between_transactions(wallet_id, conn)
+    await check_wallet_daily_withdraw_limit(wallet_id, amount_msat, conn)
 
 
-async def check_wallet_limits(wallet_id, conn, amount_msat):
-    await check_time_limit_between_transactions(conn, wallet_id)
-    await check_wallet_daily_withdraw_limit(conn, wallet_id, amount_msat)
-
-
-async def check_time_limit_between_transactions(conn, wallet_id):
+async def check_time_limit_between_transactions(
+    wallet_id: str, conn: Optional[Connection] = None
+):
     limit = settings.lnbits_wallet_limit_secs_between_trans
     if not limit or limit <= 0:
         return
-
     payments = await get_payments(
         since=int(time.time()) - limit,
         wallet_id=wallet_id,
         limit=1,
         conn=conn,
     )
-
     if len(payments) == 0:
         return
-
     raise PaymentError(
         status="failed",
         message=f"The time limit of {limit} seconds between payments has been reached.",
     )
 
 
-async def check_wallet_daily_withdraw_limit(conn, wallet_id, amount_msat):
+async def check_wallet_daily_withdraw_limit(
+    wallet_id: str, amount_msat: int, conn: Optional[Connection] = None
+):
     limit = settings.lnbits_wallet_limit_daily_max_withdraw
     if not limit:
         return
@@ -689,39 +687,33 @@ def fee_reserve_total(amount_msat: int, internal: bool = False) -> int:
 
 
 async def send_payment_notification(wallet: Wallet, payment: Payment):
-    await websocket_updater(
-        wallet.inkey,
-        json.dumps(
-            {
-                "wallet_balance": wallet.balance,
-                "payment": payment.dict(),
-            }
-        ),
-    )
-
-    await websocket_updater(
-        payment.payment_hash, json.dumps({"pending": payment.pending})
+    await websocket_manager.send_data(payment.json(), wallet.inkey)
+    # json.dumps(
+    #     {
+    #         "wallet_balance": wallet.balance,
+    #         "payment": payment,
+    #     }
+    # ),
+    await websocket_manager.send_data(
+        json.dumps({"pending": payment.pending}), payment.payment_hash
     )
 
 
 async def update_wallet_balance(wallet_id: str, amount: int):
     async with db.connect() as conn:
-        payment_hash, _ = await create_invoice(
+        payment = await create_invoice(
             wallet_id=wallet_id,
             amount=amount,
             memo="Admin top up",
             internal=True,
             conn=conn,
         )
-        internal_payment = await check_internal(payment_hash, conn=conn)
-        assert internal_payment, "newly created checking_id cannot be retrieved"
-
-        internal_payment.status = PaymentState.SUCCESS
-        await update_payment(internal_payment, conn=conn)
+        payment.status = PaymentState.SUCCESS
+        await update_payment(payment, conn=conn)
         # notify receiver asynchronously
         from lnbits.tasks import internal_invoice_queue
 
-        await internal_invoice_queue.put(internal_payment.checking_id)
+        await internal_invoice_queue.put(payment.checking_id)
 
 
 async def check_admin_settings():
@@ -876,8 +868,8 @@ class WebsocketConnectionManager:
 websocket_manager = WebsocketConnectionManager()
 
 
-async def websocket_updater(item_id, data):
-    return await websocket_manager.send_data(f"{data}", item_id)
+async def websocket_updater(item_id: str, data: str):
+    return await websocket_manager.send_data(data, item_id)
 
 
 async def switch_to_voidwallet() -> None:

lnbits/core/views/extension_api.py

@@ -1,3 +1,5 @@
+import sys
+import traceback
 from http import HTTPStatus
 
 from bolt11 import decode as bolt11_decode
@@ -84,6 +86,8 @@ async def api_install_extension(data: CreateExtension):
 
     except Exception as exc:
         logger.warning(exc)
+        etype, _, tb = sys.exc_info()
+        traceback.print_exception(etype, exc, tb)
         ext_info.clean_extension_files()
         detail = (
             str(exc)
@@ -430,7 +434,7 @@ async def get_pay_to_enable_invoice(
             ),
         )
 
-    payment_hash, payment_request = await create_invoice(
+    payment = await create_invoice(
         wallet_id=ext.meta.pay_to_enable.wallet,
         amount=data.amount,
         memo=f"Enable '{ext.name}' extension.",
@@ -441,10 +445,10 @@ async def get_pay_to_enable_invoice(
         user_ext = UserExtension(user=user.id, extension=ext_id, active=False)
         await create_user_extension(user_ext)
     user_ext_info = user_ext.extra if user_ext.extra else UserExtensionInfo()
-    user_ext_info.payment_hash_to_enable = payment_hash
+    user_ext_info.payment_hash_to_enable = payment.payment_hash
     user_ext.extra = user_ext_info
     await update_user_extension(user_ext)
-    return {"payment_hash": payment_hash, "payment_request": payment_request}
+    return {"payment_hash": payment.payment_hash, "payment_request": payment.bolt11}
 
 
 @extension_router.get(

lnbits/core/views/payment_api.py

@@ -3,13 +3,12 @@ import json
 import uuid
 from http import HTTPStatus
 from math import ceil
-from typing import List, Optional, Union
+from typing import List, Optional
 from urllib.parse import urlparse
 
 import httpx
 from fastapi import (
     APIRouter,
-    Body,
     Depends,
     Header,
     HTTPException,
@@ -21,7 +20,6 @@ from loguru import logger
 from sse_starlette.sse import EventSourceResponse
 
 from lnbits import bolt11
-from lnbits.core.db import db
 from lnbits.core.models import (
     CreateInvoice,
     CreateLnurl,
@@ -121,7 +119,7 @@ async def api_payments_paginated(
     return page
 
 
-async def api_payments_create_invoice(data: CreateInvoice, wallet: Wallet):
+async def _api_payments_create_invoice(data: CreateInvoice, wallet: Wallet):
     description_hash = b""
     unhashed_description = b""
     memo = data.memo or settings.lnbits_site_title
@@ -145,60 +143,42 @@ async def api_payments_create_invoice(data: CreateInvoice, wallet: Wallet):
         # do not save memo if description_hash or unhashed_description is set
         memo = ""
 
-    async with db.connect() as conn:
-        payment_hash, payment_request = await create_invoice(
-            wallet_id=wallet.id,
-            amount=data.amount,
-            memo=memo,
-            currency=data.unit,
-            description_hash=description_hash,
-            unhashed_description=unhashed_description,
-            expiry=data.expiry,
-            extra=data.extra,
-            webhook=data.webhook,
-            internal=data.internal,
-            conn=conn,
-        )
-        # NOTE: we get the checking_id with a seperate query because create_invoice
-        # does not return it and it would be a big hustle to change its return type
-        # (used across extensions)
-        payment_db = await get_standalone_payment(payment_hash, conn=conn)
-        assert payment_db is not None, "payment not found"
-        checking_id = payment_db.checking_id
+    payment = await create_invoice(
+        wallet_id=wallet.id,
+        amount=data.amount,
+        memo=memo,
+        currency=data.unit,
+        description_hash=description_hash,
+        unhashed_description=unhashed_description,
+        expiry=data.expiry,
+        extra=data.extra,
+        webhook=data.webhook,
+        internal=data.internal,
+    )
 
-    invoice = bolt11.decode(payment_request)
-
-    lnurl_response: Union[None, bool, str] = None
+    # lnurl_response is not saved in the database
     if data.lnurl_callback:
         headers = {"User-Agent": settings.user_agent}
         async with httpx.AsyncClient(headers=headers) as client:
             try:
                 r = await client.get(
                     data.lnurl_callback,
-                    params={
-                        "pr": payment_request,
-                    },
+                    params={"pr": payment.bolt11},
                     timeout=10,
                 )
                 if r.is_error:
-                    lnurl_response = r.text
+                    payment.extra["lnurl_response"] = r.text
                 else:
                     resp = json.loads(r.text)
                     if resp["status"] != "OK":
-                        lnurl_response = resp["reason"]
+                        payment.extra["lnurl_response"] = resp["reason"]
                     else:
-                        lnurl_response = True
+                        payment.extra["lnurl_response"] = True
             except (httpx.ConnectError, httpx.RequestError) as ex:
                 logger.error(ex)
-                lnurl_response = False
+                payment.extra["lnurl_response"] = False
 
-    return {
-        "payment_hash": invoice.payment_hash,
-        "payment_request": payment_request,
-        "lnurl_response": lnurl_response,
-        # maintain backwards compatibility with API clients:
-        "checking_id": checking_id,
-    }
+    return payment
 
 
 @payment_router.post(
@@ -220,30 +200,25 @@ async def api_payments_create_invoice(data: CreateInvoice, wallet: Wallet):
     },
 )
 async def api_payments_create(
+    invoice_data: CreateInvoice,
     wallet: WalletTypeInfo = Depends(require_invoice_key),
-    invoice_data: CreateInvoice = Body(...),
-):
+) -> Payment:
     if invoice_data.out is True and wallet.key_type == KeyType.admin:
         if not invoice_data.bolt11:
             raise HTTPException(
                 status_code=HTTPStatus.BAD_REQUEST,
-                detail="BOLT11 string is invalid or not given",
+                detail="Missing BOLT11 invoice",
             )
-
-        payment_hash = await pay_invoice(
+        payment = await pay_invoice(
             wallet_id=wallet.wallet.id,
             payment_request=invoice_data.bolt11,
             extra=invoice_data.extra,
         )
-        return {
-            "payment_hash": payment_hash,
-            # maintain backwards compatibility with API clients:
-            "checking_id": payment_hash,
-        }
+        return payment
 
     elif not invoice_data.out:
         # invoice key
-        return await api_payments_create_invoice(invoice_data, wallet.wallet)
+        return await _api_payments_create_invoice(invoice_data, wallet.wallet)
     else:
         raise HTTPException(
             status_code=HTTPStatus.UNAUTHORIZED,
@@ -269,7 +244,7 @@ async def api_payments_fee_reserve(invoice: str = Query("invoice")) -> JSONRespo
 @payment_router.post("/lnurl")
 async def api_payments_pay_lnurl(
     data: CreateLnurl, wallet: WalletTypeInfo = Depends(require_admin_key)
-):
+) -> Payment:
     domain = urlparse(data.callback).netloc
 
     headers = {"User-Agent": settings.user_agent}
@@ -313,15 +288,12 @@ async def api_payments_pay_lnurl(
         raise HTTPException(
             status_code=HTTPStatus.BAD_REQUEST,
             detail=(
-                (
-                    f"{domain} returned an invalid invoice. Expected"
-                    f" {amount_msat} msat, got {invoice.amount_msat}."
-                ),
+                f"{domain} returned an invalid invoice. Expected"
+                f" {amount_msat} msat, got {invoice.amount_msat}."
             ),
         )
 
     extra = {}
-
     if params.get("successAction"):
         extra["success_action"] = params["successAction"]
     if data.comment:
@@ -330,19 +302,14 @@ async def api_payments_pay_lnurl(
         extra["fiat_currency"] = data.unit
         extra["fiat_amount"] = data.amount / 1000
     assert data.description is not None, "description is required"
-    payment_hash = await pay_invoice(
+
+    payment = await pay_invoice(
         wallet_id=wallet.wallet.id,
         payment_request=params["pr"],
         description=data.description,
         extra=extra,
     )
-
-    return {
-        "success_action": params.get("successAction"),
-        "payment_hash": payment_hash,
-        # maintain backwards compatibility with API clients:
-        "checking_id": payment_hash,
-    }
+    return payment
 
 
 async def subscribe_wallet_invoices(request: Request, wallet: Wallet):

lnbits/db.py

@@ -654,6 +654,7 @@ def dict_to_model(_row: dict, model: type[TModel]) -> TModel:
         if issubclass(type_, BaseModel) and value:
             _dict[key] = dict_to_submodel(type_, value)
             continue
+        # TODO: remove this when all sub models are migrated to Pydantic
         if type_ is dict and value:
             _dict[key] = json.loads(value)
             continue

lnbits/static/js/wallet.js

@@ -144,9 +144,11 @@ window.app = Vue.createApp({
         )
         .then(response => {
           this.receive.status = 'success'
-          this.receive.paymentReq = response.data.payment_request
+          this.receive.paymentReq = response.data.bolt11
           this.receive.paymentHash = response.data.payment_hash
 
+          // TODO: lnurl_callback and lnurl_response
+          // WITHDRAW
           if (response.data.lnurl_response !== null) {
             if (response.data.lnurl_response === false) {
               response.data.lnurl_response = `Unable to connect`
@@ -393,12 +395,13 @@ window.app = Vue.createApp({
                   dismissPaymentMsg()
                   clearInterval(this.parse.paymentChecker)
                   // show lnurlpay success action
-                  if (response.data.success_action) {
-                    switch (response.data.success_action.tag) {
+                  const extra = response.data.extra
+                  if (extra.success_action) {
+                    switch (extra.success_action.tag) {
                       case 'url':
                         Quasar.Notify.create({
-                          message: `<a target="_blank" style="color: inherit" href="${response.data.success_action.url}">${response.data.success_action.url}</a>`,
-                          caption: response.data.success_action.description,
+                          message: `<a target="_blank" style="color: inherit" href="${extra.success_action.url}">${extra.success_action.url}</a>`,
+                          caption: extra.success_action.description,
                           html: true,
                           type: 'positive',
                           timeout: 0,
@@ -407,7 +410,7 @@ window.app = Vue.createApp({
                         break
                       case 'message':
                         Quasar.Notify.create({
-                          message: response.data.success_action.message,
+                          message: extra.success_action.message,
                           type: 'positive',
                           timeout: 0,
                           closeBtn: true
@@ -418,14 +421,14 @@ window.app = Vue.createApp({
                           .getPayment(this.g.wallet, response.data.payment_hash)
                           .then(({data: payment}) =>
                             decryptLnurlPayAES(
-                              response.data.success_action,
+                              extra.success_action,
                               payment.preimage
                             )
                           )
                           .then(value => {
                             Quasar.Notify.create({
                               message: value,
-                              caption: response.data.success_action.description,
+                              caption: extra.success_action.description,
                               html: true,
                               type: 'positive',
                               timeout: 0,

tests/conftest.py

@@ -26,7 +26,6 @@ from lnbits.core.crud import (
 )
 from lnbits.core.models import Account, CreateInvoice, PaymentState, User
 from lnbits.core.services import create_user_account, update_wallet_balance
-from lnbits.core.views.payment_api import api_payments_create_invoice
 from lnbits.db import DB_TYPE, SQLITE, Database
 from lnbits.settings import AuthMethods, Settings
 from lnbits.settings import settings as lnbits_settings
@@ -222,12 +221,16 @@ async def adminkey_headers_to(to_wallet):
 
 
 @pytest_asyncio.fixture(scope="session")
-async def invoice(to_wallet):
+async def invoice(client, adminkey_headers_from):
     data = await get_random_invoice_data()
     invoice_data = CreateInvoice(**data)
-    invoice = await api_payments_create_invoice(invoice_data, to_wallet)
-    yield invoice
-    del invoice
+    response = await client.post(
+        "/api/v1/payments", headers=adminkey_headers_from, json=invoice_data.dict()
+    )
+    assert response.is_success
+    data = response.json()
+    assert data["checking_id"]
+    yield data["bolt11"]
 
 
 @pytest_asyncio.fixture(scope="session")

tests/regtest/test_services_create_invoice.py

@@ -12,35 +12,35 @@ description = "test create invoice"
 
 @pytest.mark.asyncio
 async def test_create_invoice(from_wallet):
-    payment_hash, pr = await create_invoice(
+    payment = await create_invoice(
         wallet_id=from_wallet.id,
         amount=1000,
         memo=description,
     )
-    invoice = decode(pr)
-    assert invoice.payment_hash == payment_hash
+    invoice = decode(payment.bolt11)
+    assert invoice.payment_hash == payment.payment_hash
     assert invoice.amount_msat == 1000000
     assert invoice.description == description
 
     funding_source = get_funding_source()
-    status = await funding_source.get_invoice_status(payment_hash)
+    status = await funding_source.get_invoice_status(payment.payment_hash)
     assert isinstance(status, PaymentStatus)
     assert status.pending
 
 
 @pytest.mark.asyncio
 async def test_create_internal_invoice(from_wallet):
-    payment_hash, pr = await create_invoice(
+    payment = await create_invoice(
         wallet_id=from_wallet.id, amount=1000, memo=description, internal=True
     )
-    invoice = decode(pr)
-    assert invoice.payment_hash == payment_hash
+    invoice = decode(payment.bolt11)
+    assert invoice.payment_hash == payment.payment_hash
     assert invoice.amount_msat == 1000000
     assert invoice.description == description
 
     # Internal invoices are not on fundingsource. so we should get some kind of error
     # that the invoice is not found, but we get status pending
     funding_source = get_funding_source()
-    status = await funding_source.get_invoice_status(payment_hash)
+    status = await funding_source.get_invoice_status(payment.payment_hash)
     assert isinstance(status, PaymentStatus)
     assert status.pending

tests/regtest/test_services_pay_invoice.py

@@ -1,8 +1,5 @@
 import pytest
 
-from lnbits.core.crud import (
-    get_standalone_payment,
-)
 from lnbits.core.services import (
     PaymentError,
     PaymentState,
@@ -14,13 +11,11 @@ description = "test pay invoice"
 
 @pytest.mark.asyncio
 async def test_services_pay_invoice(to_wallet, real_invoice):
-    payment_hash = await pay_invoice(
+    payment = await pay_invoice(
         wallet_id=to_wallet.id,
         payment_request=real_invoice.get("bolt11"),
         description=description,
     )
-    assert payment_hash
-    payment = await get_standalone_payment(payment_hash)
     assert payment
     assert not payment.status == PaymentState.SUCCESS
     assert payment.memo == description