Fix payments chart (#1851)

* feat: payment history endpoint

* test payment history

* use new endpoint in frontend

* refactor tests

lnbits/core/crud.py

@@ -1,6 +1,6 @@
 import datetime
 import json
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional
 from urllib.parse import urlparse
 from uuid import UUID, uuid4
 
@@ -9,7 +9,7 @@ import shortuuid
 from lnbits import bolt11
 from lnbits.core.db import db
 from lnbits.core.models import WalletType
-from lnbits.db import Connection, Database, Filters, Page
+from lnbits.db import DB_TYPE, SQLITE, Connection, Database, Filters, Page
 from lnbits.extension_manager import InstallableExtension
 from lnbits.settings import (
     AdminSettings,
@@ -23,6 +23,7 @@ from .models import (
     BalanceCheck,
     Payment,
     PaymentFilters,
+    PaymentHistoryPoint,
     TinyURL,
     User,
     Wallet,
@@ -655,6 +656,79 @@ async def update_payment_extra(
     )
 
 
+async def update_pending_payments(wallet_id: str):
+    pending_payments = await get_payments(
+        wallet_id=wallet_id,
+        pending=True,
+        exclude_uncheckable=True,
+    )
+    for payment in pending_payments:
+        await payment.check_status()
+
+
+DateTrunc = Literal["hour", "day", "month"]
+sqlite_formats = {
+    "hour": "%Y-%m-%d %H:00:00",
+    "day": "%Y-%m-%d 00:00:00",
+    "month": "%Y-%m-01 00:00:00",
+}
+
+
+async def get_payments_history(
+    wallet_id: Optional[str] = None,
+    group: DateTrunc = "day",
+    filters: Optional[Filters] = None,
+) -> List[PaymentHistoryPoint]:
+    if not filters:
+        filters = Filters()
+    where = ["(pending = False OR amount < 0)"]
+    values = []
+    if wallet_id:
+        where.append("wallet = ?")
+        values.append(wallet_id)
+
+    if DB_TYPE == SQLITE and group in sqlite_formats:
+        date_trunc = f"strftime('{sqlite_formats[group]}', time, 'unixepoch')"
+    elif group in ("day", "hour", "month"):
+        date_trunc = f"date_trunc('{group}', time)"
+    else:
+        raise ValueError(f"Invalid group value: {group}")
+
+    transactions = await db.fetchall(
+        f"""
+        SELECT {date_trunc} date,
+               SUM(CASE WHEN amount > 0 THEN amount ELSE 0 END) income,
+               SUM(CASE WHEN amount < 0 THEN abs(amount) + abs(fee) ELSE 0 END) spending
+        FROM apipayments
+        {filters.where(where)}
+        GROUP BY date
+        ORDER BY date DESC
+        """,
+        filters.values(values),
+    )
+    if wallet_id:
+        wallet = await get_wallet(wallet_id)
+        if wallet:
+            balance = wallet.balance_msat
+        else:
+            raise ValueError("Unknown wallet")
+    else:
+        balance = await get_total_balance()
+
+    # since we dont know the balance at the starting point,
+    # we take the current balance and walk backwards
+    results: list[PaymentHistoryPoint] = []
+    for row in transactions:
+        results.insert(
+            0,
+            PaymentHistoryPoint(
+                balance=balance, date=row[0], income=row[1], spending=row[2]
+            ),
+        )
+        balance -= row.income - row.spending
+    return results
+
+
 async def delete_wallet_payment(
     checking_id: str, wallet_id: str, conn: Optional[Connection] = None
 ) -> None:

lnbits/core/models.py

@@ -258,6 +258,13 @@ class PaymentFilters(FilterModel):
     webhook_status: Optional[int]
 
 
+class PaymentHistoryPoint(BaseModel):
+    date: datetime.datetime
+    income: int
+    spending: int
+    balance: int
+
+
 class BalanceCheck(BaseModel):
     wallet: str
     service: str

lnbits/core/static/js/wallet.js

@@ -3,45 +3,24 @@
 Vue.component(VueQrcode.name, VueQrcode)
 Vue.use(VueQrcodeReader)
 
-function generateChart(canvas, payments) {
+function generateChart(canvas, rawData) {
-  var txs = []
+  const data = rawData.reduce(
-  var n = 0
+    (previous, current) => {
-  var data = {
+      previous.labels.push(current.date)
+      previous.income.push(current.income)
+      previous.spending.push(current.spending)
+      previous.cumulative.push(current.balance)
+      return previous
+    },
+    {
       labels: [],
       income: [],
-    outcome: [],
+      spending: [],
       cumulative: []
     }
-
-  _.each(
-    payments.filter(p => !p.pending).sort((a, b) => a.time - b.time),
-    tx => {
-      txs.push({
-        hour: Quasar.utils.date.formatDate(tx.date, 'YYYY-MM-DDTHH:00'),
-        sat: tx.sat
-      })
-    }
   )
 
-  _.each(_.groupBy(txs, 'hour'), (value, day) => {
+  return new Chart(canvas.getContext('2d'), {
-    var income = _.reduce(
-      value,
-      (memo, tx) => (tx.sat >= 0 ? memo + tx.sat : memo),
-      0
-    )
-    var outcome = _.reduce(
-      value,
-      (memo, tx) => (tx.sat < 0 ? memo + Math.abs(tx.sat) : memo),
-      0
-    )
-    n = n + income - outcome
-    data.labels.push(day)
-    data.income.push(income)
-    data.outcome.push(outcome)
-    data.cumulative.push(n)
-  })
-
-  new Chart(canvas.getContext('2d'), {
     type: 'bar',
     data: {
       labels: data.labels,
@@ -64,7 +43,7 @@ function generateChart(canvas, payments) {
           backgroundColor: window.Color('rgb(76,175,80)').alpha(0.5).rgbString() // green
         },
         {
-          data: data.outcome,
+          data: data.spending,
           type: 'bar',
           label: 'out',
           barPercentage: 0.75,
@@ -85,7 +64,7 @@ function generateChart(canvas, payments) {
           {
             type: 'time',
             display: true,
-            offset: true,
+            //offset: true,
             time: {
               minUnit: 'hour',
               stepSize: 3
@@ -248,7 +227,14 @@ new Vue({
         loading: false
       },
       paymentsChart: {
-        show: false
+        show: false,
+        group: {value: 'hour', label: 'Hour'},
+        groupOptions: [
+          {value: 'month', label: 'Month'},
+          {value: 'day', label: 'Day'},
+          {value: 'hour', label: 'Hour'}
+        ],
+        instance: null
       },
       disclaimerDialog: {
         show: false,
@@ -301,8 +287,26 @@ new Vue({
     },
     showChart: function () {
       this.paymentsChart.show = true
+      LNbits.api
+        .request(
+          'GET',
+          '/api/v1/payments/history?group=' + this.paymentsChart.group.value,
+          this.g.wallet.adminkey
+        )
+        .then(response => {
           this.$nextTick(() => {
-        generateChart(this.$refs.canvas, this.payments)
+            if (this.paymentsChart.instance) {
+              this.paymentsChart.instance.destroy()
+            }
+            this.paymentsChart.instance = generateChart(
+              this.$refs.canvas,
+              response.data
+            )
+          })
+        })
+        .catch(err => {
+          LNbits.utils.notifyApiError(err)
+          this.paymentsChart.show = false
         })
     },
     focusInput(el) {
@@ -803,6 +807,9 @@ new Vue({
   watch: {
     payments: function () {
       this.fetchBalance()
+    },
+    'paymentsChart.group': function () {
+      this.showChart()
     }
   },
   created: function () {

lnbits/core/templates/core/wallet.html

@@ -827,6 +827,19 @@
       <q-dialog v-model="paymentsChart.show">
         <q-card class="q-pa-sm" style="width: 800px; max-width: unset">
           <q-card-section>
+            <div class="row q-gutter-sm justify-between">
+              <div class="text-h6">Payments Chart</div>
+              <q-select
+                label="Group"
+                filled
+                dense
+                v-model="paymentsChart.group"
+                style="min-width: 120px"
+                :options="paymentsChart.groupOptions"
+              >
+              </q-select>
+            </div>
+
             <canvas ref="canvas" width="600" height="400"></canvas>
           </q-card-section>
         </q-card>

lnbits/core/views/api.py

@@ -40,6 +40,8 @@ from lnbits.core.models import (
     DecodePayment,
     Payment,
     PaymentFilters,
+    PaymentHistoryPoint,
+    Query,
     User,
     Wallet,
     WalletType,
@@ -71,6 +73,7 @@ from lnbits.utils.exchange_rates import (
 )
 
 from ..crud import (
+    DateTrunc,
     add_installed_extension,
     create_tinyurl,
     create_webpush_subscription,
@@ -81,6 +84,7 @@ from ..crud import (
     drop_extension_db,
     get_dbversions,
     get_payments,
+    get_payments_history,
     get_payments_paginated,
     get_standalone_payment,
     get_tinyurl,
@@ -88,6 +92,7 @@ from ..crud import (
     get_wallet_for_key,
     get_webpush_subscription,
     save_balance_check,
+    update_pending_payments,
     update_wallet,
 )
 from ..services import (
@@ -155,16 +160,7 @@ async def api_payments(
     wallet: WalletTypeInfo = Depends(get_key_type),
     filters: Filters = Depends(parse_filters(PaymentFilters)),
 ):
-    pending_payments = await get_payments(
+    await update_pending_payments(wallet.wallet.id)
-        wallet_id=wallet.wallet.id,
-        pending=True,
-        exclude_uncheckable=True,
-        filters=filters,
-    )
-    for payment in pending_payments:
-        await check_transaction_status(
-            wallet_id=payment.wallet_id, payment_hash=payment.payment_hash
-        )
     return await get_payments(
         wallet_id=wallet.wallet.id,
         pending=True,
@@ -173,6 +169,21 @@ async def api_payments(
     )
 
 
+@api_router.get(
+    "/api/v1/payments/history",
+    name="Get payments history",
+    response_model=List[PaymentHistoryPoint],
+    openapi_extra=generate_filter_params_openapi(PaymentFilters),
+)
+async def api_payments_history(
+    wallet: WalletTypeInfo = Depends(get_key_type),
+    group: DateTrunc = Query("day"),
+    filters: Filters[PaymentFilters] = Depends(parse_filters(PaymentFilters)),
+):
+    await update_pending_payments(wallet.wallet.id)
+    return await get_payments_history(wallet.wallet.id, group, filters)
+
+
 @api_router.get(
     "/api/v1/payments/paginated",
     name="Payment List",
@@ -185,16 +196,7 @@ async def api_payments_paginated(
     wallet: WalletTypeInfo = Depends(get_key_type),
     filters: Filters = Depends(parse_filters(PaymentFilters)),
 ):
-    pending = await get_payments_paginated(
+    await update_pending_payments(wallet.wallet.id)
-        wallet_id=wallet.wallet.id,
-        pending=True,
-        exclude_uncheckable=True,
-        filters=filters,
-    )
-    for payment in pending.data:
-        await check_transaction_status(
-            wallet_id=payment.wallet_id, payment_hash=payment.payment_hash
-        )
     page = await get_payments_paginated(
         wallet_id=wallet.wallet.id,
         pending=True,

tests/conftest.py

@@ -1,5 +1,6 @@
 # ruff: noqa: E402
 import asyncio
+from time import time
 
 import uvloop
 
@@ -11,11 +12,16 @@ from fastapi.testclient import TestClient
 from httpx import AsyncClient
 
 from lnbits.app import create_app
-from lnbits.core.crud import create_account, create_wallet, get_user
+from lnbits.core.crud import (
+    create_account,
+    create_wallet,
+    get_user,
+    update_payment_status,
+)
 from lnbits.core.models import CreateInvoice
 from lnbits.core.services import update_wallet_balance
 from lnbits.core.views.api import api_payments_create_invoice
-from lnbits.db import Database
+from lnbits.db import DB_TYPE, SQLITE, Database
 from lnbits.settings import settings
 from tests.helpers import (
     clean_database,
@@ -173,6 +179,31 @@ async def real_invoice():
     del invoice
 
 
+@pytest_asyncio.fixture(scope="session")
+async def fake_payments(client, adminkey_headers_from):
+    # Because sqlite only stores timestamps with milliseconds
+    # we have to wait a second to ensure a different timestamp than previous invoices
+    if DB_TYPE == SQLITE:
+        await asyncio.sleep(1)
+    ts = time()
+
+    fake_data = [
+        CreateInvoice(amount=10, memo="aaaa", out=False),
+        CreateInvoice(amount=100, memo="bbbb", out=False),
+        CreateInvoice(amount=1000, memo="aabb", out=False),
+    ]
+
+    for invoice in fake_data:
+        response = await client.post(
+            "/api/v1/payments", headers=adminkey_headers_from, json=invoice.dict()
+        )
+        assert response.is_success
+        await update_payment_status(response.json()["checking_id"], pending=False)
+
+    params = {"time[ge]": ts, "time[le]": time()}
+    return fake_data, params
+
+
 @pytest_asyncio.fixture(scope="function")
 async def hold_invoice():
     invoice = get_hold_invoice(100)

tests/core/views/test_api.py

@@ -1,23 +1,21 @@
 import asyncio
 import hashlib
-from time import time
 
 import pytest
 
 from lnbits import bolt11
 from lnbits.core.crud import get_standalone_payment, update_payment_details
-from lnbits.core.models import Payment
+from lnbits.core.models import CreateInvoice, Payment
 from lnbits.core.views.admin_api import api_auditor
 from lnbits.core.views.api import api_payment
-from lnbits.db import DB_TYPE, SQLITE
 from lnbits.settings import settings
 from lnbits.wallets import get_wallet_class
-from tests.conftest import CreateInvoice, api_payments_create_invoice
 
 from ...helpers import (
     cancel_invoice,
     get_random_invoice_data,
     is_fake,
+    is_regtest,
     pay_real_invoice,
     settle_invoice,
 )
@@ -250,29 +248,13 @@ async def test_pay_invoice_adminkey(client, invoice, adminkey_headers_from):
 
 
 @pytest.mark.asyncio
-async def test_get_payments(client, from_wallet, adminkey_headers_from):
+async def test_get_payments(client, adminkey_headers_from, fake_payments):
-    # Because sqlite only stores timestamps with milliseconds we have to wait a second
+    fake_data, filters = fake_payments
-    # to ensure a different timestamp than previous invoices due to this limitation
-    # both payments (normal and paginated) are tested at the same time as they are
-    # almost identical anyways
-    if DB_TYPE == SQLITE:
-        await asyncio.sleep(1)
-    ts = time()
-
-    fake_data = [
-        CreateInvoice(amount=10, memo="aaaa"),
-        CreateInvoice(amount=100, memo="bbbb"),
-        CreateInvoice(amount=1000, memo="aabb"),
-    ]
-
-    for invoice in fake_data:
-        await api_payments_create_invoice(invoice, from_wallet)
 
     async def get_payments(params: dict):
-        params["time[ge]"] = ts
         response = await client.get(
             "/api/v1/payments",
-            params=params,
+            params=filters | params,
             headers=adminkey_headers_from,
         )
         assert response.status_code == 200
@@ -298,9 +280,14 @@ async def test_get_payments(client, from_wallet, adminkey_headers_from):
     payments = await get_payments({"amount[gt]": 10000})
     assert len(payments) == 2
 
+
+@pytest.mark.asyncio
+async def test_get_payments_paginated(client, adminkey_headers_from, fake_payments):
+    fake_data, filters = fake_payments
+
     response = await client.get(
         "/api/v1/payments/paginated",
-        params={"limit": 2, "time[ge]": ts},
+        params=filters | {"limit": 2},
         headers=adminkey_headers_from,
     )
     assert response.status_code == 200
@@ -309,6 +296,38 @@ async def test_get_payments(client, from_wallet, adminkey_headers_from):
     assert paginated["total"] == len(fake_data)
 
 
+@pytest.mark.asyncio
+@pytest.mark.skipif(
+    is_regtest, reason="payments wont be confirmed rightaway in regtest"
+)
+async def test_get_payments_history(client, adminkey_headers_from, fake_payments):
+    fake_data, filters = fake_payments
+
+    response = await client.get(
+        "/api/v1/payments/history",
+        params=filters,
+        headers=adminkey_headers_from,
+    )
+
+    assert response.status_code == 200
+    data = response.json()
+    assert len(data) == 1
+    assert data[0]["spending"] == sum(
+        payment.amount * 1000 for payment in fake_data if payment.out
+    )
+    assert data[0]["income"] == sum(
+        payment.amount * 1000 for payment in fake_data if not payment.out
+    )
+
+    response = await client.get(
+        "/api/v1/payments/history?group=INVALID",
+        params=filters,
+        headers=adminkey_headers_from,
+    )
+
+    assert response.status_code == 400
+
+
 # check POST /api/v1/payments/decode
 @pytest.mark.asyncio
 async def test_decode_invoice(client, invoice):