From 96649fb31c622f28d8663c98af035c5f8dd1c531 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Tue, 19 Nov 2024 14:54:10 +0200 Subject: [PATCH] feat: create entry --- lnbits/core/crud/__init__.py | 3 +++ lnbits/core/crud/audit.py | 12 ++++++++++++ lnbits/core/migrations.py | 21 +++++++++++++++++++++ lnbits/core/models/__init__.py | 3 +++ lnbits/core/models/audit.py | 30 ++++++++++++++++++++++++++++++ lnbits/core/tasks.py | 14 ++++++++++---- lnbits/middleware.py | 24 ++++++++++++------------ lnbits/settings.py | 9 +++++---- 8 files changed, 96 insertions(+), 20 deletions(-) create mode 100644 lnbits/core/crud/audit.py create mode 100644 lnbits/core/models/audit.py diff --git a/lnbits/core/crud/__init__.py b/lnbits/core/crud/__init__.py index 234e7b7e3..7433656f0 100644 --- a/lnbits/core/crud/__init__.py +++ b/lnbits/core/crud/__init__.py @@ -1,3 +1,4 @@ +from .audit import create_audit_entry from .db_versions import ( delete_dbversion, get_db_version, @@ -82,6 +83,8 @@ from .webpush import ( ) __all__ = [ + # audit + "create_audit_entry", # db_versions "get_db_version", "get_db_versions", diff --git a/lnbits/core/crud/audit.py b/lnbits/core/crud/audit.py new file mode 100644 index 000000000..4131ea69e --- /dev/null +++ b/lnbits/core/crud/audit.py @@ -0,0 +1,12 @@ +from typing import Optional + +from lnbits.core.db import db +from lnbits.core.models import AuditEntry +from lnbits.db import Connection + + +async def create_audit_entry( + entry: AuditEntry, + conn: Optional[Connection] = None, +) -> None: + await (conn or db).insert("audit", entry) diff --git a/lnbits/core/migrations.py b/lnbits/core/migrations.py index 6efe2042e..145205874 100644 --- a/lnbits/core/migrations.py +++ b/lnbits/core/migrations.py @@ -664,3 +664,24 @@ async def m028_update_settings(db: Connection): await _insert_key_value(key, value) await db.execute("drop table settings") + + +async def m029_create_audit_table(db): + await db.execute( + f""" + CREATE TABLE IF NOT EXISTS audit ( + id {db.serial_primary_key}, + ip_address TEXT, + user_id TEXT, + path TEXT, + route_path TEXT, + request_type TEXT, + request_method TEXT, + query_string TEXT, + response_code TEXT, + duration REAL NOT NULL, + delete_at TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT {db.timestamp_now} + ); + """ + ) diff --git a/lnbits/core/models/__init__.py b/lnbits/core/models/__init__.py index aef5146b8..d5ffe7282 100644 --- a/lnbits/core/models/__init__.py +++ b/lnbits/core/models/__init__.py @@ -1,3 +1,4 @@ +from .audit import AuditEntry from .lnurl import CreateLnurl, CreateLnurlAuth, PayLnurlWData from .misc import ( BalanceDelta, @@ -41,6 +42,8 @@ from .wallets import BaseWallet, CreateWallet, KeyType, Wallet, WalletTypeInfo from .webpush import CreateWebPushSubscription, WebPushSubscription __all__ = [ + # audit + "AuditEntry", # lnurl "CreateLnurl", "CreateLnurlAuth", diff --git a/lnbits/core/models/audit.py b/lnbits/core/models/audit.py new file mode 100644 index 000000000..7b438512f --- /dev/null +++ b/lnbits/core/models/audit.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Optional + +from pydantic import BaseModel, Field + +from lnbits.settings import settings + + +class AuditEntry(BaseModel): + id: Optional[int] = None + ip_address: Optional[str] = None + user_id: Optional[str] = None + path: Optional[str] = None + route_path: Optional[str] = None + request_type: Optional[str] = None + request_method: Optional[str] = None + query_string: Optional[str] = None + response_code: Optional[str] = None + duration: float + delete_at: Optional[datetime] = None + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + def __init__(self, **data): + super.__init__(**data) + if settings.lnbits_audit_retention_days > 0: + self.delete_at = self.created_at + timedelta( + days=settings.lnbits_audit_retention_days + ) diff --git a/lnbits/core/tasks.py b/lnbits/core/tasks.py index 5e2b39805..9c7bed4fc 100644 --- a/lnbits/core/tasks.py +++ b/lnbits/core/tasks.py @@ -5,11 +5,12 @@ import httpx from loguru import logger from lnbits.core.crud import ( + create_audit_entry, get_wallet, get_webpush_subscriptions_for_user, mark_webhook_sent, ) -from lnbits.core.models import Payment +from lnbits.core.models import AuditEntry, Payment from lnbits.core.services import ( get_balance_delta, send_payment_notification, @@ -162,8 +163,13 @@ async def send_payment_push_notification(payment: Payment): async def wait_for_audit_data(): """ - . + Waits for audit entries to be pushed to the queue. + Then it inserts the entries into the DB. """ while settings.lnbits_running: - data: dict = await audit_queue.get() - print("### data", data) + data: AuditEntry = await audit_queue.get() + try: + await create_audit_entry(data) + except Exception as ex: + logger.warning(ex) + await asyncio.sleep(3) diff --git a/lnbits/middleware.py b/lnbits/middleware.py index 7a643f371..33385e232 100644 --- a/lnbits/middleware.py +++ b/lnbits/middleware.py @@ -14,6 +14,7 @@ from starlette.middleware.gzip import GZipMiddleware from starlette.types import ASGIApp, Receive, Scope, Send from lnbits.core.db import core_app_extra +from lnbits.core.models import AuditEntry from lnbits.helpers import template_renderer from lnbits.settings import settings @@ -151,19 +152,18 @@ class AuditMiddleware(BaseHTTPMiddleware): path = request.scope.get("path", None) response_code = str(response.status_code) if response else None if not settings.is_http_request_auditable(http_method, path, response_code): - print("### NOT", http_method, path, response_code) return None - data = { - "ip": request.client.host if request.client else None, - "user_id": request.scope.get("user_id", None), - "path": path, - "route_path": getattr(request.scope.get("route", {}), "path", None), - "request_type": request.scope.get("type", None), - "request_method": http_method, - "query_string": request.scope.get("query_string", None), - "response_code": response_code, - "duration": duration, - } + data = AuditEntry( + ip_address=request.client.host if request.client else None, + user_id=request.scope.get("user_id", None), + path=path, + route_path=getattr(request.scope.get("route", {}), "path", None), + request_type=request.scope.get("type", None), + request_method=http_method, + query_string=request.scope.get("query_string", None), + response_code=response_code, + duration=duration, + ) await self.audit_queue.put(data) except Exception as ex: logger.warning(ex) diff --git a/lnbits/settings.py b/lnbits/settings.py index 080fd2a2f..6631558c8 100644 --- a/lnbits/settings.py +++ b/lnbits/settings.py @@ -27,7 +27,6 @@ def list_parse_fallback(v: str): return [] - class LNbitsSettings(BaseModel): @classmethod def validate_list(cls, val): @@ -514,7 +513,9 @@ class KeycloakAuthSettings(LNbitsSettings): class AuditSettings(LNbitsSettings): lnbits_audit_enabled: bool = Field(default=True) - # If true the client IP address will be loged + # number of days to keep the audit entry + lnbits_audit_retention_days: int = Field(default=7) + lnbits_audit_log_ip: bool = Field(default=False) # List of paths to be included (regex match). Empty list means all. @@ -525,7 +526,7 @@ class AuditSettings(LNbitsSettings): ) # List of HTTP methods to be included. Empty lists means all. - # GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS + # Options (case-sensitive): GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS lnbits_audit_http_methods: list[str] = Field(default=[]) # List of HTTP codes to be included (regex match). Empty lists means all. @@ -768,6 +769,7 @@ class SettingsField(BaseModel): value: Optional[Any] tag: str = "core" + def _re_fullmatch_safe(pattern: str, string: str): try: return re.fullmatch(pattern, string) is not None @@ -776,7 +778,6 @@ def _re_fullmatch_safe(pattern: str, string: str): return False - def set_cli_settings(**kwargs): for key, value in kwargs.items(): setattr(settings, key, value)