Feat: audit (#2779)

This commit is contained in:
Vlad Stan 2024-11-27 13:06:35 +02:00 committed by GitHub
parent f97f27121a
commit fa8d7c665b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 1331 additions and 8 deletions

View File

@ -24,7 +24,10 @@ from lnbits.core.crud import (
from lnbits.core.helpers import migrate_extension_database
from lnbits.core.services.extensions import deactivate_extension, get_valid_extensions
from lnbits.core.tasks import ( # watchdog_task
audit_queue,
killswitch_task,
purge_audit_data,
wait_for_audit_data,
wait_for_paid_invoices,
)
from lnbits.exceptions import register_exception_handlers
@ -49,6 +52,7 @@ from .core.db import core_app_extra
from .core.models.extensions import Extension, ExtensionMeta, InstallableExtension
from .core.services import check_admin_settings, check_webpush_settings
from .middleware import (
AuditMiddleware,
CustomGZipMiddleware,
ExtensionsRedirectMiddleware,
InstalledExtensionMiddleware,
@ -149,6 +153,8 @@ def create_app() -> FastAPI:
CustomGZipMiddleware, minimum_size=1000, exclude_paths=["/api/v1/payments/sse"]
)
app.add_middleware(AuditMiddleware, audit_queue=audit_queue)
# required for SSO login
app.add_middleware(SessionMiddleware, secret_key=settings.auth_secret_key)
@ -414,6 +420,7 @@ def register_async_tasks(app: FastAPI):
if not settings.lnbits_extensions_deactivate_all:
create_task(check_and_register_extensions(app))
create_permanent_task(wait_for_audit_data)
create_permanent_task(check_pending_payments)
create_permanent_task(invoice_listener)
create_permanent_task(internal_invoice_listener)
@ -427,6 +434,7 @@ def register_async_tasks(app: FastAPI):
# TODO: implement watchdog properly
# create_permanent_task(watchdog_task)
create_permanent_task(killswitch_task)
create_permanent_task(purge_audit_data)
# server logs for websocket
if settings.lnbits_admin_ui:

View File

@ -3,6 +3,7 @@ from fastapi import APIRouter, FastAPI
from .db import core_app_extra, db
from .views.admin_api import admin_router
from .views.api import api_router
from .views.audit_api import audit_router
from .views.auth_api import auth_router
from .views.extension_api import extension_router
@ -38,6 +39,7 @@ def init_core_routers(app: FastAPI):
app.include_router(tinyurl_router)
app.include_router(webpush_router)
app.include_router(users_router)
app.include_router(audit_router)
__all__ = ["core_app", "core_app_extra", "db"]

View File

@ -1,3 +1,4 @@
from .audit import create_audit_entry
from .db_versions import (
delete_dbversion,
get_db_version,
@ -82,6 +83,8 @@ from .webpush import (
)
__all__ = [
# audit
"create_audit_entry",
# db_versions
"get_db_version",
"get_db_versions",

84
lnbits/core/crud/audit.py Normal file
View File

@ -0,0 +1,84 @@
from typing import Optional
from lnbits.core.db import db
from lnbits.core.models import AuditEntry, AuditFilters
from lnbits.core.models.audit import AuditCountStat
from lnbits.db import Connection, Filters, Page
async def create_audit_entry(
entry: AuditEntry,
conn: Optional[Connection] = None,
) -> None:
await (conn or db).insert("audit", entry)
async def get_audit_entries(
filters: Optional[Filters[AuditFilters]] = None,
conn: Optional[Connection] = None,
) -> Page[AuditEntry]:
return await (conn or db).fetch_page(
"SELECT * from audit",
[],
{},
filters=filters,
model=AuditEntry,
)
async def delete_expired_audit_entries(
conn: Optional[Connection] = None,
):
await (conn or db).execute(
f"""
DELETE from audit
WHERE delete_at < {db.timestamp_now}
""",
)
async def get_count_stats(
field: str,
filters: Optional[Filters[AuditFilters]] = None,
conn: Optional[Connection] = None,
) -> list[AuditCountStat]:
if field not in ["request_method", "component", "response_code"]:
return []
if not filters:
filters = Filters()
clause = filters.where()
data = await (conn or db).fetchall(
query=f"""
SELECT {field} as field, count({field}) as total
FROM audit
{clause}
GROUP BY {field}
ORDER BY {field}
""",
values=filters.values(),
model=AuditCountStat,
)
return data
async def get_long_duration_stats(
filters: Optional[Filters[AuditFilters]] = None,
conn: Optional[Connection] = None,
) -> list[AuditCountStat]:
if not filters:
filters = Filters()
clause = filters.where()
long_duration_paths = await (conn or db).fetchall(
query=f"""
SELECT path as field, max(duration) as total FROM audit
{clause}
GROUP BY path
ORDER BY total DESC
LIMIT 5
""",
values=filters.values(),
model=AuditCountStat,
)
return long_duration_paths

View File

@ -664,3 +664,23 @@ async def m028_update_settings(db: Connection):
await _insert_key_value(key, value)
await db.execute("drop table settings")
async def m029_create_audit_table(db):
await db.execute(
f"""
CREATE TABLE IF NOT EXISTS audit (
component TEXT,
ip_address TEXT,
user_id TEXT,
path TEXT,
request_type TEXT,
request_method TEXT,
request_details TEXT,
response_code TEXT,
duration REAL NOT NULL,
delete_at TIMESTAMP,
created_at TIMESTAMP NOT NULL DEFAULT {db.timestamp_now}
);
"""
)

View File

@ -1,3 +1,4 @@
from .audit import AuditEntry, AuditFilters
from .lnurl import CreateLnurl, CreateLnurlAuth, PayLnurlWData
from .misc import (
BalanceDelta,
@ -41,6 +42,9 @@ from .wallets import BaseWallet, CreateWallet, KeyType, Wallet, WalletTypeInfo
from .webpush import CreateWebPushSubscription, WebPushSubscription
__all__ = [
# audit
"AuditEntry",
"AuditFilters",
# lnurl
"CreateLnurl",
"CreateLnurlAuth",

View File

@ -0,0 +1,62 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Optional
from pydantic import BaseModel, Field
from lnbits.db import FilterModel
from lnbits.settings import settings
class AuditEntry(BaseModel):
component: Optional[str] = None
ip_address: Optional[str] = None
user_id: Optional[str] = None
path: Optional[str] = None
request_type: Optional[str] = None
request_method: Optional[str] = None
request_details: Optional[str] = None
response_code: Optional[str] = None
duration: float
delete_at: Optional[datetime] = None
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
def __init__(self, **data):
super().__init__(**data)
retention_days = max(0, settings.lnbits_audit_retention_days) or 365
self.delete_at = self.created_at + timedelta(days=retention_days)
class AuditFilters(FilterModel):
__search_fields__ = [
"ip_address",
"user_id",
"path",
"request_method",
"response_code",
"component",
]
__sort_fields__ = [
"created_at",
"duration",
]
ip_address: Optional[str] = None
user_id: Optional[str] = None
path: Optional[str] = None
request_method: Optional[str] = None
response_code: Optional[str] = None
component: Optional[str] = None
class AuditCountStat(BaseModel):
field: str
total: float
class AuditStats(BaseModel):
request_method: list[AuditCountStat] = []
response_code: list[AuditCountStat] = []
component: list[AuditCountStat] = []
long_duration: list[AuditCountStat] = []

View File

@ -5,11 +5,13 @@ import httpx
from loguru import logger
from lnbits.core.crud import (
create_audit_entry,
get_wallet,
get_webpush_subscriptions_for_user,
mark_webhook_sent,
)
from lnbits.core.models import Payment
from lnbits.core.crud.audit import delete_expired_audit_entries
from lnbits.core.models import AuditEntry, Payment
from lnbits.core.services import (
get_balance_delta,
send_payment_notification,
@ -19,6 +21,7 @@ from lnbits.settings import get_funding_source, settings
from lnbits.tasks import send_push_notification
api_invoice_listeners: Dict[str, asyncio.Queue] = {}
audit_queue: asyncio.Queue = asyncio.Queue()
async def killswitch_task():
@ -157,3 +160,31 @@ async def send_payment_push_notification(payment: Payment):
f"https://{subscription.host}/wallet?usr={wallet.user}&wal={wallet.id}"
)
await send_push_notification(subscription, title, body, url)
async def wait_for_audit_data():
"""
Waits for audit entries to be pushed to the queue.
Then it inserts the entries into the DB.
"""
while settings.lnbits_running:
data: AuditEntry = await audit_queue.get()
try:
await create_audit_entry(data)
except Exception as ex:
logger.warning(ex)
await asyncio.sleep(3)
async def purge_audit_data():
"""
Remove audit entries which have passed their retention period.
"""
while settings.lnbits_running:
try:
await delete_expired_audit_entries()
except Exception as ex:
logger.warning(ex)
# clean every hour
await asyncio.sleep(60 * 60)

View File

@ -0,0 +1,212 @@
<q-tab-panel name="audit">
<q-card-section class="q-pa-none">
<h6 class="q-my-none q-mb-sm">Audit</h6>
<div class="row q-mb-lg">
<div class="col-md-6 col-sm-12 q-pr-sm">
<q-item tag="label" v-ripple>
<q-item-section avatar>
<q-toggle
size="md"
v-model="formData.lnbits_audit_enabled"
checked-icon="check"
color="green"
unchecked-icon="clear"
/>
</q-item-section>
<q-item-section>
<q-item-label>Enable audit</q-item-label>
<q-item-label caption
>Record HTTP requests according with the specified
filters</q-item-label
>
</q-item-section>
</q-item>
</div>
<div class="col-md-6 col-sm-12">
<q-input
filled
v-model="formData.lnbits_audit_retention_days"
type="number"
label="Retention days"
hint="Number of days to keep the audit entry."
>
</q-input>
</div>
</div>
<q-separator class="q-mb-lg q-mt-sm"></q-separator>
<div class="row">
<div class="col-md-3 col-sm-12 q-pr-sm">
<q-item tag="label" v-ripple>
<q-item-section avatar>
<q-toggle
size="md"
v-model="formData.lnbits_audit_log_request_body"
checked-icon="check"
color="green"
unchecked-icon="clear"
/>
</q-item-section>
<q-item-section>
<q-item-label>Record Request Body</q-item-label>
<q-item-label caption
>Warning:
<ul>
<li>confidential data (like passwords) will be logged</li>
<li>the request body can have large size.</li>
</ul>
Use it with caution.
</q-item-label>
</q-item-section>
</q-item>
</div>
<div class="col-md-3 col-sm-12 q-pr-sm">
<q-item tag="label" v-ripple>
<q-item-section avatar>
<q-toggle
size="md"
v-model="formData.lnbits_audit_log_ip_address"
checked-icon="check"
color="green"
unchecked-icon="clear"
/>
</q-item-section>
<q-item-section>
<q-item-label>Record IP Address</q-item-label>
<q-item-label caption>Save the client IP address.</q-item-label>
</q-item-section>
</q-item>
</div>
<div class="col-md-3 col-sm-12 q-pr-sm">
<q-item tag="label" v-ripple>
<q-item-section avatar>
<q-toggle
size="md"
v-model="formData.lnbits_audit_log_path_params"
checked-icon="check"
color="green"
unchecked-icon="clear"
/>
</q-item-section>
<q-item-section>
<q-item-label>Record Path Parameters</q-item-label>
<q-item-label caption>Recommended. </q-item-label>
</q-item-section>
</q-item>
</div>
<div class="col-md-3 col-sm-12 q-pr-sm">
<q-item tag="label" v-ripple>
<q-item-section avatar>
<q-toggle
size="md"
v-model="formData.lnbits_audit_log_query_params"
checked-icon="check"
color="green"
unchecked-icon="clear"
/>
</q-item-section>
<q-item-section>
<q-item-label>Record Query Parameters</q-item-label>
<q-item-label caption>Recommended.</q-item-label>
</q-item-section>
</q-item>
</div>
</div>
<q-separator class="q-mb-xl q-mt-sm"></q-separator>
<div class="row q-mb-lg">
<div class="col-md-6 col-sm-12 q-pr-sm">
<p>Include HTTP Methods</p>
<q-select
filled
v-model="formData.lnbits_audit_http_methods"
multiple
hint="List of HTTP methods to be included. Empty lists means all."
label="HTTP Methods"
:options="['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS']"
></q-select>
</div>
<div class="col-md-6 col-sm-12 q-pr-sm">
<p>Include HTTP Response Codes</p>
<q-input
filled
v-model="formAddIncludeResponseCode"
@keydown.enter="addIncludeResponseCode"
type="text"
label="HTTP Response code (regex)"
hint="List of HTTP codes to be included (regex match). Empty lists means all. Eg: 4.*, 5.*"
>
<q-btn @click="addIncludeResponseCode" dense flat icon="add"></q-btn>
</q-input>
<div>
<q-chip
v-for="code in formData.lnbits_audit_http_response_codes"
:key="code"
removable
@remove="removeIncludeResponseCode(code)"
color="primary"
text-color="white"
:label="code"
>
</q-chip>
</div>
</div>
</div>
<div class="row">
<div class="col-md-6 col-sm-12 q-pr-sm">
<p>Include Paths</p>
<q-input
filled
v-model="formAddIncludePath"
@keydown.enter="addIncludePath"
type="text"
label="HTTP Path (regex)"
hint="List of paths to be included (regex match). Empty list means all."
>
<q-btn @click="addIncludePath" dense flat icon="add"></q-btn>
</q-input>
<div>
<q-chip
v-for="path in formData.lnbits_audit_include_paths"
:key="path"
removable
@remove="removeIncludePath(path)"
color="primary"
text-color="white"
:label="path"
>
</q-chip>
</div>
<br />
</div>
<div class="col-md-6 col-sm-12">
<p>Exclude Paths</p>
<q-input
filled
v-model="formAddExcludePath"
@keydown.enter="addExcludePath"
type="text"
label="HTTP Path (regex)"
hint="List of paths to be excluded (regex match). Empty list means none."
>
<q-btn @click="addExcludePath" dense flat icon="add"></q-btn>
</q-input>
<div>
<q-chip
v-for="path in formData.lnbits_audit_exclude_paths"
:key="path"
removable
@remove="removeExcludePath(path)"
color="primary"
text-color="white"
:label="path"
>
</q-chip>
</div>
<br />
<br />
</div>
</div>
</q-card-section>
</q-tab-panel>

View File

@ -101,6 +101,12 @@
:label="$t('notifications')"
@update="val => tab = val.name"
></q-tab>
<q-tab
name="audit"
icon="playlist_add_check_circle"
:label="$t('audit')"
@update="val => tab = val.name"
></q-tab>
<q-tab
name="site_customisation"
icon="language"
@ -125,7 +131,7 @@
{% include "admin/_tab_extensions.html" %} {% include
"admin/_tab_notifications.html" %} {% include
"admin/_tab_security.html" %} {% include "admin/_tab_theme.html"
%}
%}{% include "admin/_tab_audit.html"%}
</q-tab-panels>
</q-form>
</template>

View File

@ -0,0 +1,173 @@
{% extends "base.html" %} {% from "macros.jinja" import window_vars with context
%} {% block page %}
<div class="row q-col-gutter-md justify-center q-mb-xl">
<div class="col-lg-3 col-md-6 col-sm-12 text-center">
<q-card class="q-pt-sm">
<strong>Components</strong>
<div style="width: 250px" class="q-pa-sm">
<canvas ref="componentUseChart"></canvas>
</div>
</q-card>
</div>
<div class="col-lg-3 col-md-6 col-sm-12 text-center">
<q-card class="q-pt-sm">
<strong>To 5 Long Running Endpoints</strong>
<div style="width: 250px; height: 250px" class="q-pa-sm">
<canvas ref="longDurationChart"></canvas>
</div>
</q-card>
</div>
<div class="col-lg-3 col-md-6 col-sm-12 text-center">
<q-card class="q-pt-sm">
<strong>HTTP Request Methods</strong>
<div style="width: 250px; height: 250px" class="q-pa-sm">
<canvas ref="requestMethodChart"></canvas>
</div>
</q-card>
</div>
<div class="col-lg-3 col-md-6 col-sm-12 text-center">
<q-card class="q-pt-sm">
<strong>HTTP Response Codes</strong>
<div style="width: 250px; height: 250px" class="q-pa-sm">
<canvas ref="responseCodeChart"></canvas>
</div>
</q-card>
</div>
</div>
<div class="row q-col-gutter-md justify-center">
<div class="col">
<q-card class="q-pa-md">
<q-table
row-key="id"
:rows="auditEntries"
:columns="auditTable.columns"
v-model:pagination="auditTable.pagination"
:filter="auditTable.search"
:loading="auditTable.loading"
@request="fetchAudit"
>
<template v-slot:header="props">
<q-tr :props="props">
<q-th v-for="col in props.cols" :key="col.name" :props="props">
<q-input
v-if="['ip_address', 'user_id', 'path',].includes(col.name)"
v-model="searchData[col.name]"
@keydown.enter="searchAuditBy()"
@update:model-value="searchAuditBy()"
dense
type="text"
filled
clearable
:label="col.label"
>
<template v-slot:append>
<q-icon
name="search"
@click="searchAuditBy()"
class="cursor-pointer"
/>
</template>
</q-input>
<q-select
v-else-if="['component', 'response_code','request_method'].includes(col.name)"
v-model="searchData[col.name]"
:options="searchOptions[col.name]"
@update:model-value="searchAuditBy()"
:label="col.label"
clearable
style="width: 100px"
></q-select>
<span v-else v-text="col.label"></span>
</q-th>
</q-tr>
</template>
<template v-slot:body="props">
<q-tr auto-width :props="props">
<q-td v-for="col in props.cols" :key="col.name" :props="props">
<div v-if="col.name == 'created_at'">
<q-btn
icon="description"
:disable="!props.row.request_details"
size="sm"
flat
class="cursor-pointer q-mr-xs"
@click="showDetailsDialog(props.row)"
>
<q-tooltip>Request Details</q-tooltip>
</q-btn>
<span v-text="formatDate(props.row.created_at)"></span>
<q-tooltip v-if="props.row.delete_at">
<span
v-text="'Will be deleted at: ' + formatDate(props.row.delete_at)"
></span>
</q-tooltip>
</div>
<div
v-else-if="['user_id', 'request_details'].includes(col.name)"
>
<q-btn
v-if="props.row[col.name]"
icon="content_copy"
size="sm"
flat
class="cursor-pointer q-mr-xs"
@click="copyText(props.row[col.name])"
>
<q-tooltip>Copy</q-tooltip>
</q-btn>
<span v-text="shortify(props.row[col.name])"> </span>
<q-tooltip>
<span v-text="props.row[col.name]"></span>
</q-tooltip>
</div>
<span
v-else
v-text="props.row[col.name]"
@click="searchAuditBy(col.name, props.row[col.name])"
class="cursor-pointer"
></span>
</q-td>
</q-tr>
</template>
</q-table>
</q-card>
</div>
</div>
<q-dialog v-model="auditDetailsDialog.show" position="top">
<q-card class="q-pa-md q-pt-md lnbits__dialog-card">
<strong>HTTP Request Details</strong>
<q-input
filled
dense
v-model.trim="auditDetailsDialog.data"
type="textarea"
rows="25"
></q-input>
<div class="row q-mt-lg">
<q-btn
@click="copyText(auditDetailsDialog.data)"
icon="copy_content"
color="grey"
flat
v-text="$t('copy')"
></q-btn>
<q-btn
v-close-popup
flat
color="grey"
class="q-ml-auto"
v-text="$t('close')"
></q-btn>
</div>
</q-card>
</q-dialog>
{% endblock %} {% block scripts %} {{ window_vars(user) }}
<script src="{{ static_url_for('static', 'js/audit.js') }}"></script>
{% endblock %}

View File

@ -0,0 +1,49 @@
from fastapi import APIRouter, Depends
from lnbits.core.crud.audit import (
get_audit_entries,
get_count_stats,
get_long_duration_stats,
)
from lnbits.core.models import AuditEntry, AuditFilters
from lnbits.core.models.audit import AuditStats
from lnbits.db import Filters, Page
from lnbits.decorators import check_admin, parse_filters
from lnbits.helpers import generate_filter_params_openapi
audit_router = APIRouter(
prefix="/audit/api/v1", dependencies=[Depends(check_admin)], tags=["Audit"]
)
@audit_router.get(
"",
name="Get audit entries",
summary="Get paginated list audit entries",
openapi_extra=generate_filter_params_openapi(AuditFilters),
)
async def api_get_audit(
filters: Filters = Depends(parse_filters(AuditFilters)),
) -> Page[AuditEntry]:
return await get_audit_entries(filters)
@audit_router.get(
"/stats",
name="Get audit entries",
summary="Get paginated list audit entries",
openapi_extra=generate_filter_params_openapi(AuditFilters),
)
async def api_get_audit_stats(
filters: Filters = Depends(parse_filters(AuditFilters)),
) -> AuditStats:
request_mothod_stats = await get_count_stats("request_method", filters)
response_code_stats = await get_count_stats("response_code", filters)
components_stats = await get_count_stats("component", filters)
long_duration_stats = await get_long_duration_stats(filters)
return AuditStats(
request_method=request_mothod_stats,
response_code=response_code_stats,
component=components_stats,
long_duration=long_duration_stats,
)

View File

@ -385,6 +385,20 @@ async def users_index(request: Request, user: User = Depends(check_admin)):
)
@generic_router.get("/audit", response_class=HTMLResponse)
async def audit_index(request: Request, user: User = Depends(check_admin)):
if not settings.lnbits_audit_enabled:
raise HTTPException(HTTPStatus.NOT_FOUND, "Audit not enabled")
return template_renderer().TemplateResponse(
"audit/index.html",
{
"request": request,
"user": user.json(),
},
)
@generic_router.get("/uuidv4/{hex_value}")
async def hex_to_uuid4(hex_value: str):
try:

View File

@ -90,6 +90,7 @@ class KeyChecker(SecurityBase):
detail="Wallet not found.",
)
request.scope["user_id"] = wallet.user
if self.expected_key_type is KeyType.admin and wallet.adminkey != key_value:
raise HTTPException(
status_code=HTTPStatus.UNAUTHORIZED,
@ -148,6 +149,7 @@ async def check_user_exists(
if not account:
raise HTTPException(HTTPStatus.UNAUTHORIZED, "User not found.")
r.scope["user_id"] = account.id
if not settings.is_user_allowed(account.id):
raise HTTPException(HTTPStatus.UNAUTHORIZED, "User not allowed.")

View File

@ -86,6 +86,8 @@ def template_renderer(additional_folders: Optional[list] = None) -> Jinja2Templa
t.env.globals["LNBITS_EXTENSIONS_DEACTIVATE_ALL"] = (
settings.lnbits_extensions_deactivate_all
)
t.env.globals["LNBITS_AUDIT_ENABLED"] = settings.lnbits_audit_enabled
t.env.globals["LNBITS_SERVICE_FEE"] = settings.lnbits_service_fee
t.env.globals["LNBITS_SERVICE_FEE_MAX"] = settings.lnbits_service_fee_max
t.env.globals["LNBITS_SERVICE_FEE_WALLET"] = settings.lnbits_service_fee_wallet

View File

@ -1,15 +1,21 @@
import asyncio
import json
from datetime import datetime, timezone
from http import HTTPStatus
from typing import Any, List, Union
from typing import Any, List, Optional, Union
from fastapi import FastAPI, Request
from fastapi import FastAPI, Request, Response
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
from loguru import logger
from slowapi import _rate_limit_exceeded_handler
from slowapi.errors import RateLimitExceeded
from slowapi.middleware import SlowAPIMiddleware
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.middleware.gzip import GZipMiddleware
from starlette.types import ASGIApp, Receive, Scope, Send
from lnbits.core.db import core_app_extra
from lnbits.core.models import AuditEntry
from lnbits.helpers import template_renderer
from lnbits.settings import settings
@ -120,6 +126,94 @@ class ExtensionsRedirectMiddleware:
await self.app(scope, receive, send)
class AuditMiddleware(BaseHTTPMiddleware):
def __init__(self, app: ASGIApp, audit_queue: asyncio.Queue) -> None:
super().__init__(app)
self.audit_queue = audit_queue
# delete_time purge after X days
# time, # include pats, exclude paths (regex)
async def dispatch(self, request: Request, call_next) -> Response:
start_time = datetime.now(timezone.utc)
request_details = await self._request_details(request)
response: Optional[Response] = None
try:
response = await call_next(request)
assert response
return response
finally:
duration = (datetime.now(timezone.utc) - start_time).total_seconds()
await self._log_audit(request, response, duration, request_details)
async def _log_audit(
self,
request: Request,
response: Optional[Response],
duration: float,
request_details: Optional[str],
):
try:
http_method = request.scope.get("method", None)
path: Optional[str] = getattr(request.scope.get("route", {}), "path", None)
response_code = str(response.status_code) if response else None
if not settings.audit_http_request(http_method, path, response_code):
return None
ip_address = (
request.client.host
if settings.lnbits_audit_log_ip_address and request.client
else None
)
user_id = request.scope.get("user_id", None)
if settings.is_super_user(user_id):
user_id = "super_user"
component = "core"
if path and not path.startswith("/api"):
component = path.split("/")[1]
data = AuditEntry(
component=component,
ip_address=ip_address,
user_id=user_id,
path=path,
request_type=request.scope.get("type", None),
request_method=http_method,
request_details=request_details,
response_code=response_code,
duration=duration,
)
await self.audit_queue.put(data)
except Exception as ex:
logger.warning(ex)
async def _request_details(self, request: Request) -> Optional[str]:
if not settings.audit_http_request_details():
return None
try:
http_method = request.scope.get("method", None)
path = request.scope.get("path", None)
if not settings.audit_http_request(http_method, path):
return None
details: dict = {}
if settings.lnbits_audit_log_path_params:
details["path_params"] = request.path_params
if settings.lnbits_audit_log_query_params:
details["query_params"] = dict(request.query_params)
if settings.lnbits_audit_log_request_body:
_body = await request.body()
details["body"] = _body.decode("utf-8")
details_str = json.dumps(details)
# Make sure the super_user id is not leaked
return details_str.replace(settings.super_user, "super_user")
except Exception as e:
logger.warning(e)
return None
def add_ratelimit_middleware(app: FastAPI):
core_app_extra.register_new_ratelimiter()
# latest https://slowapi.readthedocs.io/en/latest/

View File

@ -4,6 +4,7 @@ import importlib
import importlib.metadata
import inspect
import json
import re
from enum import Enum
from hashlib import sha256
from os import path
@ -509,6 +510,92 @@ class KeycloakAuthSettings(LNbitsSettings):
keycloak_client_secret: str = Field(default="")
class AuditSettings(LNbitsSettings):
lnbits_audit_enabled: bool = Field(default=True)
# number of days to keep the audit entry
lnbits_audit_retention_days: int = Field(default=7)
lnbits_audit_log_ip_address: bool = Field(default=False)
lnbits_audit_log_path_params: bool = Field(default=True)
lnbits_audit_log_query_params: bool = Field(default=True)
lnbits_audit_log_request_body: bool = Field(default=False)
# List of paths to be included (regex match). Empty list means all.
lnbits_audit_include_paths: list[str] = Field(default=[".*api/v1/.*"])
# List of paths to be excluded (regex match). Empty list means none.
lnbits_audit_exclude_paths: list[str] = Field(default=["/static"])
# List of HTTP methods to be included. Empty lists means all.
# Options (case-sensitive): GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS
lnbits_audit_http_methods: list[str] = Field(
default=["POST", "PUT", "PATCH", "DELETE"]
)
# List of HTTP codes to be included (regex match). Empty lists means all.
lnbits_audit_http_response_codes: list[str] = Field(default=["4.*", "5.*"])
def audit_http_request_details(self) -> bool:
return (
self.lnbits_audit_log_path_params
or self.lnbits_audit_log_query_params
or self.lnbits_audit_log_request_body
)
def audit_http_request(
self,
http_method: Optional[str] = None,
path: Optional[str] = None,
http_response_code: Optional[str] = None,
) -> bool:
if not self.lnbits_audit_enabled:
return False
if len(self.lnbits_audit_http_methods) != 0:
if not http_method:
return False
if http_method not in self.lnbits_audit_http_methods:
return False
if not self._is_http_request_path_auditable(path):
return False
if not self._is_http_response_code_auditable(http_response_code):
return False
return True
def _is_http_request_path_auditable(self, path: Optional[str]):
if len(self.lnbits_audit_exclude_paths) != 0 and path:
for exclude_path in self.lnbits_audit_exclude_paths:
if _re_fullmatch_safe(exclude_path, path):
return False
if len(self.lnbits_audit_include_paths) != 0:
if not path:
return False
for include_path in self.lnbits_audit_include_paths:
if _re_fullmatch_safe(include_path, path):
return True
return False
def _is_http_response_code_auditable(
self, http_response_code: Optional[str]
) -> bool:
if not http_response_code:
# No response code means only request filters should apply
return True
if len(self.lnbits_audit_http_response_codes) == 0:
return True
for response_code in self.lnbits_audit_http_response_codes:
if _re_fullmatch_safe(response_code, http_response_code):
return True
return False
class EditableSettings(
UsersSettings,
ExtensionsSettings,
@ -520,6 +607,7 @@ class EditableSettings(
LightningSettings,
WebPushSettings,
NodeUISettings,
AuditSettings,
AuthSettings,
NostrAuthSettings,
GoogleAuthSettings,
@ -674,7 +762,7 @@ class Settings(EditableSettings, ReadOnlySettings, TransientSettings, BaseSettin
or user_id == self.super_user
)
def is_super_user(self, user_id: str) -> bool:
def is_super_user(self, user_id: Optional[str] = None) -> bool:
return user_id == self.super_user
def is_admin_user(self, user_id: str) -> bool:
@ -702,6 +790,14 @@ class SettingsField(BaseModel):
tag: str = "core"
def _re_fullmatch_safe(pattern: str, string: str):
try:
return re.fullmatch(pattern, string) is not None
except Exception as _:
logger.warning(f"Regex error for pattern {pattern}")
return False
def set_cli_settings(**kwargs):
for key, value in kwargs.items():
setattr(settings, key, value)

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -5,6 +5,7 @@ window.localisation.en = {
site_customisation: 'Site Customisation',
funding: 'Funding',
users: 'Users',
audit: 'Audit',
apps: 'Apps',
channels: 'Channels',
transactions: 'Transactions',

View File

@ -42,6 +42,9 @@ window.app = Vue.createApp({
formAllowedIPs: '',
formBlockedIPs: '',
nostrAcceptedUrl: '',
formAddIncludePath: '',
formAddExcludePath: '',
formAddIncludeResponseCode: '',
isSuperUser: false,
wallet: {},
cancel: {},
@ -103,6 +106,58 @@ window.app = Vue.createApp({
let allowed_users = this.formData.lnbits_allowed_users
this.formData.lnbits_allowed_users = allowed_users.filter(u => u !== user)
},
addIncludePath() {
if (!this.formAddIncludePath) {
return
}
const paths = this.formData.lnbits_audit_include_paths
if (!paths.includes(this.formAddIncludePath)) {
this.formData.lnbits_audit_include_paths = [
...paths,
this.formAddIncludePath
]
}
this.formAddIncludePath = ''
},
removeIncludePath(path) {
this.formData.lnbits_audit_include_paths =
this.formData.lnbits_audit_include_paths.filter(p => p !== path)
},
addExcludePath() {
if (!this.formAddExcludePath) {
return
}
const paths = this.formData.lnbits_audit_exclude_paths
if (!paths.includes(this.formAddExcludePath)) {
this.formData.lnbits_audit_exclude_paths = [
...paths,
this.formAddExcludePath
]
}
this.formAddExcludePath = ''
},
removeExcludePath(path) {
this.formData.lnbits_audit_exclude_paths =
this.formData.lnbits_audit_exclude_paths.filter(p => p !== path)
},
addIncludeResponseCode() {
if (!this.formAddIncludeResponseCode) {
return
}
const codes = this.formData.lnbits_audit_http_response_codes
if (!codes.includes(this.formAddIncludeResponseCode)) {
this.formData.lnbits_audit_http_response_codes = [
...codes,
this.formAddIncludeResponseCode
]
}
this.formAddIncludeResponseCode = ''
},
removeIncludeResponseCode(code) {
this.formData.lnbits_audit_http_response_codes =
this.formData.lnbits_audit_http_response_codes.filter(c => c !== code)
},
addExtensionsManifest() {
const addManifest = this.formAddExtensionsManifest.trim()
const manifests = this.formData.lnbits_extensions_manifests

386
lnbits/static/js/audit.js Normal file
View File

@ -0,0 +1,386 @@
window.app = Vue.createApp({
el: '#vue',
mixins: [window.windowMixin],
data: function () {
return {
auditEntries: [],
searchData: {
user_id: '',
ip_address: '',
request_type: '',
component: '',
request_method: '',
response_code: '',
path: ''
},
searchOptions: {
component: [],
request_method: [],
response_code: []
},
auditTable: {
columns: [
{
name: 'created_at',
align: 'center',
label: 'Date',
field: 'created_at',
sortable: true
},
{
name: 'duration',
align: 'left',
label: 'Duration (sec)',
field: 'duration',
sortable: true
},
{
name: 'component',
align: 'left',
label: 'Component',
field: 'component',
sortable: false
},
{
name: 'request_method',
align: 'left',
label: 'Method',
field: 'request_method',
sortable: false
},
{
name: 'response_code',
align: 'left',
label: 'Code',
field: 'response_code',
sortable: false
},
{
name: 'user_id',
align: 'left',
label: 'User Id',
field: 'user_id',
sortable: false
},
{
name: 'ip_address',
align: 'left',
label: 'IP Address',
field: 'ip_address',
sortable: false
},
{
name: 'path',
align: 'left',
label: 'Path',
field: 'path',
sortable: false
}
],
pagination: {
sortBy: 'created_at',
rowsPerPage: 10,
page: 1,
descending: true,
rowsNumber: 10
},
search: null,
hideEmpty: true,
loading: false
},
auditDetailsDialog: {
data: null,
show: false
}
}
},
async created() {
await this.fetchAudit()
},
mounted() {
this.initCharts()
},
methods: {
async fetchAudit(props) {
try {
const params = LNbits.utils.prepareFilterQuery(this.auditTable, props)
const {data} = await LNbits.api.request(
'GET',
`/audit/api/v1?${params}`
)
this.auditTable.pagination.rowsNumber = data.total
this.auditEntries = data.data
await this.fetchAuditStats(props)
} catch (error) {
console.warn(error)
LNbits.utils.notifyApiError(error)
} finally {
this.auditTable.loading = false
}
},
async fetchAuditStats(props) {
try {
const params = LNbits.utils.prepareFilterQuery(this.auditTable, props)
const {data} = await LNbits.api.request(
'GET',
`/audit/api/v1/stats?${params}`
)
const request_methods = data.request_method.map(rm => rm.field)
this.searchOptions.request_method = [
...new Set(this.searchOptions.request_method.concat(request_methods))
]
this.requestMethodChart.data.labels = request_methods
this.requestMethodChart.data.datasets[0].data = data.request_method.map(
rm => rm.total
)
this.requestMethodChart.update()
const response_codes = data.response_code.map(rm => rm.field)
this.searchOptions.response_code = [
...new Set(this.searchOptions.response_code.concat(response_codes))
]
this.responseCodeChart.data.labels = response_codes
this.responseCodeChart.data.datasets[0].data = data.response_code.map(
rm => rm.total
)
this.responseCodeChart.update()
const components = data.component.map(rm => rm.field)
this.searchOptions.component = [
...new Set(this.searchOptions.component.concat(components))
]
this.componentUseChart.data.labels = components
this.componentUseChart.data.datasets[0].data = data.component.map(
rm => rm.total
)
this.componentUseChart.update()
this.longDurationChart.data.labels = data.long_duration.map(
rm => rm.field
)
this.longDurationChart.data.datasets[0].data = data.long_duration.map(
rm => rm.total
)
this.longDurationChart.update()
} catch (error) {
console.warn(error)
LNbits.utils.notifyApiError(error)
}
},
async searchAuditBy(fieldName, fieldValue) {
if (fieldName) {
this.searchData[fieldName] = fieldValue
}
// remove empty fields
this.auditTable.filter = Object.entries(this.searchData).reduce(
(a, [k, v]) => (v ? ((a[k] = v), a) : a),
{}
)
await this.fetchAudit()
},
showDetailsDialog(entry) {
const details = JSON.parse(entry?.request_details || '')
try {
if (details.body) {
details.body = JSON.parse(details.body)
}
} catch (e) {
// do nothing
}
this.auditDetailsDialog.data = JSON.stringify(details, null, 4)
this.auditDetailsDialog.show = true
},
formatDate: function (value) {
return Quasar.date.formatDate(new Date(value), 'YYYY-MM-DD HH:mm')
},
shortify(value) {
valueLength = (value || '').length
if (valueLength <= 10) {
return value
}
return `${value.substring(0, 5)}...${value.substring(valueLength - 5, valueLength)}`
},
initCharts() {
this.responseCodeChart = new Chart(
this.$refs.responseCodeChart.getContext('2d'),
{
type: 'doughnut',
options: {
responsive: true,
plugins: {
legend: {
position: 'bottom'
},
title: {
display: false,
text: 'HTTP Response Codes'
}
},
onClick: (_, elements, chart) => {
if (elements[0]) {
const i = elements[0].index
this.searchAuditBy('response_code', chart.data.labels[i])
}
}
},
data: {
datasets: [
{
label: '',
data: [20, 10],
backgroundColor: [
'rgb(100, 99, 200)',
'rgb(54, 162, 235)',
'rgb(255, 205, 86)',
'rgb(255, 5, 86)',
'rgb(25, 205, 86)',
'rgb(255, 205, 250)'
]
}
],
labels: []
}
}
)
this.requestMethodChart = new Chart(
this.$refs.requestMethodChart.getContext('2d'),
{
type: 'bar',
options: {
responsive: true,
maintainAspectRatio: false,
plugins: {
title: {
display: false
}
},
onClick: (_, elements, chart) => {
if (elements[0]) {
const i = elements[0].index
this.searchAuditBy('request_method', chart.data.labels[i])
}
}
},
data: {
datasets: [
{
label: '',
data: [],
backgroundColor: [
'rgb(255, 99, 132)',
'rgb(54, 162, 235)',
'rgb(255, 205, 86)',
'rgb(255, 5, 86)',
'rgb(25, 205, 86)',
'rgb(255, 205, 250)'
],
hoverOffset: 4
}
]
}
}
)
this.componentUseChart = new Chart(
this.$refs.componentUseChart.getContext('2d'),
{
type: 'pie',
options: {
responsive: true,
plugins: {
legend: {
position: 'xxx'
},
title: {
display: false,
text: 'Components'
}
},
onClick: (_, elements, chart) => {
if (elements[0]) {
const i = elements[0].index
this.searchAuditBy('component', chart.data.labels[i])
}
}
},
data: {
datasets: [
{
data: [],
backgroundColor: [
'rgb(255, 99, 132)',
'rgb(54, 162, 235)',
'rgb(255, 205, 86)',
'rgb(255, 5, 86)',
'rgb(25, 205, 86)',
'rgb(255, 205, 250)',
'rgb(100, 205, 250)',
'rgb(120, 205, 250)',
'rgb(140, 205, 250)',
'rgb(160, 205, 250)'
],
hoverOffset: 4
}
]
}
}
)
this.longDurationChart = new Chart(
this.$refs.longDurationChart.getContext('2d'),
{
type: 'bar',
options: {
responsive: true,
indexAxis: 'y',
maintainAspectRatio: false,
plugins: {
legend: {
title: {
display: false,
text: 'Long Duration'
}
}
},
onClick: (_, elements, chart) => {
if (elements[0]) {
const i = elements[0].index
this.searchAuditBy('path', chart.data.labels[i])
}
}
},
data: {
datasets: [
{
label: '',
data: [],
backgroundColor: [
'rgb(255, 99, 132)',
'rgb(54, 162, 235)',
'rgb(255, 205, 86)',
'rgb(255, 5, 86)',
'rgb(25, 205, 86)',
'rgb(255, 205, 250)',
'rgb(100, 205, 250)',
'rgb(120, 205, 250)',
'rgb(140, 205, 250)',
'rgb(160, 205, 250)'
],
hoverOffset: 4
}
]
}
}
)
}
}
})

View File

@ -125,7 +125,7 @@ window.app.component('lnbits-extension-list', {
window.app.component('lnbits-manage', {
template: '#lnbits-manage',
props: ['showAdmin', 'showNode', 'showExtensions', 'showUsers'],
props: ['showAdmin', 'showNode', 'showExtensions', 'showUsers', 'showAudit'],
methods: {
isActive: function (path) {
return window.location.pathname === path

View File

@ -163,6 +163,7 @@
<lnbits-manage
:show-admin="'{{LNBITS_ADMIN_UI}}' == 'True'"
:show-users="'{{LNBITS_ADMIN_UI}}' == 'True'"
:show-audit="'{{LNBITS_AUDIT_ENABLED}}' == 'True'"
:show-node="'{{LNBITS_NODE_UI}}' == 'True'"
:show-extensions="'{{LNBITS_EXTENSIONS_DEACTIVATE_ALL}}' == 'False'"
></lnbits-manage>

View File

@ -195,6 +195,24 @@
<q-item-label lines="1" v-text="$t('users')"></q-item-label>
</q-item-section>
</q-item>
<q-item
v-if="showAudit"
clickable
tag="a"
href="/audit"
:active="isActive('/audit')"
>
<q-item-section side>
<q-icon
name="playlist_add_check_circle"
:color="isActive('/audit') ? 'primary' : 'grey-5'"
size="md"
></q-icon>
</q-item-section>
<q-item-section>
<q-item-label lines="1" v-text="$t('audit')"></q-item-label>
</q-item-section>
</q-item>
</div>
<q-item
v-if="showExtensions"