refactor: make settings key-value in DB (#2766)

Makefile

@@ -78,6 +78,7 @@ test-migration:
 	HOST=0.0.0.0 \
 	PORT=5002 \
 	LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \
+	LNBITS_ADMIN_UI=False \
 	timeout 5s poetry run lnbits --host 0.0.0.0 --port 5002 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
 	LNBITS_DATA_FOLDER="./tests/data" \
 	LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \

lnbits/core/crud/settings.py

@@ -1,21 +1,25 @@
 import json
-from typing import Optional
+from typing import Any, Optional
+
+from loguru import logger
 
 from lnbits.core.db import db
 from lnbits.settings import (
     AdminSettings,
     EditableSettings,
+    SettingsField,
     SuperSettings,
     settings,
 )
 
 
 async def get_super_settings() -> Optional[SuperSettings]:
-    row: dict = await db.fetchone("SELECT * FROM settings")
-    if not row:
-        return None
-    editable_settings = json.loads(row["editable_settings"])
-    return SuperSettings(**{"super_user": row["super_user"], **editable_settings})
+    data = await get_settings_by_tag("core")
+    if data:
+        super_user = await get_settings_field("super_user")
+        super_user_id = super_user.value if super_user else None
+        return SuperSettings(**{"super_user": super_user_id, **data})
+    return None
 
 
 async def get_admin_settings(is_super_user: bool = False) -> Optional[AdminSettings]:
@@ -34,38 +38,83 @@ async def get_admin_settings(is_super_user: bool = False) -> Optional[AdminSetti
     return admin_settings
 
 
-async def delete_admin_settings() -> None:
-    await db.execute("DELETE FROM settings")
-
-
-async def update_admin_settings(data: EditableSettings) -> None:
-    row: dict = await db.fetchone("SELECT editable_settings FROM settings")
-    editable_settings = json.loads(row["editable_settings"]) if row else {}
+async def update_admin_settings(
+    data: EditableSettings, tag: Optional[str] = "core"
+) -> None:
+    editable_settings = await get_settings_by_tag("core") or {}
     editable_settings.update(data.dict(exclude_unset=True))
-    await db.execute(
-        "UPDATE settings SET editable_settings = :settings",
-        {"settings": json.dumps(editable_settings)},
-    )
+    for key, value in editable_settings.items():
+        try:
+            await set_settings_field(key, value, tag)
+        except Exception as exc:
+            logger.warning(exc)
+            logger.warning(f"Failed to update settings for '{tag}.{key}'.")
 
 
 async def update_super_user(super_user: str) -> SuperSettings:
-    await db.execute(
-        "UPDATE settings SET super_user = :user",
-        {"user": super_user},
-    )
+    await set_settings_field("super_user", super_user)
     settings = await get_super_settings()
     assert settings, "updated super_user settings could not be retrieved"
     return settings
 
 
-async def create_admin_settings(super_user: str, new_settings: dict):
-    await db.execute(
-        """
-        INSERT INTO settings (super_user, editable_settings)
-        VALUES (:user, :settings)
-        """,
-        {"user": super_user, "settings": json.dumps(new_settings)},
-    )
+async def delete_admin_settings(tag: Optional[str] = "core") -> None:
+    await db.execute("DELETE FROM settings WHERE tag = :tag", {"tag": tag})
+
+
+async def create_admin_settings(super_user: str, new_settings: dict) -> SuperSettings:
+    data = {"super_user": super_user, **new_settings}
+    for key, value in data.items():
+        await set_settings_field(key, value)
+
     settings = await get_super_settings()
     assert settings, "created admin settings could not be retrieved"
     return settings
+
+
+async def get_settings_field(
+    id_: str, tag: Optional[str] = "core"
+) -> Optional[SettingsField]:
+
+    row: dict = await db.fetchone(
+        """
+            SELECT * FROM system_settings
+            WHERE  id = :id AND tag = :tag
+        """,
+        {"id": id_, "tag": tag},
+    )
+    if not row:
+        return None
+    return SettingsField(id=row["id"], value=json.loads(row["value"]), tag=row["tag"])
+
+
+async def set_settings_field(
+    id_: str, value: Optional[Any], tag: Optional[str] = "core"
+):
+    value = json.dumps(value) if value is not None else None
+    await db.execute(
+        """
+        INSERT INTO system_settings (id, value, tag)
+        VALUES (:id, :value, :tag)
+        ON CONFLICT (id, tag) DO UPDATE SET value = :value
+        """,
+        {"id": id_, "value": value, "tag": tag or "core"},
+    )
+
+
+async def get_settings_by_tag(tag: str) -> Optional[dict[str, Any]]:
+    rows: list[dict] = await db.fetchall(
+        "SELECT * FROM system_settings WHERE tag = :tag", {"tag": tag}
+    )
+    if len(rows) == 0:
+        return None
+    data: dict[str, Any] = {}
+    for row in rows:
+        try:
+            data[row["id"]] = json.loads(row["value"]) if row["value"] else None
+        except Exception as _:
+            logger.warning(
+                f"""Failed to load settings value for '{tag}.{row["id"]}'."""
+            )
+    data.pop("super_user")
+    return data

lnbits/core/migrations.py

@@ -1,5 +1,6 @@
 import json
 from time import time
+from typing import Any
 
 from loguru import logger
 from sqlalchemy.exc import OperationalError
@@ -629,3 +630,37 @@ async def m027_update_apipayments_data(db: Connection):
                 "checking_id": payment.get("checking_id"),
             },
         )
+
+
+async def m028_update_settings(db: Connection):
+
+    await db.execute(
+        """
+        CREATE TABLE IF NOT EXISTS system_settings (
+            id TEXT PRIMARY KEY,
+            value TEXT,
+            tag TEXT NOT NULL DEFAULT 'core',
+
+            UNIQUE (id, tag)
+        );
+    """
+    )
+
+    async def _insert_key_value(id_: str, value: Any):
+        await db.execute(
+            """
+            INSERT INTO system_settings (id, value, tag)
+            VALUES (:id, :value, :tag)
+            """,
+            {"id": id_, "value": json.dumps(value), "tag": "core"},
+        )
+
+    row: dict = await db.fetchone("SELECT * FROM settings")
+    if row:
+        await _insert_key_value("super_user", row["super_user"])
+        editable_settings = json.loads(row["editable_settings"])
+
+        for key, value in editable_settings.items():
+            await _insert_key_value(key, value)
+
+    await db.execute("drop table settings")

lnbits/core/services/settings.py

@@ -29,7 +29,8 @@ async def check_webpush_settings():
             "lnbits_webpush_pubkey": pubkey,
         }
         update_cached_settings(push_settings)
-        await update_admin_settings(EditableSettings(**push_settings))
+        if settings.lnbits_admin_ui:
+            await update_admin_settings(EditableSettings(**push_settings))
 
     logger.info("Initialized webpush settings with generated VAPID key pair.")
     logger.info(f"Pubkey: {settings.lnbits_webpush_pubkey}")

lnbits/settings.py

@@ -684,6 +684,12 @@ class AdminSettings(EditableSettings):
     lnbits_allowed_funding_sources: Optional[list[str]]
 
 
+class SettingsField(BaseModel):
+    id: str
+    value: Optional[Any]
+    tag: str = "core"
+
+
 def set_cli_settings(**kwargs):
     for key, value in kwargs.items():
         setattr(settings, key, value)