catch everything and restart in main background tasks.

lnbits/app.py

@@ -18,11 +18,12 @@ from .helpers import (
 )
 from .proxy_fix import ASGIProxyFix
 from .tasks import (
+    webhook_handler,
+    invoice_listener,
     run_deferred_async,
     check_pending_payments,
-    invoice_listener,
     internal_invoice_listener,
-    webhook_handler,
+    catch_everything_and_restart,
 )
 from .settings import WALLET
 
@@ -118,10 +119,10 @@ def register_async_tasks(app):
 
     @app.before_serving
     async def listeners():
-        run_deferred_async(app.nursery)
-        app.nursery.start_soon(check_pending_payments)
-        app.nursery.start_soon(invoice_listener, app.nursery)
-        app.nursery.start_soon(internal_invoice_listener, app.nursery)
+        run_deferred_async()
+        app.nursery.start_soon(catch_everything_and_restart, check_pending_payments)
+        app.nursery.start_soon(catch_everything_and_restart, invoice_listener)
+        app.nursery.start_soon(catch_everything_and_restart, internal_invoice_listener)
 
     @app.after_serving
     async def stop_listeners():

lnbits/tasks.py

@@ -1,8 +1,9 @@
 import time
 import trio
+import traceback
 from http import HTTPStatus
-from typing import Optional, List, Callable
-from quart_trio import QuartTrio
+from quart import current_app
+from typing import List, Callable
 
 from lnbits.settings import WALLET
 from lnbits.core.crud import (
@@ -24,9 +25,22 @@ def record_async(func: Callable) -> Callable:
     return recorder
 
 
-def run_deferred_async(nursery):
+def run_deferred_async():
     for func in deferred_async:
-        nursery.start_soon(func)
+        current_app.nursery.start_soon(catch_everything_and_restart, func)
+
+
+async def catch_everything_and_restart(func):
+    try:
+        await func()
+    except trio.Cancelled:
+        raise  # because we must pass this up
+    except Exception as exc:
+        print("caught exception in background task:", exc)
+        print(traceback.format_exc())
+        print("will restart the task in 5 seconds.")
+        await trio.sleep(5)
+        await catch_everything_and_restart(func)
 
 
 async def send_push_promise(a, b) -> None:
@@ -54,14 +68,14 @@ async def webhook_handler():
 internal_invoice_paid, internal_invoice_received = trio.open_memory_channel(0)
 
 
-async def internal_invoice_listener(nursery):
+async def internal_invoice_listener():
     async for checking_id in internal_invoice_received:
-        nursery.start_soon(invoice_callback_dispatcher, checking_id)
+        current_app.nursery.start_soon(invoice_callback_dispatcher, checking_id)
 
 
-async def invoice_listener(nursery):
+async def invoice_listener():
     async for checking_id in WALLET.paid_invoices_stream():
-        nursery.start_soon(invoice_callback_dispatcher, checking_id)
+        current_app.nursery.start_soon(invoice_callback_dispatcher, checking_id)
 
 
 async def check_pending_payments():