feat: migrate in chunks (#2936)

lnbits/core/migrations.py

@@ -602,20 +602,30 @@ async def m026_update_payment_table(db: Connection):
 async def m027_update_apipayments_data(db: Connection):
     result = None
     try:
-        result = await db.execute("SELECT * FROM apipayments")
+        result = await db.execute("SELECT * FROM apipayments LIMIT 100")
     except Exception as exc:
         logger.warning("Could not select, trying again after cache cleared.")
         logger.debug(exc)
         await db.execute("COMMIT")
 
-    result = await db.execute("SELECT * FROM apipayments")
+    offset = 0
+    limit = 1000
+    payments: list[dict[Any, Any]] = []
+    logger.info("Updating payments")
+    while len(payments) > 0 or offset == 0:
+        logger.info(f"Updating {offset} to {offset+limit}")
 
+        result = await db.execute(
+            f"SELECT * FROM apipayments  ORDER BY time LIMIT {limit} OFFSET {offset}"
+        )
         payments = result.mappings().all()
+        logger.info(f"Payments count: {len(payments)}")
+
         for payment in payments:
             tag = None
             created_at = payment.get("time")
             if payment.get("extra"):
-            extra = json.loads(payment.get("extra"))
+                extra = json.loads(str(payment.get("extra")))
                 tag = extra.get("tag")
             tsph = db.timestamp_placeholder("created_at")
             await db.execute(
@@ -630,6 +640,8 @@ async def m027_update_apipayments_data(db: Connection):
                     "checking_id": payment.get("checking_id"),
                 },
             )
+        offset += limit
+    logger.info("Payments updated")
 
 
 async def m028_update_settings(db: Connection):