New: Compact the LND channel.db on-demand from REPAIR and before backups (#2761)

* add lnd.compact.sh
* add interactive channel.db compacting to backups
* improve text output
* compact: restart LND after manual compacting

CHANGES.md

@@ -3,6 +3,7 @@
 ## What's new in Version 1.7.2 of RaspiBlitz?
 
 - New: Verify git commits and tags everywhere possible [issue](https://github.com/rootzoll/raspiblitz/issues/2686)
+- New: Compact the LND channel.db on-demand from REPAIR and before backups [issue](https://github.com/rootzoll/raspiblitz/issues/2752)
 - Update: C-lightning v0.10.2 [details](https://github.com/ElementsProject/lightning/releases/tag/v0.10.2)
 - Update: Lightning Terminal v0.6.0-alpha with Lightning Node Connect over Tor [details](https://github.com/lightninglabs/lightning-terminal/releases/tag/v0.6.0-alpha)
 - Update: BTCPayServer v1.3.3 with UPDATE option [details](https://github.com/btcpayserver/btcpayserver/releases/tag/v1.3.3)

home.admin/98repairBlockchain.sh

@@ -33,6 +33,7 @@ elif [ "${CHOICE}" = "REINDEX" ]; then
     sudo /home/admin/config.scripts/network.reindex.sh
 
 elif [ "${CHOICE}" = "BACKUP" ]; then
+    /home/admin/config.scripts/lnd.compact.sh interactive
     sudo /home/admin/config.scripts/lnd.backup.sh lnd-export-gui
     echo "PRESS ENTER to continue."
     read key

home.admin/98repairMenu.sh

@@ -20,6 +20,7 @@ Download LND Data Backup now?
       echo "*************************************"
       echo "please wait .."
       sleep 2
+      /home/admin/config.scripts/lnd.compact.sh interactive
       /home/admin/config.scripts/lnd.backup.sh lnd-export-gui
       echo
       echo "PRESS ENTER to continue once you are done downloading."
@@ -52,6 +53,7 @@ OPTIONS+=(SOFTWARE "Run Softwaretest (DebugReport)")
 if [ "${lightning}" == "lnd" ] || [ "${lnd}" == "on" ]; then
   OPTIONS+=(BACKUP-LND "Backup your LND data (Rescue-File)")
   OPTIONS+=(RESET-LND "Delete LND & start new node/wallet")
+  OPTIONS+=(COMPACT "Compact the LND channel.db")
 fi
 if [ "${lightning}" == "cl" ] || [ "${cl}" == "on" ]; then
   OPTIONS+=(REPAIR-CL "Repair/Backup C-Lightning")
@@ -64,7 +66,7 @@ OPTIONS+=(RESET-ALL "Delete HDD completely to start fresh")
 OPTIONS+=(DELETE-ELEC "Delete Electrum Index")
 OPTIONS+=(DELETE-INDEX "Delete Bitcoin Transaction-Index")
 
-CHOICE=$(whiptail --clear --title "Repair Options" --menu "" 18 62 11 "${OPTIONS[@]}" 2>&1 >/dev/tty)
+CHOICE=$(whiptail --clear --title "Repair Options" --menu "" 19 62 12 "${OPTIONS[@]}" 2>&1 >/dev/tty)
 
 clear
 case $CHOICE in
@@ -76,12 +78,21 @@ case $CHOICE in
     read key
     ;;
   BACKUP-LND)
+    /home/admin/config.scripts/lnd.compact.sh interactive
     sudo /home/admin/config.scripts/lnd.backup.sh lnd-export-gui
     echo
     echo "Press ENTER when your backup download is done to shutdown."
     read key
     /home/admin/config.scripts/blitz.shutdown.sh
     ;;
+  COMPACT)
+    /home/admin/config.scripts/lnd.compact.sh interactive
+    echo "# Starting lnd.service ..."
+    sudo systemctl start lnd
+    echo
+    echo "Press ENTER to return to main menu."
+    read key
+    ;;
   REPAIR-CL)
     sudo /home/admin/99clRepairMenu.sh
     echo

home.admin/99updateMenu.sh

@@ -41,6 +41,7 @@ Do you want to download Lightning Data Backup now?
       echo "please wait .."
       sleep 2
       if [ "${lightning}" == "lnd" ]; then
+        /home/admin/config.scripts/lnd.compact.sh interactive
         /home/admin/config.scripts/lnd.backup.sh lnd-export-gui
       elif [ "${lightning}" == "cl" ]; then
         /home/admin/config.scripts/cl.backup.sh cl-export-gui

home.admin/config.scripts/lnd.compact.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+if [ "$1" = "-h" ] || [ "$1" = "--help" ] || [ "$1" = "-help" ]; then
+  echo "# script to compact the LND channel.db"
+  echo "# lnd.compact.sh <interactive>"
+  exit 1
+fi
+
+channelDBsize=$(sudo du -h /mnt/hdd/lnd/data/graph/mainnet/channel.db | awk '{print $1}')
+echo
+echo "The current channel.db size: $channelDBsize"
+echo "If compacting the database the first time it can take a long time, but reduces the size 2-3 times."
+echo "Can monitor the background process in a new window with:"
+echo "'tail -f /home/admin/lnd.db.bolt.auto-compact.log'"
+
+if [ "$1" = interactive ];then
+  read -p "Do you want to compact the database now (yes/no) ?" confirm && [[ $confirm == [yY]||$confirm == [yY][eE][sS] ]]||exit 1
+fi
+
+echo "# Stop LND"
+sudo systemctl stop lnd
+
+trap "exit" INT TERM ERR
+trap "kill 0" EXIT
+
+echo "# Run LND with --db.bolt.auto-compact"
+sudo -u bitcoin /usr/local/bin/lnd --configfile=/home/bitcoin/.lnd/lnd.conf --db.bolt.auto-compact > /home/admin/lnd.db.bolt.auto-compact.log &
+
+echo "# Compacting channel.db, this can take a long time"
+
+counter=0
+while [ $(sudo -u bitcoin lncli state 2>&1 | grep -c "connection refused") -gt 0 ]; do
+  echo
+  echo "# Waiting for LND to start "
+  echo "# Checking again in 10 seconds (${counter})"
+  counter=$((counter+1))
+  sleep 10
+done
+
+echo "# LND state:"
+sudo -u bitcoin lncli state
+
+counter=0
+while [ $(sudo -u bitcoin lncli state | grep -c "WAITING_TO_START") -gt 0 ]; do
+  echo
+  echo "# Compacting channel.db, this can take a long time"
+  echo "# Checking again in a minute (${counter})"
+  echo
+  counter=$((counter+1))
+  sleep 60
+done
+
+echo "# LND state:"
+sudo -u bitcoin lncli state
+
+sudo killall lnd >> /home/admin/lnd.db.bolt.auto-compact.log 2>&1
+
+echo
+echo "# Finished compacting."
+echo "# Showing logs:"
+cat /home/admin/lnd.db.bolt.auto-compact.log
+echo
+channelDBsize=$(sudo du -h /mnt/hdd/lnd/data/graph/mainnet/channel.db | awk '{print $1}')
+echo "# The current channel.db size: $channelDBsize"
+echo "# Exiting. Now can start LND again."
+
+exit 0