diff --git a/.github/workflows/amd64-fatpack-image.yml b/.github/workflows/amd64-fatpack-image.yml index 59b32b551..362298003 100644 --- a/.github/workflows/amd64-fatpack-image.yml +++ b/.github/workflows/amd64-fatpack-image.yml @@ -13,7 +13,6 @@ on: # - 'home.admin/bitcoin.install.sh' # - 'home.admin/lnd.install.sh' # - 'home.admin/cl.install.sh' - # - 'home.admin/cl-plugin.cln-grpc.sh' # - 'home.admin/tor.install.sh' # - 'home.admin/blitz.i2pd.sh' # - 'home.admin/blitz.web.sh' @@ -34,7 +33,6 @@ on: # - 'home.admin/bitcoin.install.sh' # - 'home.admin/lnd.install.sh' # - 'home.admin/cl.install.sh' - # - 'home.admin/cl-plugin.cln-grpc.sh' # - 'home.admin/tor.install.sh' # - 'home.admin/blitz.i2pd.sh' # - 'home.admin/blitz.web.sh' diff --git a/home.admin/_provision.setup.sh b/home.admin/_provision.setup.sh index e761c306e..157943117 100755 --- a/home.admin/_provision.setup.sh +++ b/home.admin/_provision.setup.sh @@ -331,17 +331,27 @@ fi if [ "${lightning}" == "cl" ]; then ################################### - # c-lightning - echo "############## c-lightning" >> ${logFile} + # core lightning + echo "############## core lightning" >> ${logFile} - # install c-lightning (when not done by sd card fatpack) + # install core lightning (when not done by sd card fatpack) # if already installed - will skip /home/admin/_cache.sh set message "Core Lightning Install" - /home/admin/config.scripts/cl.install.sh install >> ${logFile} + echo "# Starting CLN binary installation..." >> ${logFile} + if ! /home/admin/config.scripts/cl.install.sh install >> ${logFile} 2>&1; then + /home/admin/config.scripts/blitz.error.sh _provision.setup.sh "cl-install-binary" "cl.install.sh install failed" "Check ${logFile} for compilation errors. Possible causes: insufficient disk space, compilation failure, missing dependencies" ${logFile} + exit 20 + fi + echo "# CLN binary installation completed successfully" >> ${logFile} echo "# switch mainnet config on" >> ${logFile} /home/admin/_cache.sh set message "Core Lightning Setup" - /home/admin/config.scripts/cl.install.sh on mainnet >> ${logFile} + echo "# Starting CLN mainnet configuration..." >> ${logFile} + if ! /home/admin/config.scripts/cl.install.sh on mainnet >> ${logFile} 2>&1; then + /home/admin/config.scripts/blitz.error.sh _provision.setup.sh "cl-config-mainnet" "cl.install.sh on mainnet failed" "Check ${logFile} for configuration errors. CLN binary may have installed but configuration failed" ${logFile} + exit 21 + fi + echo "# CLN mainnet configuration completed successfully" >> ${logFile} # OLD WALLET FROM CLIGHTNING RESCUE if [ "${clrescue}" != "" ]; then @@ -439,7 +449,7 @@ if [ "${lightning}" == "cl" ]; then fi - # stop c-lightning for the rest of the provision process + # stop core lightning for the rest of the provision process echo "stopping lightningd for the rest provision again (will start on next boot)" >> ${logFile} systemctl stop lightningd >> ${logFile} diff --git a/home.admin/config.scripts/blitz.data.sh b/home.admin/config.scripts/blitz.data.sh index 3384057bc..378c2e3cb 100644 --- a/home.admin/config.scripts/blitz.data.sh +++ b/home.admin/config.scripts/blitz.data.sh @@ -656,10 +656,10 @@ if [ "$action" = "status" ]; then # STORAGE if [ ${#storageDevice} -gt 0 ]; then - if [ ${storageSizeGB} -lt $((storageFullMinGB - 1)) ]; then + if [ ${storageSizeGB} -lt $(storageFullMinGB) ]; then storageWarning='only-pruned' fi - if [ ${storageSizeGB} -lt $((storagePrunedMinGB - 1)) ]; then + if [ ${storageSizeGB} -lt $(storagePrunedMinGB) ]; then storageWarning='too-small' fi fi @@ -1575,6 +1575,7 @@ if [ "$action" = "setup" ]; then parted /dev/${actionDevice} --script mklabel msdos parted /dev/${actionDevice} --script mkpart primary fat32 1MiB 513MiB parted /dev/${actionDevice} --script mkpart primary ext4 541MB 100% + partprobe /dev/${actionDevice} wipefs -a /dev/${actionDevicePartitionBase}1 2>/dev/null mkfs.fat -F 32 /dev/${actionDevicePartitionBase}1 wipefs -a /dev/${actionDevicePartitionBase}2 2>/dev/null @@ -1632,6 +1633,7 @@ if [ "$action" = "setup" ]; then parted /dev/${actionDevice} --script mkpart primary fat32 1MiB 513MiB >> ${logFile} parted /dev/${actionDevice} --script mkpart primary ext4 541MB 65GB >> ${logFile} parted /dev/${actionDevice} --script mkpart primary ext4 65GB 100% >> ${logFile} + partprobe /dev/${actionDevice} echo "# .. formating" >> ${logFile} wipefs -a /dev/${actionDevicePartitionBase}1 2>/dev/null >> ${logFile} mkfs.fat -F 32 /dev/${actionDevicePartitionBase}1 >> ${logFile} @@ -1657,6 +1659,7 @@ if [ "$action" = "setup" ]; then wipefs -a /dev/${actionDevice} >> ${logFile} parted /dev/${actionDevice} --script mklabel msdos >> ${logFile} parted /dev/${actionDevice} --script mkpart primary ext4 1MB 100% >> ${logFile} + partprobe /dev/${actionDevice} echo "# .. formating" >> ${logFile} wipefs -a /dev/${actionDevicePartitionBase}1 >> ${logFile} mkfs -t ext4 /dev/${actionDevicePartitionBase}1 >> ${logFile} @@ -1678,6 +1681,7 @@ if [ "$action" = "setup" ]; then wipefs -a /dev/${actionDevice} 2>/dev/null parted /dev/${actionDevice} --script mklabel msdos parted /dev/${actionDevice} --script mkpart primary ext4 1MB 100% + partprobe /dev/${actionDevice} echo "# .. formating" >> ${logFile} wipefs -a /dev/${actionDevicePartitionBase}1 2>/dev/null mkfs -t ext4 /dev/${actionDevicePartitionBase}1 diff --git a/home.admin/config.scripts/cl-plugin.cln-grpc.sh b/home.admin/config.scripts/cl-plugin.cln-grpc.sh deleted file mode 100644 index 12461050c..000000000 --- a/home.admin/config.scripts/cl-plugin.cln-grpc.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash - -# command info -if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "-help" ]; then - echo - echo "Install the cln-grpc plugin for CLN" - echo "Usage:" - echo "cl-plugin.cln-grpc.sh install - called by build_sdcard.sh" - echo "cl-plugin.cln-grpc.sh on " - echo "cl-plugin.cln-grpc.sh off " - echo "cl-plugin.cln-grpc.sh status " - echo "cl-plugin.cln-grpc.sh update " - echo - exit 1 -fi - -echo "# cl-plugin.cln-grpc.sh $*" - -if [ "$2" = testnet ] || [ "$2" = signet ]; then - NETWORK=$2 -else - NETWORK=mainnet -fi -source <(/home/admin/config.scripts/network.aliases.sh getvars cl $NETWORK) - -# netprefix is: "" | t | s -# portprefix is: "" | 1 | 3 -PORT="${portprefix}4772" - -function buildGRPCplugin() { - echo "# - check the cln-grpc plugin" - if [ ! -f /usr/local/libexec/c-lightning/plugins/cln-grpc ] || [ ! -f /home/bitcoin/cl-plugins-available/cln-grpc ]; then - # check if the source code is present - if [ ! -d /home/bitcoin/lightning/plugins/grpc-plugin ]; then - echo "# - install Core Lightning ..." - /home/admin/config.scripts/cl.install.sh install || exit 1 - fi - if [ -f /usr/local/libexec/c-lightning/plugins/cln-grpc ]; then - echo "# - cln-grpc plugin was installed" - else - echo "# install dependencies" - sudo apt-get install protobuf-compiler -y - echo "# rust for cln-grpc, includes rustfmt" - sudo -u bitcoin curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sudo -u bitcoin sh -s -- -y - cd /home/bitcoin/lightning/plugins/grpc-plugin || exit 1 - echo "# build" - sudo -u bitcoin /home/bitcoin/.cargo/bin/cargo build --target-dir /home/bitcoin/cln-grpc-build - echo "# delete old dir or binary" - sudo rm -rf /home/bitcoin/cl-plugins-available/cln-grpc - echo "# move to /home/bitcoin/cl-plugins-available/" - sudo mkdir -p /home/bitcoin/cl-plugins-available - sudo -u bitcoin mv /home/bitcoin/cln-grpc-build/debug/cln-grpc /home/bitcoin/cl-plugins-available/ - fi - else - echo "# - cln-grpc plugin was already built/installed" - fi - echo "# Cleaning" - sudo rm -rf /home/bitcoin/.rustup - sudo rm -rf /home/bitcoin/.cargo/ - sudo rm -rf /home/bitcoin/.cache - sudo rm -rf /home/bitcoin/cln-grpc-build -} - -function switchOn() { - if ! $lightningcli_alias plugin list | grep "/home/bitcoin/${netprefix}cl-plugins-enabled/cln-grpc"; then - buildGRPCplugin - - if [ ! -f /usr/local/libexec/c-lightning/plugins/cln-grpc ]; then - # symlink to plugin directory - echo "# symlink cln-grpc to /home/bitcoin/${netprefix}cl-plugins-enabled/" - # delete old symlink - sudo rm -f /home/bitcoin/${netprefix}cl-plugins-enabled/cln-grpc - sudo ln -s /home/bitcoin/cl-plugins-available/cln-grpc /home/bitcoin/${netprefix}cl-plugins-enabled/ - fi - # blitz.conf.sh set [key] [value] [?conffile] - /home/admin/config.scripts/blitz.conf.sh set "grpc-port" "${PORT}" "${CLCONF}" "noquotes" - /home/admin/config.scripts/blitz.conf.sh set "${netprefix}clnGRPCport" "${PORT}" - - # firewall - sudo ufw allow "${PORT}" comment "${netprefix}clnGRPCport" - # Tor - /home/admin/config.scripts/tor.onion-service.sh "${netprefix}clnGRPCport" "${PORT}" "${PORT}" - source <(/home/admin/_cache.sh get state) - if [ "${state}" == "ready" ]; then - sudo systemctl restart ${netprefix}lightningd - fi - echo "# cl-plugin.cln-grpc.sh on --> done" - - else - echo "# cl-plugin.cln-grpc.sh on --> already installed and running" - fi - exit 0 -} - -if [ "$1" = install ]; then - buildGRPCplugin - echo "# cl-plugin.cln-grpc.sh install --> done" - exit 0 - -elif [ "$1" = status ]; then - - portActive=$(nc -vz 127.0.0.1 $PORT 2>&1 | grep -c "succeeded") - echo "port=${PORT}" - echo "portActive=${portActive}" - exit 0 - -elif [ "$1" = on ]; then - switchOn - -elif [ "$1" = off ]; then - sed -i "/^grpc-port/d" "${CLCONF}" - sudo rm -rf /home/bitcoin/${netprefix}cl-plugins-enabled/cln-grpc - if [ "$(echo "$@" | grep -c purge)" -gt 0 ]; then - sudo rm -rf /home/bitcoin/cl-plugins-available/cln-grpc - fi - /home/admin/config.scripts/blitz.conf.sh set "${netprefix}clnGRPCport" "off" - # firewall - sudo ufw deny "${PORT}" comment "${netprefix}clnGRPCport" - # Tor - /home/admin/config.scripts/tor.onion-service.sh off "${netprefix}clnGRPCport" - exit 0 - -elif [ "$1" = update ]; then - if [ "$(echo "$@" | grep -c source)" -gt 0 ]; then - cd /home/bitcoin/lightning/ || exit 1 - sudo -u bitcoin git pull - fi - sudo rm -rf /home/bitcoin/cl-plugins-available/cln-grpc - buildGRPCplugin - sudo systemctl stop ${netprefix}lightningd - switchOn - echo "# cl-plugin.cln-grpc.sh update --> done" - -else - echo "FAIL - Unknown Parameter $1" - exit 1 -fi diff --git a/home.admin/config.scripts/cl.check.sh b/home.admin/config.scripts/cl.check.sh index 521ded0f9..c908d1058 100644 --- a/home.admin/config.scripts/cl.check.sh +++ b/home.admin/config.scripts/cl.check.sh @@ -68,22 +68,39 @@ if [ "$1" == "prestart" ]; then fi fi - if [ $(grep -c "^grpc-port" <${CLCONF}) -gt 0 ]; then - if [ ! -f /home/bitcoin/${netprefix}cl-plugins-enabled/cln-grpc ] || [ "$(eval echo \$${netprefix}cln-grpc-port)" = "off" ]; then - echo "# The cln-grpc plugin is not present but in config" - sed -i "/^grpc-port/d" ${CLCONF} - rm -rf /home/bitcoin/${netprefix}cl-plugins-enabled/cln-grpc - else - echo "# Make sure that the correct GRPC port is used for $NETWORK" - sed -i "/^grpc-port=*/grpc-port=${portprefix}4772/g" ${CLCONF} + if [ -f /home/bitcoin/${netprefix}cl-plugins-enabled/cln-grpc ] || + [ -f /usr/local/libexec/c-lightning/plugins/cln-grpc ] || + [ "$(eval echo \$${netprefix}cln-grpc-port)" != "off" ]; then + if [ $(grep -c "^grpc-port" <${CLCONF}) -eq 0 ]; then + echo "# Create grpc-port entry" + echo "grpc-port=${portprefix}4772" | tee -a ${CLCONF} fi + echo "# Make sure that the correct GRPC port is used for $NETWORK" + sed -i "/^grpc-port=*/grpc-port=${portprefix}4772/g" ${CLCONF} + else + echo "# The cln-grpc plugin is not present but in config" + sed -i "/^grpc-port/d" ${CLCONF} + rm -rf /home/bitcoin/${netprefix}cl-plugins-enabled/cln-grpc fi - if ! grep "^clnrest-port=${portprefix}7378" <${CLCONF}; then - echo "clnrest-port=${portprefix}7378" | tee -a ${CLCONF} - fi - if ! grep "^clnrest-host=0.0.0.0" <${CLCONF}; then - echo "clnrest-host=0.0.0.0" | tee -a ${CLCONF} + if [ -f /home/bitcoin/${netprefix}cl-plugins-enabled/clnrest ] || + [ -f /usr/local/libexec/c-lightning/plugins/clnrest ]; then + if [ $(grep -c "^clnrest-port" <${CLCONF}) -eq 0 ]; then + echo "# Create clnrest-port entry" + echo "clnrest-port=${portprefix}7378" | tee -a ${CLCONF} + fi + if [ $(grep -c "^clnrest-host" <${CLCONF}) -eq 0 ]; then + echo "# Create clnrest-host entry" + echo "clnrest-host=0.0.0.0" | tee -a ${CLCONF} + fi + echo "# Make sure that the correct clnrest port is used for $NETWORK" + sed -i "s/^clnrest-port=*/clnrest-port=${portprefix}7378/g" ${CLCONF} + sed -i "s/^clnrest-host=*/clnrest-host=0.0.0.0/g" ${CLCONF} + else + echo "# The clnrest plugin is not present but in config" + sed -i "/^clnrest-port/d" ${CLCONF} + sed -i "/^clnrest-host/d" ${CLCONF} + rm -rf /home/bitcoin/${netprefix}cl-plugins-enabled/clnrest fi exit 0 diff --git a/home.admin/config.scripts/cl.install-service.sh b/home.admin/config.scripts/cl.install-service.sh index a743bd108..b43ffc83c 100644 --- a/home.admin/config.scripts/cl.install-service.sh +++ b/home.admin/config.scripts/cl.install-service.sh @@ -58,19 +58,21 @@ After=network-online.target [Service] ExecStartPre=-/home/admin/config.scripts/cl.check.sh prestart $CHAIN -ExecStart=/bin/sh -c '${passwordInput}/usr/local/bin/lightningd --conf=${CLCONF} ${encryptedHSMoption} --pid-file=/run/lightningd/${netprefix}lightningd.pid --rpc-file-mode 0660' +ExecStart=/bin/sh -c '${passwordInput}/usr/local/bin/lightningd --conf=${CLCONF} ${encryptedHSMoption} --rpc-file-mode 0660' ExecStartPost=-/home/admin/config.scripts/cl.check.sh poststart $CHAIN # Creates /run/lightningd owned by bitcoin RuntimeDirectory=lightningd User=bitcoin Group=bitcoin -# Type=forking hangs on restart +# Use Type=simple - most reliable for CLN Type=simple -PIDFile=/run/lightningd/${netprefix}lightningd.pid +# Don't use PIDFile with Type=simple to avoid conflicts Restart=always RestartSec=60 TimeoutSec=240 +# Increase startup timeout for CLN to fully initialize +TimeoutStartSec=300 StandardOutput=null StandardError=journal diff --git a/home.admin/config.scripts/cl.install.sh b/home.admin/config.scripts/cl.install.sh index 4889ed497..882c739e3 100644 --- a/home.admin/config.scripts/cl.install.sh +++ b/home.admin/config.scripts/cl.install.sh @@ -50,7 +50,7 @@ function installDependencies() { sudo pip3 install --upgrade pip # for wss-proxy - https://docs.corelightning.org/docs/installation#wss-proxy sudo -u bitcoin pip3 config set global.break-system-packages true - sudo -u bitcoin pip3 install --user pyln-client websockets + sudo -u bitcoin pip3 install --user pyln-client websockets grpcio-tools # poetry sudo pip3 install poetry if ! grep -Eq '^PATH="$HOME/.local/bin:$PATH"' /home/bitcoin/.profile; then @@ -61,14 +61,28 @@ function installDependencies() { sudo -u bitcoin poetry install # rust deps for cln-grpc and clnrest plugins - if ! sudo -u bitcoin bash -c 'command -v cargo'; then - sudo -u bitcoin bash -c 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y' + echo "# Install Rust to /opt/rust/" + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sudo RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/rust sh -s -- --no-modify-path -y + echo "# Set /opt/rust write access for rust group" + if ! getent group rust >/dev/null 2>&1; then + sudo groupadd rust fi - # Ensure /home/bitcoin/.cargo/bin is in PATH for the bitcoin user - if ! grep -Fq '.cargo/bin' /home/bitcoin/.profile; then - echo -e '\n# set PATH so it includes Cargo'\''s bin if it exists\nif [ -d "$HOME/.cargo/bin" ] ; then\n PATH="$HOME/.cargo/bin:$PATH"\nfi' | sudo tee -a /home/bitcoin/.profile + sudo chown -R root:rust /opt/rust + sudo chmod -R g+w /opt/rust + sudo usermod -a -G rust bitcoin + echo "# Set the default Rust toolchain" + sudo RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/rust /opt/rust/bin/rustup default stable + echo "# Make Rust binaries available system-wide" + sudo ln -sf /opt/rust/bin/* /usr/local/bin/ + echo "# Set up system-wide environment variables for Rust" + if ! grep -q "RUSTUP_HOME=/opt/rust" /etc/environment; then + echo 'RUSTUP_HOME=/opt/rust' | sudo tee -a /etc/environment fi - export PATH="/home/bitcoin/.cargo/bin:$PATH" + if ! grep -q "CARGO_HOME=/opt/rust" /etc/environment; then + echo 'CARGO_HOME=/opt/rust' | sudo tee -a /etc/environment + fi + sudo apt-get install -y protobuf-compiler # remove old clnrest dir if exists @@ -80,17 +94,34 @@ function installDependencies() { function buildAndInstallCLbinaries() { echo "- configure" echo - sudo -u bitcoin ./configure + sudo -u bitcoin RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/rust ./configure || exit 1 echo echo "- make" echo - sudo -u bitcoin make - echo - echo "- make check VALGRIND=0" - sudo -u bitcoin make check VALGRIND=0 + sudo -u bitcoin RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/rust make || exit 1 echo echo "- install to /usr/local/bin/" - sudo make install || exit 1 + sudo make RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/rust install || exit 1 +} + +function runTests() { + # for the tests - install Core Lightning test dependencies matching pyproject.toml versions + # based on https://github.com/ElementsProject/lightning/blob/master/contrib/pyln-testing/pyproject.toml + echo "- install Core Lightning test dependencies" + sudo -u bitcoin pip3 install --user --upgrade \ + "pytest>=7" \ + "ephemeral-port-reserve>=1.1.4" \ + "psycopg2-binary>=2.9" \ + "python-bitcoinlib>=0.11.0" \ + "jsonschema>=4.4.0" \ + "Flask>=2" \ + "cheroot>=8,<=10" \ + "psutil>=5.9" \ + "requests>=2.31.0" \ + python-socketio websocket-client flaky + echo "- run tests" + echo + sudo -u bitcoin RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/rust make check || exit 1 } echo "# Running: 'cl.install.sh $*'" @@ -160,7 +191,7 @@ if [ "$1" = "install" ]; then installDependencies - buildAndInstallCLbinaries + buildAndInstallCLbinaries || exit 1 installed=$(sudo -u bitcoin lightning-cli --version) if [ ${#installed} -eq 0 ]; then @@ -256,7 +287,7 @@ if [ "$1" = on ] || [ "$1" = update ] || [ "$1" = testPR ]; then ) echo "# Building from source Core Lightning $currentCLversion" - buildAndInstallCLbinaries + buildAndInstallCLbinaries || exit 1 fi @@ -277,7 +308,9 @@ if [ "$1" = on ] || [ "$1" = update ] || [ "$1" = testPR ]; then sudo -u bitcoin mkdir /home/bitcoin/cl-plugins-available 2>/dev/null echo "# Store the lightning data in /mnt/hdd/app-data/.lightning" - sudo mkdir -p /mnt/hdd/app-data/.lightning + # Create the main and network-specific lightning directories + sudo mkdir -p "/mnt/hdd/app-data/.lightning/${CLNETWORK}" + sudo chown -R bitcoin:bitcoin /mnt/hdd/app-data/.lightning echo "# Symlink to /home/bitcoin/" sudo rm -rf /home/bitcoin/.lightning # not a symlink, delete sudo ln -s /mnt/hdd/app-data/.lightning /home/bitcoin/ @@ -285,10 +318,6 @@ if [ "$1" = on ] || [ "$1" = update ] || [ "$1" = testPR ]; then sudo rm -rf /home/admin/.lightning # not a symlink, delete sudo ln -s /mnt/hdd/app-data/.lightning /home/admin/ - if [ ${CLNETWORK} != "bitcoin" ] && [ ! -d /home/bitcoin/.lightning/${CLNETWORK} ]; then - sudo -u bitcoin mkdir /home/bitcoin/.lightning/${CLNETWORK} - fi - if ! sudo ls ${CLCONF} 2>/dev/null; then echo "# Create ${CLCONF}" echo "# lightningd configuration for ${network} ${CHAIN}