diff --git a/FAQ.md b/FAQ.md index 4a6c1438a..2970681f4 100644 --- a/FAQ.md +++ b/FAQ.md @@ -302,6 +302,17 @@ You can also put an empty file just called `hdmi` (without any ending) onto the ## Debug +### How do I generate a Debug Report? + +If your RaspiBlitz is not working correctly and you like to get help from the community, it's good to provide more debug information, so others can better diagnose your problem - please follow the following steps to generate a debug report: + +- SSH into your raspiblitz as admin user with your password A +- If you see the menu - use CTRL+C to get to the terminal +- To generate debug report run: `debug`, optionally create a link with `debug -l` +- Then copy all output beginning with `*** RASPIBLITZ LOGS ***` and share this + +*PLEASE NOTICE: It's possible that these logs can contain private information (like IPs, node IDs, ...) - just share publicly what you feel OK with.* + ### I have the full blockchain on another storage. How do I copy it to the RaspiBlitz? Copying a already synced blockchain from another storage (e.g. your Laptop or external hard drive) can be a quick way to get the RaspiBlitz started or replacing a corrupted blockchain with a fresh one. Also that way you have synced and verified the blockchain yourself, and are not trusting the RaspiBlitz Torrent downloads (Don't trust, verify). @@ -318,16 +329,13 @@ If everything described above is in order, start the setup of the new RaspiBlitz Once you finished all the transfers, the Raspiblitz will make a quick-check on the data - but that will not guarantee that everything in detail was OK with the transfer. Check further FAQ answers if you get stuck or see a final sync with a value below 90%. -### How do I generate a Debug Report? +### Bitcoind tells me to reindex - how can I do this? -If your RaspiBlitz is not working correctly and you like to get help from the community, it's good to provide more debug information, so others can better diagnose your problem - please follow the following steps to generate a debug report: + To find/access information fast in large data sets like the Bitcoin blockhain indexes are needed. Those indexes can get corrupted on your HDD/SSD and to repair them they need to be rebuild - re-indexed. Bitcoind has two different options to do this - a fast way called "reindex-chainstate" (which just rebuilds the UTXO set from the blocks as you have them) and the slow but complete way called just "reindex" that would even recheck all your block data - see for details here: https://bitcoin.stackexchange.com/questions/60709/when-should-i-use-reindex-chainstate-and-when-reindex + + So if you read in your debug logs of bitcoind that you should "reindex" you can try first just to do a fast "reindex-chainstate" and if that didnt worked a slow and full "reindex". -- SSH into your raspiblitz as admin user with your password A -- If you see the menu - use CTRL+C to get to the terminal -- To generate debug report run: `debug`, optionally create a link with `debug -l` -- Then copy all output beginning with `*** RASPIBLITZ LOGS ***` and share this - -*PLEASE NOTICE: It's possible that these logs can contain private information (like IPs, node IDs, ...) - just share publicly what you feel OK with.* +See the raspiblitz script `./config.scripts/network.reindex.sh` or the REAPIR menu to start these processes. ### Why is my "final sync" taking so long? diff --git a/home.admin/00infoBlitz.sh b/home.admin/00infoBlitz.sh index c4dc75702..bd525df63 100755 --- a/home.admin/00infoBlitz.sh +++ b/home.admin/00infoBlitz.sh @@ -353,7 +353,7 @@ else appInfoLine="" # Electrum Server - electrs - if [ "${ElectRS}" = "on" ]; then + if [ "${ElectRS}" == "on" ]; then error="" source <(sudo /home/admin/config.scripts/bonus.electrs.sh status-sync 2>/dev/null) if [ ${#infoSync} -gt 0 ]; then @@ -361,15 +361,10 @@ else fi fi - # BTC RPC EXPLORER - if [ "${BTCRPCexplorer}" = "on" ]; then - error="" - source <(sudo /home/admin/config.scripts/bonus.btc-rpc-explorer.sh status 2>/dev/null) - if [ ${#error} -gt 0 ]; then - appInfoLine="ERROR BTC-RPC-Explorer: ${error} (try restart)" - elif [ "${isIndexed}" = "0" ]; then - appInfoLine="BTC-RPC-Explorer: ${indexInfo}" - fi + # Transaction Index + source <(/home/admin/config.scripts/network.txindex.sh status) + if [ "${txindex}" == "1" ] && [ "${isIndexed}" != "1" ]; then + appInfoLine="Transaction Index: ${indexInfo}" fi if [ ${#appInfoLine} -gt 0 ]; then diff --git a/home.admin/98repairMenu.sh b/home.admin/98repairMenu.sh index af202a29c..d22457493 100755 --- a/home.admin/98repairMenu.sh +++ b/home.admin/98repairMenu.sh @@ -65,6 +65,9 @@ RaspiBlitz image to your SD card. " 12 40 } +# get status of txindex +source <(/home/admin/config.scripts/network.txindex.sh status) + OPTIONS=() #OPTIONS+=(HARDWARE "Run Hardwaretest") OPTIONS+=(SOFTWARE "Run Softwaretest (DebugReport)") @@ -76,8 +79,13 @@ if [ "${lightning}" == "cl" ] || [ "${cl}" == "on" ]; then fi OPTIONS+=(MIGRATION "Migrate Blitz Data to new Hardware") OPTIONS+=(COPY-SOURCE "Copy Blockchain Source Modus") -OPTIONS+=(REINDEX "Redindex Bitcoin Blockchain") -OPTIONS+=(DELETE-INDEX "Delete Bitcoin Transaction-Index") +if [ "${txindex}" == "1" ]; then + OPTIONS+=(DELETE-INDEX "Reindex Bitcoin Transaction-Index") +elif [ "${indexByteSize}" != "0" ]; then + OPTIONS+=(DELETE-INDEX "Delete Bitcoin Transaction-Index") +fi +OPTIONS+=(REINDEX-UTXO "Redindex Just Bitcoin Chainstate (Fast)") +OPTIONS+=(REINDEX-FULL "Redindex Full Bitcoin Blockchain (Slow)") OPTIONS+=(RESET-CHAIN "Delete Blockchain & Re-Download") OPTIONS+=(RESET-HDD "Delete HDD Data but keep Blockchain") OPTIONS+=(RESET-ALL "Delete HDD completely to start fresh") @@ -157,8 +165,12 @@ case $CHOICE in /home/admin/config.scripts/network.txindex.sh delete exit 0; ;; - REINDEX) - /home/admin/config.scripts/network.reindex.sh reindex main + REINDEX-UTXO) + /home/admin/config.scripts/network.reindex.sh reindex-chainstate mainnet + exit 0; + ;; + REINDEX-FULL) + /home/admin/config.scripts/network.reindex.sh reindex mainnet exit 0; ;; COPY-SOURCE) diff --git a/home.admin/config.scripts/bonus.electrs.sh b/home.admin/config.scripts/bonus.electrs.sh index b16f2ce90..932af2697 100755 --- a/home.admin/config.scripts/bonus.electrs.sh +++ b/home.admin/config.scripts/bonus.electrs.sh @@ -98,20 +98,21 @@ if [ "$1" = "status-sync" ] || [ "$1" = "status" ]; then echo "serviceRunning=${serviceRunning}" if [ ${serviceRunning} -eq 1 ]; then - # Experimental try to get sync Info - syncedToBlock=$(sudo journalctl -u electrs --no-pager -n2000 | grep "height=" | tail -n1| cut -d= -f3) - blockchainHeight=$(sudo -u bitcoin ${network}-cli getblockchaininfo 2>/dev/null | jq -r '.headers' | sed 's/[^0-9]*//g') - lastBlockchainHeight=$(($blockchainHeight -1)) - syncProgress=0 - if [ "${syncedToBlock}" != "" ] && [ "${blockchainHeight}" != "" ] && [ "${blockchainHeight}" != "0" ]; then - syncProgress="$(echo "$syncedToBlock" "$blockchainHeight" | awk '{printf "%.2f", $1 / $2 * 100}')" - fi - echo "syncProgress=${syncProgress}%" - if [ "${syncedToBlock}" = "${blockchainHeight}" ] || [ "${syncedToBlock}" = "${lastBlockchainHeight}" ]; then - echo "tipSynced=1" - else - echo "tipSynced=0" - fi + # Experimental try to get sync Info (electrs debug info would need more details) + #source <(/home/admin/_cache.sh get btc_mainnet_blocks_headers) + #blockchainHeight="${btc_mainnet_blocks_headers}" + #lastBlockchainHeight=$(($blockchainHeight -1)) + #syncedToBlock=$(sudo journalctl -u electrs --no-pager -n2000 | grep "height=" | tail -n1| cut -d= -f3) + #syncProgress=0 + #if [ "${syncedToBlock}" != "" ] && [ "${blockchainHeight}" != "" ] && [ "${blockchainHeight}" != "0" ]; then + # syncProgress="$(echo "$syncedToBlock" "$blockchainHeight" | awk '{printf "%.2f", $1 / $2 * 100}')" + #fi + #echo "syncProgress=${syncProgress}%" + #if [ "${syncedToBlock}" = "${blockchainHeight}" ] || [ "${syncedToBlock}" = "${lastBlockchainHeight}" ]; then + # echo "tipSynced=1" + #else + # echo "tipSynced=0" + #fi # check if initial sync was done, by setting a file as once electrs is the first time responding on port 50001 electrumResponding=$(echo '{"jsonrpc":"2.0","method":"server.ping","params":[],"id":"electrs-check"}' | netcat -w 2 127.0.0.1 50001 | grep -c "result") @@ -135,7 +136,7 @@ if [ "$1" = "status-sync" ] || [ "$1" = "status" ]; then fi else - echo "tipSynced=0" + # echo "tipSynced=0" echo "initialSynced=0" echo "electrumResponding=0" echo "infoSync='Not running - check: sudo journalctl -u electrs'" diff --git a/home.admin/config.scripts/network.reindex.sh b/home.admin/config.scripts/network.reindex.sh index b67b51264..f78ff514c 100755 --- a/home.admin/config.scripts/network.reindex.sh +++ b/home.admin/config.scripts/network.reindex.sh @@ -4,28 +4,29 @@ if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "-help" ]; then echo "script to run re-index if the blockchain - blocks will not be deleted but re-indexed" echo "will trigger reboot after started and progress can be monitored thru normal sync status" - echo "network.reindex.sh reindex [mainnet|testnet|signet] --> use to start re-index chain" + echo "There are two ways to re-index - for details see: https://bitcoin.stackexchange.com/a/60711" + echo "network.reindex.sh reindex [mainnet|testnet|signet] --> re-index chain & repair corrupt blocks" + echo "network.reindex.sh reindex-chainstate [mainnet|testnet|signet] --> only re-build UTXO set (fast)" exit 1 fi source /mnt/hdd/raspiblitz.conf -################### -# START -################### -if [ "$1" = "reindex" ]; then +if [ "$1" = "reindex" ] || [ "$1" = "reindex-chainstate" ]; then + + action="$1" # network prefixes if [ "$2" = "mainnet" ]; then - echo "# network.reindex.sh reindex --> mainnet" + echo "# network.reindex.sh ${action} --> mainnet" prefix="" netparam="" elif [ "$2" = "testnet" ]; then - echo "# network.reindex.sh reindex --> testnet" + echo "# network.reindex.sh ${action} --> testnet" prefix="t" netparam="-testnet " elif [ "$2" = "signet" ]; then - echo "# network.reindex.sh reindex --> signet" + echo "# network.reindex.sh ${action} --> signet" prefix="s" netparam="-signet " else @@ -38,8 +39,8 @@ if [ "$1" = "reindex" ]; then sudo systemctl stop ${prefix}${network}d # starting reindex - echo "# starting ${network} service with -reindex flag" - sudo -u bitcoin /usr/local/bin/${network}d ${netparam}-daemon -reindex -conf=/mnt/hdd/${network}/${network}.conf -datadir=/mnt/hdd/${network} 1>&2 + echo "# starting ${network} service with -${action} flag" + sudo -u bitcoin /usr/local/bin/${network}d ${netparam}-daemon -blockfilterindex=0 -${action} -conf=/mnt/hdd/${network}/${network}.conf -datadir=/mnt/hdd/${network} 1>&2 echo "# waiting 10 secs" sleep 10 echo "# going into reboot - reindex process can be monitored like normal blockchain sync status" diff --git a/home.admin/config.scripts/network.txindex.sh b/home.admin/config.scripts/network.txindex.sh index 3649bba60..df5b183b6 100755 --- a/home.admin/config.scripts/network.txindex.sh +++ b/home.admin/config.scripts/network.txindex.sh @@ -30,36 +30,47 @@ if [ "$1" = "status" ]; then echo "##### STATUS TXINDEX" + indexByteSize=$(sudo du -s /mnt/hdd/bitcoin/indexes/txindex 2>/dev/null | cut -f1) + if [ "${indexByteSize}" == "" ]; then + indexByteSize=0 + fi + echo "txindex=${txindex}" + echo "indexByteSize=${indexByteSize}" if [ ${txindex} -eq 0 ]; then exit 0 fi # try to gather if still indexing + source <(/home/admin/_cache.sh get btc_mainnet_blocks_headers) + blockchainHeight="${btc_mainnet_blocks_headers}" indexedToBlock=$(sudo tail -n 200 /mnt/hdd/${network}${pathAdd}/debug.log | grep "Syncing txindex with block chain from height" | tail -n 1 | cut -d " " -f 9 | sed 's/[^0-9]*//g') - blockchainHeight=$(sudo -u bitcoin ${network}-cli getblockchaininfo 2>/dev/null | jq -r '.blocks' | sed 's/[^0-9]*//g') indexFinished=$(sudo tail -n 200 /mnt/hdd/${network}${pathAdd}/debug.log | grep -c "txindex is enabled at height") - echo "indexedToBlock=${indexedToBlock}" - echo "blockchainHeight=${blockchainHeight}" - echo "indexFinished=${indexFinished}" + if [ ${#indexedToBlock} -eq 0 ] || [ ${indexFinished} -gt 0 ] || [ "${indexedToBlock}" = "${blockchainHeight}" ]; then echo "isIndexed=1" + indexedToBlock=$blockchainHeight + indexFinished=1 indexInfo="OK" else echo "isIndexed=0" if [ ${#indexedToBlock} -gt 0 ] && [ ${#blockchainHeight} -gt 0 ]; then progressPercent=$(printf %.2f $(echo "${indexedToBlock}/${blockchainHeight}*100" | bc -l)) - indexInfo="Indexing is at ${progressPercent}% (please wait)" + indexInfo="Building ${progressPercent}% (please wait)" else - indexInfo="Indexing is running (please wait)" + indexInfo="Building (please wait)" fi echo "indexInfo='${indexInfo}'" - fi + fi + + echo "indexFinished=${indexFinished}" + echo "indexedToBlock=${indexedToBlock}" + echo "blockchainHeight=${blockchainHeight}" + exit 0 fi - ################### # switch on ################### @@ -84,27 +95,26 @@ if [ "$1" = "1" ] || [ "$1" = "on" ]; then fi fi - ################### # switch off ################### if [ "$1" = "0" ] || [ "$1" = "off" ]; then + echo "# changing config ..." sudo sed -i "s/^txindex=.*/txindex=0/g" /mnt/hdd/${network}/${network}.conf + echo "# deinstalling apps needing txindex ..." + sudo -u admin /home/admin/config.scripts/bonus.btc-rpc-explorer.sh off + echo "# restarting bitcoind ..." sudo systemctl restart ${network}d exit 0 fi - ################### # delete (and make sure all using apps are deinstalled) # on version update check all bonus scripts that this network.txindex.sh on ################### if [ "$1" = "delete" ]; then - echo "# deinstalling apps needing txindex ..." - sudo -u admin /home/admin/config.scripts/bonus.btc-rpc-explorer.sh off - echo "# changing config ..." + echo "# stopping bitcoind ..." sudo systemctl stop ${network}d - sudo sed -i "s/^txindex=.*/txindex=0/g" /mnt/hdd/${network}/${network}.conf echo "# deleting tx index ..." sudo rm -r /mnt/hdd/${network}/indexes/txindex echo "# restarting bitcoind ..."