mirror of
https://github.com/bitcoin/bitcoin.git
synced 2026-06-03 17:54:19 +02:00
Merge bitcoin/bitcoin#29553: assumeutxo: Add dumptxoutset height param, remove shell scripts
94b0adcc37rpc, refactor: Prevent potential race conditions in dumptxoutset (Fabian Jahr)e868a6e070doc: Improve assumeutxo guide and add more docs/comments (Fabian Jahr)b29c21fc92assumeutxo: Remove devtools/utxo_snapshot.sh (Fabian Jahr)20a1c77aa7contrib: Remove test_utxo_snapshots.sh (Fabian Jahr)8426850352test: Test for dumptxoutset at specific height (Fabian Jahr)993cafe7e4RPC: Add type parameter to dumptxoutset (Fabian Jahr)fccf4f91d2RPC: Extract ReconsiderBlock helper (Fabian Jahr)446ce51c21RPC: Extract InvalidateBlock helper (Fabian Jahr) Pull request description: This adds a height parameter to the `dumptxoutset` RPC. This internalizes the workflow that was previously done by scripts: roll back the chain to the height we actually want the snapshot from, create the snapshot, roll forward to the real tip again. The nice thing about internalizing this functionality is that we can write tests for the code and it gives us more options to make the functionality robust. The shell scripts we have so far will be more cumbersome to maintain in the long run, especially since we will only notice later when we have broken them. I think it's safe to remove these `test_utxo_snapshots.sh` as well when we have this option in `dumptxoutset` because we have also added some good additional functional test coverage for this functionality. ACKs for top commit: Sjors: re-utACK94b0adcc37achow101: ACK94b0adcc37mzumsande: ACK94b0adcc37pablomartin4btc: re-ACK94b0adcc37Tree-SHA512: a4c9af5f687d1ca7bfb579a36f363882823386b5fa80c05de531b05a2782b5da6ff5baf3ada4bca8f32f63975d86f1948175abed9affe51fc958472b5f838dab
This commit is contained in:
Ava Chow
@@ -1,209 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Demonstrate the creation and usage of UTXO snapshots.
|
||||
#
|
||||
# A server node starts up, IBDs up to a certain height, then generates a UTXO
|
||||
# snapshot at that point.
|
||||
#
|
||||
# The server then downloads more blocks (to create a diff from the snapshot).
|
||||
#
|
||||
# We bring a client up, load the UTXO snapshot, and we show the client sync to
|
||||
# the "network tip" and then start a background validation of the snapshot it
|
||||
# loaded. We see the background validation chainstate removed after validation
|
||||
# completes.
|
||||
#
|
||||
# The shellcheck rule SC2086 (quoted variables) disablements are necessary
|
||||
# since this rule needs to be violated in order to get bitcoind to pick up on
|
||||
# $EARLY_IBD_FLAGS for the script to work.
|
||||
|
||||
export LC_ALL=C
|
||||
set -e
|
||||
|
||||
BASE_HEIGHT=${1:-30000}
|
||||
INCREMENTAL_HEIGHT=20000
|
||||
FINAL_HEIGHT=$((BASE_HEIGHT + INCREMENTAL_HEIGHT))
|
||||
|
||||
SERVER_DATADIR="$(pwd)/utxodemo-data-server-$BASE_HEIGHT"
|
||||
CLIENT_DATADIR="$(pwd)/utxodemo-data-client-$BASE_HEIGHT"
|
||||
UTXO_DAT_FILE="$(pwd)/utxo.$BASE_HEIGHT.dat"
|
||||
|
||||
# Chosen to try to not interfere with any running bitcoind processes.
|
||||
SERVER_PORT=8633
|
||||
SERVER_RPC_PORT=8632
|
||||
|
||||
CLIENT_PORT=8733
|
||||
CLIENT_RPC_PORT=8732
|
||||
|
||||
SERVER_PORTS="-port=${SERVER_PORT} -rpcport=${SERVER_RPC_PORT}"
|
||||
CLIENT_PORTS="-port=${CLIENT_PORT} -rpcport=${CLIENT_RPC_PORT}"
|
||||
|
||||
# Ensure the client exercises all indexes to test that snapshot use works
|
||||
# properly with indexes.
|
||||
ALL_INDEXES="-txindex -coinstatsindex -blockfilterindex=1"
|
||||
|
||||
if ! command -v jq >/dev/null ; then
|
||||
echo "This script requires jq to parse JSON RPC output. Please install it."
|
||||
echo "(e.g. sudo apt install jq)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DUMP_OUTPUT="dumptxoutset-output-$BASE_HEIGHT.json"
|
||||
|
||||
finish() {
|
||||
echo
|
||||
echo "Killing server and client PIDs ($SERVER_PID, $CLIENT_PID) and cleaning up datadirs"
|
||||
echo
|
||||
rm -f "$UTXO_DAT_FILE" "$DUMP_OUTPUT"
|
||||
rm -rf "$SERVER_DATADIR" "$CLIENT_DATADIR"
|
||||
kill -9 "$SERVER_PID" "$CLIENT_PID"
|
||||
}
|
||||
|
||||
trap finish EXIT
|
||||
|
||||
# Need to specify these to trick client into accepting server as a peer
|
||||
# it can IBD from, otherwise the default values prevent IBD from the server node.
|
||||
EARLY_IBD_FLAGS="-maxtipage=9223372036854775207 -minimumchainwork=0x00"
|
||||
|
||||
server_rpc() {
|
||||
./src/bitcoin-cli -rpcport=$SERVER_RPC_PORT -datadir="$SERVER_DATADIR" "$@"
|
||||
}
|
||||
client_rpc() {
|
||||
./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir="$CLIENT_DATADIR" "$@"
|
||||
}
|
||||
server_sleep_til_boot() {
|
||||
while ! server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
|
||||
}
|
||||
client_sleep_til_boot() {
|
||||
while ! client_rpc ping >/dev/null 2>&1; do sleep 0.1; done
|
||||
}
|
||||
server_sleep_til_shutdown() {
|
||||
while server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
|
||||
}
|
||||
|
||||
mkdir -p "$SERVER_DATADIR" "$CLIENT_DATADIR"
|
||||
|
||||
echo "Hi, welcome to the assumeutxo demo/test"
|
||||
echo
|
||||
echo "We're going to"
|
||||
echo
|
||||
echo " - start up a 'server' node, sync it via mainnet IBD to height ${BASE_HEIGHT}"
|
||||
echo " - create a UTXO snapshot at that height"
|
||||
echo " - IBD ${INCREMENTAL_HEIGHT} more blocks on top of that"
|
||||
echo
|
||||
echo "then we'll demonstrate assumeutxo by "
|
||||
echo
|
||||
echo " - starting another node (the 'client') and loading the snapshot in"
|
||||
echo " * first you'll have to modify the code slightly (chainparams) and recompile"
|
||||
echo " * don't worry, we'll make it easy"
|
||||
echo " - observing the client sync ${INCREMENTAL_HEIGHT} blocks on top of the snapshot from the server"
|
||||
echo " - observing the client validate the snapshot chain via background IBD"
|
||||
echo
|
||||
read -p "Press [enter] to continue" _
|
||||
|
||||
echo
|
||||
echo "-- Starting the demo. You might want to run the two following commands in"
|
||||
echo " separate terminal windows:"
|
||||
echo
|
||||
echo " watch -n0.1 tail -n 30 $SERVER_DATADIR/debug.log"
|
||||
echo " watch -n0.1 tail -n 30 $CLIENT_DATADIR/debug.log"
|
||||
echo
|
||||
read -p "Press [enter] to continue" _
|
||||
|
||||
echo
|
||||
echo "-- IBDing the blocks (height=$BASE_HEIGHT) required to the server node..."
|
||||
# shellcheck disable=SC2086
|
||||
./src/bitcoind -logthreadnames=1 $SERVER_PORTS \
|
||||
-datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -stopatheight="$BASE_HEIGHT" >/dev/null
|
||||
|
||||
echo
|
||||
echo "-- Creating snapshot at ~ height $BASE_HEIGHT ($UTXO_DAT_FILE)..."
|
||||
server_sleep_til_shutdown # wait for stopatheight to be hit
|
||||
# shellcheck disable=SC2086
|
||||
./src/bitcoind -logthreadnames=1 $SERVER_PORTS \
|
||||
-datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -connect=0 -listen=0 >/dev/null &
|
||||
SERVER_PID="$!"
|
||||
|
||||
server_sleep_til_boot
|
||||
server_rpc dumptxoutset "$UTXO_DAT_FILE" > "$DUMP_OUTPUT"
|
||||
cat "$DUMP_OUTPUT"
|
||||
kill -9 "$SERVER_PID"
|
||||
|
||||
RPC_BASE_HEIGHT=$(jq -r .base_height < "$DUMP_OUTPUT")
|
||||
RPC_AU=$(jq -r .txoutset_hash < "$DUMP_OUTPUT")
|
||||
RPC_NCHAINTX=$(jq -r .nchaintx < "$DUMP_OUTPUT")
|
||||
RPC_BLOCKHASH=$(jq -r .base_hash < "$DUMP_OUTPUT")
|
||||
|
||||
server_sleep_til_shutdown
|
||||
|
||||
echo
|
||||
echo "-- Now: add the following to CMainParams::m_assumeutxo_data"
|
||||
echo " in src/kernel/chainparams.cpp, and recompile:"
|
||||
echo
|
||||
echo " {.height = ${RPC_BASE_HEIGHT}, .hash_serialized = AssumeutxoHash{uint256{\"${RPC_AU}\"}}, .m_chain_tx_count = ${RPC_NCHAINTX}, .blockhash = consteval_ctor(uint256{\"${RPC_BLOCKHASH}\"})},"
|
||||
echo
|
||||
echo
|
||||
echo "-- IBDing more blocks to the server node (height=$FINAL_HEIGHT) so there is a diff between snapshot and tip..."
|
||||
# shellcheck disable=SC2086
|
||||
./src/bitcoind $SERVER_PORTS -logthreadnames=1 -datadir="$SERVER_DATADIR" \
|
||||
$EARLY_IBD_FLAGS -stopatheight="$FINAL_HEIGHT" >/dev/null
|
||||
|
||||
echo
|
||||
echo "-- Starting the server node to provide blocks to the client node..."
|
||||
# shellcheck disable=SC2086
|
||||
./src/bitcoind $SERVER_PORTS -logthreadnames=1 -debug=net -datadir="$SERVER_DATADIR" \
|
||||
$EARLY_IBD_FLAGS -connect=0 -listen=1 >/dev/null &
|
||||
SERVER_PID="$!"
|
||||
server_sleep_til_boot
|
||||
|
||||
echo
|
||||
echo "-- Okay, what you're about to see is the client starting up and activating the snapshot."
|
||||
echo " I'm going to display the top 14 log lines from the client on top of an RPC called"
|
||||
echo " getchainstates, which is like getblockchaininfo but for both the snapshot and "
|
||||
echo " background validation chainstates."
|
||||
echo
|
||||
echo " You're going to first see the snapshot chainstate sync to the server's tip, then"
|
||||
echo " the background IBD chain kicks in to validate up to the base of the snapshot."
|
||||
echo
|
||||
echo " Once validation of the snapshot is done, you should see log lines indicating"
|
||||
echo " that we've deleted the background validation chainstate."
|
||||
echo
|
||||
echo " Once everything completes, exit the watch command with CTRL+C."
|
||||
echo
|
||||
read -p "When you're ready for all this, hit [enter]" _
|
||||
|
||||
echo
|
||||
echo "-- Starting the client node to get headers from the server, then load the snapshot..."
|
||||
# shellcheck disable=SC2086
|
||||
./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" \
|
||||
-connect=0 -addnode=127.0.0.1:$SERVER_PORT -debug=net $EARLY_IBD_FLAGS >/dev/null &
|
||||
CLIENT_PID="$!"
|
||||
client_sleep_til_boot
|
||||
|
||||
echo
|
||||
echo "-- Initial state of the client:"
|
||||
client_rpc getchainstates
|
||||
|
||||
echo
|
||||
echo "-- Loading UTXO snapshot into client. Calling RPC in a loop..."
|
||||
while ! client_rpc loadtxoutset "$UTXO_DAT_FILE" ; do sleep 10; done
|
||||
|
||||
watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
|
||||
|
||||
echo
|
||||
echo "-- Okay, now I'm going to restart the client to make sure that the snapshot chain reloads "
|
||||
echo " as the main chain properly..."
|
||||
echo
|
||||
echo " Press CTRL+C after you're satisfied to exit the demo"
|
||||
echo
|
||||
read -p "Press [enter] to continue"
|
||||
|
||||
client_sleep_til_boot
|
||||
# shellcheck disable=SC2086
|
||||
./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" -connect=0 \
|
||||
-addnode=127.0.0.1:$SERVER_PORT "$EARLY_IBD_FLAGS" >/dev/null &
|
||||
CLIENT_PID="$!"
|
||||
client_sleep_til_boot
|
||||
|
||||
watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
|
||||
|
||||
echo
|
||||
echo "-- Done!"
|
||||
@@ -1,104 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Copyright (c) 2019-2023 The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
#
|
||||
export LC_ALL=C
|
||||
|
||||
set -ueo pipefail
|
||||
|
||||
NETWORK_DISABLED=false
|
||||
|
||||
if (( $# < 3 )); then
|
||||
echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <bitcoin-cli-call ...>'
|
||||
echo
|
||||
echo " if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the "
|
||||
echo " expected assumeutxo hash"
|
||||
echo
|
||||
echo 'Examples:'
|
||||
echo
|
||||
echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata"
|
||||
echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GENERATE_AT_HEIGHT="${1}"; shift;
|
||||
OUTPUT_PATH="${1}"; shift;
|
||||
# Most of the calls we make take a while to run, so pad with a lengthy timeout.
|
||||
BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999"
|
||||
|
||||
# Check if the node is pruned and get the pruned block height
|
||||
PRUNED=$( ${BITCOIN_CLI_CALL} getblockchaininfo | awk '/pruneheight/ {print $2}' | tr -d ',' )
|
||||
|
||||
if (( GENERATE_AT_HEIGHT < PRUNED )); then
|
||||
echo "Error: The requested snapshot height (${GENERATE_AT_HEIGHT}) should be greater than the pruned block height (${PRUNED})."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check current block height to ensure the node has synchronized past the required block
|
||||
CURRENT_BLOCK_HEIGHT=$(${BITCOIN_CLI_CALL} getblockcount)
|
||||
PIVOT_BLOCK_HEIGHT=$(( GENERATE_AT_HEIGHT + 1 ))
|
||||
|
||||
if (( PIVOT_BLOCK_HEIGHT > CURRENT_BLOCK_HEIGHT )); then
|
||||
(>&2 echo "Error: The node has not yet synchronized to block height ${PIVOT_BLOCK_HEIGHT}.")
|
||||
(>&2 echo "Please wait until the node has synchronized past this block height and try again.")
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Early exit if file at OUTPUT_PATH already exists
|
||||
if [[ -e "$OUTPUT_PATH" ]]; then
|
||||
(>&2 echo "Error: $OUTPUT_PATH already exists or is not a valid path.")
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Validate that the path is correct
|
||||
if [[ "${OUTPUT_PATH}" != "-" && ! -d "$(dirname "${OUTPUT_PATH}")" ]]; then
|
||||
(>&2 echo "Error: The directory $(dirname "${OUTPUT_PATH}") does not exist.")
|
||||
exit 1
|
||||
fi
|
||||
|
||||
function cleanup {
|
||||
(>&2 echo "Restoring chain to original height; this may take a while")
|
||||
${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}"
|
||||
|
||||
if $NETWORK_DISABLED; then
|
||||
(>&2 echo "Restoring network activity")
|
||||
${BITCOIN_CLI_CALL} setnetworkactive true
|
||||
fi
|
||||
}
|
||||
|
||||
function early_exit {
|
||||
(>&2 echo "Exiting due to Ctrl-C")
|
||||
cleanup
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Prompt the user to disable network activity
|
||||
read -p "Do you want to disable network activity (setnetworkactive false) before running invalidateblock? (Y/n): " -r
|
||||
if [[ "$REPLY" =~ ^[Yy]*$ || -z "$REPLY" ]]; then
|
||||
# User input is "Y", "y", or Enter key, proceed with the action
|
||||
NETWORK_DISABLED=true
|
||||
(>&2 echo "Disabling network activity")
|
||||
${BITCOIN_CLI_CALL} setnetworkactive false
|
||||
else
|
||||
(>&2 echo "Network activity remains enabled")
|
||||
fi
|
||||
|
||||
# Block we'll invalidate/reconsider to rewind/fast-forward the chain.
|
||||
PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) )
|
||||
|
||||
# Trap for normal exit and Ctrl-C
|
||||
trap cleanup EXIT
|
||||
trap early_exit INT
|
||||
|
||||
(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while")
|
||||
${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}"
|
||||
|
||||
if [[ "${OUTPUT_PATH}" = "-" ]]; then
|
||||
(>&2 echo "Generating txoutset info...")
|
||||
${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_3 | sed 's/^.*: "\(.\+\)\+",/\1/g'
|
||||
else
|
||||
(>&2 echo "Generating UTXO snapshot...")
|
||||
${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}"
|
||||
fi
|
||||
Reference in New Issue
Block a user