diff --git a/contrib/README.md b/contrib/README.md index f23d7ac557b..e53229ffdbc 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -51,3 +51,7 @@ UTXO Set Tools This script converts a compact-serialized UTXO set (as generated by Bitcoin Core with `dumptxoutset`) to a SQLite3 database. For more details like e.g. the created table name and schema, refer to the module docstring on top of the script, which is also contained in the command's `--help` output. + +### [Dump-to-SQLite](/contrib/utxo-tools/dump_to_sqlite.sh) ### +This script creates an UTXO set dump in SQLite3 format on the fly from a running bitcoind instance, +i.e. with the intermediate step of storing the compact-serialized UTXO set on disk is skipped. diff --git a/contrib/utxo-tools/dump_to_sqlite.sh b/contrib/utxo-tools/dump_to_sqlite.sh new file mode 100755 index 00000000000..a25b67fe5d3 --- /dev/null +++ b/contrib/utxo-tools/dump_to_sqlite.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +set -e + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +BITCOIN_CLI=$1 +OUTPUT_FILE=$2 +UTXO_TO_SQLITE=$(dirname "$0")/utxo_to_sqlite.py + +# create named pipe in unique temporary folder +TEMPPATH=$(mktemp -d) +FIFOPATH=$TEMPPATH/utxos.fifo +mkfifo "$FIFOPATH" + +# start dumping UTXO set to the pipe in background +$BITCOIN_CLI dumptxoutset "$FIFOPATH" latest & +BITCOIN_CLI_PID=$! + +# start UTXO to SQLite conversion tool, reading from pipe +$UTXO_TO_SQLITE "$FIFOPATH" "$OUTPUT_FILE" + +# wait and cleanup +wait $BITCOIN_CLI_PID +rm -r "$TEMPPATH" diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index ac1ce6285f7..a5098fbce17 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -3010,11 +3010,13 @@ static RPCHelpMan dumptxoutset() const ArgsManager& args{EnsureAnyArgsman(request.context)}; const fs::path path = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str())); + const auto path_info{fs::status(path)}; // Write to a temporary path and then move into `path` on completion // to avoid confusion due to an interruption. - const fs::path temppath = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str() + ".incomplete")); + const fs::path temppath = fs::is_fifo(path_info) ? path : // If a named pipe is passed, write directly to it + fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str() + ".incomplete")); - if (fs::exists(path)) { + if (fs::exists(path_info) && !fs::is_fifo(path_info)) { throw JSONRPCError( RPC_INVALID_PARAMETER, path.utf8string() + " already exists. If you are sure this is what you want, " @@ -3091,7 +3093,7 @@ static RPCHelpMan dumptxoutset() } UniValue result = WriteUTXOSnapshot(*chainstate, cursor.get(), &stats, tip, afile, path, temppath, node.rpc_interruption_point); - fs::rename(temppath, path); + if (!fs::is_fifo(path_info)) fs::rename(temppath, path); result.pushKV("path", path.utf8string()); return result; diff --git a/src/util/fs.h b/src/util/fs.h index ce17682497c..cf4602ad79c 100644 --- a/src/util/fs.h +++ b/src/util/fs.h @@ -90,6 +90,10 @@ static inline bool exists(const path& p) { return std::filesystem::exists(p); } +static inline bool exists(const std::filesystem::file_status& s) +{ + return std::filesystem::exists(s); +} // Allow explicit quoted stream I/O. static inline auto quoted(const std::string& s) diff --git a/test/functional/tool_utxo_to_sqlite.py b/test/functional/tool_utxo_to_sqlite.py index 2da7c42a86b..7399e7b5745 100755 --- a/test/functional/tool_utxo_to_sqlite.py +++ b/test/functional/tool_utxo_to_sqlite.py @@ -3,11 +3,12 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test utxo-to-sqlite conversion tool""" -import os.path +import os try: import sqlite3 except ImportError: pass +import platform import subprocess import sys @@ -112,6 +113,19 @@ class UtxoToSqliteTest(BitcoinTestFramework): muhash_compact_serialized = node.gettxoutsetinfo('muhash')['muhash'] assert_equal(muhash_sqlite, muhash_compact_serialized) + if platform.system() != "Windows": # FIFOs are not available on Windows + self.log.info('Convert UTXO set directly (without intermediate dump) via named pipe') + fifo_filename = os.path.join(self.options.tmpdir, "utxos.fifo") + os.mkfifo(fifo_filename) + output_direct_filename = os.path.join(self.options.tmpdir, "utxos_direct.sqlite") + p = subprocess.Popen([sys.executable, utxo_to_sqlite_path, fifo_filename, output_direct_filename], + stderr=subprocess.STDOUT) + node.dumptxoutset(fifo_filename, "latest") + p.wait(timeout=10) + muhash_direct_sqlite = calculate_muhash_from_sqlite_utxos(output_direct_filename) + assert_equal(muhash_sqlite, muhash_direct_sqlite) + os.remove(fifo_filename) + if __name__ == "__main__": UtxoToSqliteTest(__file__).main()