From 790a98aafc485e322ac8c1f5ba11c85bd261a7d1 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Tue, 24 Dec 2024 01:31:55 +0100 Subject: [PATCH 1/3] rpc: support writing UTXO set dump (`dumptxoutset`) to a named pipe This allows external tooling (e.g. converters) to consume the output directly, rather than having to write the dump to disk first and then read it from there again. Co-authored-by: Luke Dashjr --- src/rpc/blockchain.cpp | 8 +++++--- src/util/fs.h | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index ac1ce6285f7..a5098fbce17 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -3010,11 +3010,13 @@ static RPCHelpMan dumptxoutset() const ArgsManager& args{EnsureAnyArgsman(request.context)}; const fs::path path = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str())); + const auto path_info{fs::status(path)}; // Write to a temporary path and then move into `path` on completion // to avoid confusion due to an interruption. - const fs::path temppath = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str() + ".incomplete")); + const fs::path temppath = fs::is_fifo(path_info) ? path : // If a named pipe is passed, write directly to it + fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str() + ".incomplete")); - if (fs::exists(path)) { + if (fs::exists(path_info) && !fs::is_fifo(path_info)) { throw JSONRPCError( RPC_INVALID_PARAMETER, path.utf8string() + " already exists. If you are sure this is what you want, " @@ -3091,7 +3093,7 @@ static RPCHelpMan dumptxoutset() } UniValue result = WriteUTXOSnapshot(*chainstate, cursor.get(), &stats, tip, afile, path, temppath, node.rpc_interruption_point); - fs::rename(temppath, path); + if (!fs::is_fifo(path_info)) fs::rename(temppath, path); result.pushKV("path", path.utf8string()); return result; diff --git a/src/util/fs.h b/src/util/fs.h index f841e0d76c5..56a71b042cd 100644 --- a/src/util/fs.h +++ b/src/util/fs.h @@ -90,6 +90,10 @@ static inline bool exists(const path& p) { return std::filesystem::exists(p); } +static inline bool exists(const std::filesystem::file_status& s) +{ + return std::filesystem::exists(s); +} // Allow explicit quoted stream I/O. static inline auto quoted(const std::string& s) From cfda1d1b2e8c567a8e0ac68ad10411c8e1354d93 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Tue, 24 Dec 2024 01:39:19 +0100 Subject: [PATCH 2/3] test: add test for utxo-to-sqlite conversion using named pipe --- test/functional/tool_utxo_to_sqlite.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/functional/tool_utxo_to_sqlite.py b/test/functional/tool_utxo_to_sqlite.py index 2da7c42a86b..7399e7b5745 100755 --- a/test/functional/tool_utxo_to_sqlite.py +++ b/test/functional/tool_utxo_to_sqlite.py @@ -3,11 +3,12 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test utxo-to-sqlite conversion tool""" -import os.path +import os try: import sqlite3 except ImportError: pass +import platform import subprocess import sys @@ -112,6 +113,19 @@ class UtxoToSqliteTest(BitcoinTestFramework): muhash_compact_serialized = node.gettxoutsetinfo('muhash')['muhash'] assert_equal(muhash_sqlite, muhash_compact_serialized) + if platform.system() != "Windows": # FIFOs are not available on Windows + self.log.info('Convert UTXO set directly (without intermediate dump) via named pipe') + fifo_filename = os.path.join(self.options.tmpdir, "utxos.fifo") + os.mkfifo(fifo_filename) + output_direct_filename = os.path.join(self.options.tmpdir, "utxos_direct.sqlite") + p = subprocess.Popen([sys.executable, utxo_to_sqlite_path, fifo_filename, output_direct_filename], + stderr=subprocess.STDOUT) + node.dumptxoutset(fifo_filename, "latest") + p.wait(timeout=10) + muhash_direct_sqlite = calculate_muhash_from_sqlite_utxos(output_direct_filename) + assert_equal(muhash_sqlite, muhash_direct_sqlite) + os.remove(fifo_filename) + if __name__ == "__main__": UtxoToSqliteTest(__file__).main() From 4c8e9b4f35be4104002ece15b6f72c360756f5d9 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Tue, 24 Dec 2024 01:50:20 +0100 Subject: [PATCH 3/3] contrib: add script dump_to_sqlite.sh for direct SQLite3 UTXO dump --- contrib/README.md | 4 ++++ contrib/utxo-tools/dump_to_sqlite.sh | 32 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100755 contrib/utxo-tools/dump_to_sqlite.sh diff --git a/contrib/README.md b/contrib/README.md index f23d7ac557b..e53229ffdbc 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -51,3 +51,7 @@ UTXO Set Tools This script converts a compact-serialized UTXO set (as generated by Bitcoin Core with `dumptxoutset`) to a SQLite3 database. For more details like e.g. the created table name and schema, refer to the module docstring on top of the script, which is also contained in the command's `--help` output. + +### [Dump-to-SQLite](/contrib/utxo-tools/dump_to_sqlite.sh) ### +This script creates an UTXO set dump in SQLite3 format on the fly from a running bitcoind instance, +i.e. with the intermediate step of storing the compact-serialized UTXO set on disk is skipped. diff --git a/contrib/utxo-tools/dump_to_sqlite.sh b/contrib/utxo-tools/dump_to_sqlite.sh new file mode 100755 index 00000000000..a25b67fe5d3 --- /dev/null +++ b/contrib/utxo-tools/dump_to_sqlite.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +set -e + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +BITCOIN_CLI=$1 +OUTPUT_FILE=$2 +UTXO_TO_SQLITE=$(dirname "$0")/utxo_to_sqlite.py + +# create named pipe in unique temporary folder +TEMPPATH=$(mktemp -d) +FIFOPATH=$TEMPPATH/utxos.fifo +mkfifo "$FIFOPATH" + +# start dumping UTXO set to the pipe in background +$BITCOIN_CLI dumptxoutset "$FIFOPATH" latest & +BITCOIN_CLI_PID=$! + +# start UTXO to SQLite conversion tool, reading from pipe +$UTXO_TO_SQLITE "$FIFOPATH" "$OUTPUT_FILE" + +# wait and cleanup +wait $BITCOIN_CLI_PID +rm -r "$TEMPPATH"