Merge 4c8e9b4f35be4104002ece15b6f72c360756f5d9 into db2c57ae9eebdb75c58cd165ac929919969c19a9

contrib/README.md

@@ -51,3 +51,7 @@ UTXO Set Tools
 This script converts a compact-serialized UTXO set (as generated by Bitcoin Core with `dumptxoutset`)
 to a SQLite3 database. For more details like e.g. the created table name and schema, refer to the
 module docstring on top of the script, which is also contained in the command's `--help` output.
+
+### [Dump-to-SQLite](/contrib/utxo-tools/dump_to_sqlite.sh) ###
+This script creates an UTXO set dump in SQLite3 format on the fly from a running bitcoind instance,
+i.e. with the intermediate step of storing the compact-serialized UTXO set on disk is skipped.

contrib/utxo-tools/dump_to_sqlite.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Copyright (c) 2024-present The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C
+set -e
+
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 <bitcoin-cli-path> <output-file>"
+    exit 1
+fi
+
+BITCOIN_CLI=$1
+OUTPUT_FILE=$2
+UTXO_TO_SQLITE=$(dirname "$0")/utxo_to_sqlite.py
+
+# create named pipe in unique temporary folder
+TEMPPATH=$(mktemp -d)
+FIFOPATH=$TEMPPATH/utxos.fifo
+mkfifo "$FIFOPATH"
+
+# start dumping UTXO set to the pipe in background
+$BITCOIN_CLI dumptxoutset "$FIFOPATH" latest &
+BITCOIN_CLI_PID=$!
+
+# start UTXO to SQLite conversion tool, reading from pipe
+$UTXO_TO_SQLITE "$FIFOPATH" "$OUTPUT_FILE"
+
+# wait and cleanup
+wait $BITCOIN_CLI_PID
+rm -r "$TEMPPATH"

src/rpc/blockchain.cpp

@@ -3010,11 +3010,13 @@ static RPCHelpMan dumptxoutset()
 
     const ArgsManager& args{EnsureAnyArgsman(request.context)};
     const fs::path path = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str()));
+    const auto path_info{fs::status(path)};
     // Write to a temporary path and then move into `path` on completion
     // to avoid confusion due to an interruption.
-    const fs::path temppath = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str() + ".incomplete"));
+    const fs::path temppath = fs::is_fifo(path_info) ? path : // If a named pipe is passed, write directly to it
+        fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str() + ".incomplete"));
 
-    if (fs::exists(path)) {
+    if (fs::exists(path_info) && !fs::is_fifo(path_info)) {
         throw JSONRPCError(
             RPC_INVALID_PARAMETER,
             path.utf8string() + " already exists. If you are sure this is what you want, "
@@ -3091,7 +3093,7 @@ static RPCHelpMan dumptxoutset()
     }
 
     UniValue result = WriteUTXOSnapshot(*chainstate, cursor.get(), &stats, tip, afile, path, temppath, node.rpc_interruption_point);
-    fs::rename(temppath, path);
+    if (!fs::is_fifo(path_info)) fs::rename(temppath, path);
 
     result.pushKV("path", path.utf8string());
     return result;

src/util/fs.h

@@ -90,6 +90,10 @@ static inline bool exists(const path& p)
 {
     return std::filesystem::exists(p);
 }
+static inline bool exists(const std::filesystem::file_status& s)
+{
+    return std::filesystem::exists(s);
+}
 
 // Allow explicit quoted stream I/O.
 static inline auto quoted(const std::string& s)

test/functional/tool_utxo_to_sqlite.py

@@ -3,11 +3,12 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test utxo-to-sqlite conversion tool"""
-import os.path
+import os
 try:
     import sqlite3
 except ImportError:
     pass
+import platform
 import subprocess
 import sys
 
@@ -112,6 +113,19 @@ class UtxoToSqliteTest(BitcoinTestFramework):
         muhash_compact_serialized = node.gettxoutsetinfo('muhash')['muhash']
         assert_equal(muhash_sqlite, muhash_compact_serialized)
 
+        if platform.system() != "Windows":  # FIFOs are not available on Windows
+            self.log.info('Convert UTXO set directly (without intermediate dump) via named pipe')
+            fifo_filename = os.path.join(self.options.tmpdir, "utxos.fifo")
+            os.mkfifo(fifo_filename)
+            output_direct_filename = os.path.join(self.options.tmpdir, "utxos_direct.sqlite")
+            p = subprocess.Popen([sys.executable, utxo_to_sqlite_path, fifo_filename, output_direct_filename],
+                                 stderr=subprocess.STDOUT)
+            node.dumptxoutset(fifo_filename, "latest")
+            p.wait(timeout=10)
+            muhash_direct_sqlite = calculate_muhash_from_sqlite_utxos(output_direct_filename)
+            assert_equal(muhash_sqlite, muhash_direct_sqlite)
+            os.remove(fifo_filename)
+
 
 if __name__ == "__main__":
     UtxoToSqliteTest(__file__).main()