Merge bitcoin/bitcoin#31679: cmake: Install internal binaries to <prefix>/libexec/

f49840dd90 doc: Fix typo in files.md (Ryan Ofsky)
f5cf0b1ccc bitcoin wrapper: improve help output (Ryan Ofsky)
c810b168b8 doc: Add description of installed files to files.md (Ryan Ofsky)
94ffd01a02 doc: Add release notes describing libexec/ binaries (Ryan Ofsky)
cd97905ebc cmake: Move internal binaries from bin/ to libexec/ (Ryan Ofsky)

Pull request description:

  This change moves binaries that are not typically invoked directly by users from the `bin/` directory to the `libexec/` directory in CMake installs and binary releases. The goal of the PR is to introduce a distinction between internal and external binaries so starting with #31802, we can use IPC to implement features in new binaries without adding those binaries to the CLI. The change also helps reduce clutter in `bin/`, making it easier for users to identify useful tools to run. Summary of changes:

  - For **source builds** (i.e. developer builds) — There are no changes.
  - For **source installs** (i.e. `cmake --install` result) — `test_bitcoin`, `test_bitcoin-qt`, and `bench_bitcoin` are installed in `${CMAKE_PREFIX_PATH}/libexec` instead of `${CMAKE_PREFIX_PATH}/bin`, so they are no longer on the system `PATH`. However, they can still be invoked from the `libexec/` directory, or from the CLI as `bitcoin test`, `bitcoin test-gui`, and `bitcoin bench`, respectively.
  - For **binary releases** — Since `test_bitcoin` is the only test binary enabled in releases, the only change is moving `test_bitcoin` from `bin/` to `libexec/`.

  <details><summary>Details</summary>
  <p>

   The table below shows the install location of each binary after this change, and the availability of each binary.

  | Binary               | Location     | Availability         | Change                        |
  |----------------------|--------------|----------------------|-------------------------------|
  | `bitcoin`            | `bin/`       | 📦 Binary release (since #31375) | Unchanged                     |
  | `bitcoin-cli`        | `bin/`       | 📦 Binary release     | Unchanged                     |
  | `bitcoind`           | `bin/`       | 📦 Binary release     | Unchanged                     |
  | `bitcoin-qt`         | `bin/`       | 📦 Binary release     | Unchanged                     |
  | `bitcoin-tx`         | `bin/`       | 📦 Binary release     | Unchanged                     |
  | `bitcoin-util`       | `bin/`       | 📦 Binary release     | Unchanged                     |
  | `bitcoin-wallet`     | `bin/`       | 📦 Binary release     | Unchanged                     |
  | `bench_bitcoin`      | `libexec/`   | 🛠 Source build only  | Moved from `bin/`             |
  | `bitcoin-chainstate` | `libexec/`   | 🛠 Source build only  | Newly installed (was built)   |
  | `bitcoin-gui`        | `libexec/`   | 🛠 Source build only (until #31802) | Moved from `bin/`             |
  | `bitcoin-node`       | `libexec/`   | 🛠 Source build only (until #31802) | Moved from `bin/`             |
  | `test_bitcoin`       | `libexec/`    | 📦 Binary release     | Moved from `bin/`             |
  | `test_bitcoin-qt`    | `libexec/`   | 🛠 Source build only  | Moved from `bin/`             |

  </p>
  </details>

  ---

  This PR is part of the [process separation project](https://github.com/bitcoin/bitcoin/issues/28722).

ACKs for top commit:
  l0rinc:
    re-ACK f49840dd90
  Sjors:
    re-ACK f49840dd90
  achow101:
    ACK f49840dd90
  janb84:
    re ACK f49840dd90
  BrandonOdiwuor:
    Tested ACK f49840dd90
  hodlinator:
    re-ACK f49840dd90
  willcl-ark:
    utACK f49840dd90

Tree-SHA512: 858a2e1a53db11ee3c5c759bfdeea566f242b9ce5e8a898fa435222e41662b8184577c0dc2c4c058294b4de41d8cb3ba3e5d24c748c280efa4a3f84e3ec4344d

ci/test/03_test_script.sh

@@ -163,7 +163,7 @@ if [ "$RUN_UNIT_TESTS" = "true" ]; then
 fi
 
 if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
-  DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}"/bin/test_bitcoin --catch_system_errors=no -l test_suite
+  DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_BUILD_DIR}"/bin/test_bitcoin --catch_system_errors=no -l test_suite
 fi
 
 if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then

cmake/module/InstallBinaryComponent.cmake

@@ -7,14 +7,19 @@ include(GNUInstallDirs)
 
 function(install_binary_component component)
   cmake_parse_arguments(PARSE_ARGV 1
-    IC                # prefix
+    IC                          # prefix
-    "HAS_MANPAGE"     # options
+    "HAS_MANPAGE;INTERNAL"      # options
-    ""                # one_value_keywords
+    ""                          # one_value_keywords
-    ""                # multi_value_keywords
+    ""                          # multi_value_keywords
   )
   set(target_name ${component})
+  if(IC_INTERNAL)
+    set(runtime_dest ${CMAKE_INSTALL_LIBEXECDIR})
+  else()
+    set(runtime_dest ${CMAKE_INSTALL_BINDIR})
+  endif()
   install(TARGETS ${target_name}
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    RUNTIME DESTINATION ${runtime_dest}
     COMPONENT ${component}
   )
   if(INSTALL_MAN AND IC_HAS_MANPAGE)

contrib/guix/libexec/codesign.sh

@@ -110,7 +110,7 @@ mkdir -p "$DISTSRC"
 
             # Apply detached codesignatures (in-place)
             signapple apply dist/Bitcoin-Qt.app codesignatures/osx/"${HOST}"/dist/Bitcoin-Qt.app
-            find "${DISTNAME}" -wholename "*/bin/*" -type f | while read -r bin
+            find "${DISTNAME}" \( -wholename "*/bin/*" -o -wholename "*/libexec/*" \) -type f | while read -r bin
             do
                 signapple apply "${bin}" "codesignatures/osx/${HOST}/${bin}.${ARCH}sign"
             done

contrib/macdeploy/detached-sig-create.sh

@@ -44,7 +44,7 @@ ${SIGNAPPLE} apply "${UNSIGNED_BUNDLE}" "${OUTROOT}/${BUNDLE_ROOT}/${BUNDLE_NAME
 ${SIGNAPPLE} notarize --detach "${OUTROOT}/${BUNDLE_ROOT}" --passphrase "${api_key_pass}" "$2" "$3" "${UNSIGNED_BUNDLE}"
 
 # Sign each binary
-find . -maxdepth 3 -wholename "*/bin/*" -type f -exec realpath --relative-to=. {} \; | while read -r bin
+find . -maxdepth 3 \( -wholename "*/bin/*" -o -wholename "*/libexec/*" \) -type f -exec realpath --relative-to=. {} \; | while read -r bin
 do
     bin_dir=$(dirname "${bin}")
     bin_name=$(basename "${bin}")

doc/files.md

@@ -16,7 +16,7 @@
 
   - [Berkeley DB database based wallets](#berkeley-db-database-based-wallets)
 
-- [Notes](#notes)
+- [Installed Files](#installed-files)
 
 ## Data directory location
 
@@ -83,7 +83,7 @@ Wallets are SQLite databases.
 
 3. A wallet database path can be specified with the `-wallet` option.
 
-4. `wallet.dat` files must not be shared across different node instances, as that can result in key-reuse and double-spends due the lack of synchronization between instances.
+4. `wallet.dat` files must not be shared across different node instances, as that can result in key-reuse and double-spends due to the lack of synchronization between instances.
 
 5. Any copy or backup of the wallet should be done through a `backupwallet` call in order to update and lock the wallet, preventing any file corruption caused by updates during the copy.
 
@@ -123,8 +123,40 @@ Subdirectory | File(s)           | Description
 `./`         | `wallet.dat`      | Personal wallet (a BDB database) with keys and transactions
 `./`         | `.walletlock`     | BDB wallet lock file
 
-## Notes
+### Notes
 
 <a name="note1">1</a>. The `/` (slash, U+002F) is used as the platform-independent path component separator in this document.
 
 <a name="note2">2</a>. `NNNNN` matches `[0-9]{5}` regex.
+
+## Installed Files
+
+This table describes the files installed by Bitcoin Core across different platforms.
+
+| **Path**                                                   | **Description**                                                             |
+|------------------------------------------------------------|-----------------------------------------------------------------------------|
+| [README.md](README.md) or [readme.txt](README_windows.txt) | Project information and instructions                                        |
+| bitcoin.conf                                               | [Generated](../contrib/devtools/gen-bitcoin-conf.sh) configuration file     |
+| bin/bitcoin                                                | Command-line tool for interacting with Bitcoin. Calls other binaries below. |
+| bin/bitcoin-cli                                            | Tool for making node and wallet RPC calls.                                  |
+| bin/bitcoin-qt                                             | Bitcoin node and wallet GUI                                                 |
+| bin/bitcoin-tx                                             | Tool for creating and modifying transactions                                |
+| bin/bitcoin-util                                           | Miscellaneous utilities                                                     |
+| bin/bitcoin-wallet                                         | Bitcoin wallet tool                                                         |
+| bin/bitcoind                                               | Bitcoin node and wallet daemon                                              |
+| *lib/libbitcoinkernel.so*                                  | Shared library containing core consensus and validation code                |
+| *lib/pkgconfig/libbitcoinkernel.pc*                        | Pkg-config metadata for linking to `libbitcoinkernel`                       |
+| *libexec/bench_bitcoin*                                    | Benchmarking tool for measuring node performance                            |
+| *libexec/bitcoin-chainstate*                               | Tool to validate and connect blocks                                         |
+| *libexec/bitcoin-gui*                                      | IPC-enabled alternative to `bitcoin-qt`                                     |
+| *libexec/bitcoin-node*                                     | IPC-enabled alternative to `bitcoind`                                       |
+| libexec/test_bitcoin                                       | Unit test binary                                                            |
+| *libexec/test_bitcoin-qt*                                  | GUI-specific unit tests                                                     |
+| share/man/man1/                                            | Man pages for command-line tools like `bitcoin-cli`, `bitcoind`, and others |
+| share/rpcauth/                                             | Documentation and scripts for RPC authentication setup                      |
+
+### Notes
+
+- *Italicized* files are only installed in source builds if relevant CMake options are enabled. They are not included in binary releases.
+- README and bitcoin.conf files are included in binary releases but not installed in source builds.
+- On Windows, binaries have a `.exe` suffix (e.g., `bitcoin-cli.exe`).

doc/release-notes-31375-31679.md

@@ -0,0 +1,24 @@
+New command line interface
+--------------------------
+
+A new `bitcoin` command line tool has been added to make features more
+discoverable and convenient to use. The `bitcoin` tool just calls other
+executables and does not implement any functionality on its own. Specifically
+`bitcoin node` is a synonym for `bitcoind`, `bitcoin gui` is a synonym for
+`bitcoin-qt`, and `bitcoin rpc` is a synonym for `bitcoin-cli -named`. Other
+commands and options can be listed with `bitcoin help`. The new tool does not
+replace other tools, so existing commands should continue working and there are
+no plans to deprecate them.
+
+Install changes
+---------------
+
+The `test_bitcoin` executable is now located in `libexec/` rather than `bin/`.
+It can still be executed directly, or accessed through the new `bitcoin` command
+line tool as `bitcoin test`.
+
+Other executables which are only part of source releases and not built by
+default: `test_bitcoin-qt`, `bench_bitcoin`, `bitcoin-chainstate`,
+`bitcoin-node`, and `bitcoin-gui` are also now installed in `libexec/`
+instead of `bin/` and can be accessed through the `bitcoin` command line tool.
+See `bitcoin help` output for details.

doc/release-notes-31375.md

@@ -1,11 +0,0 @@
-New command line interface
---------------------------
-
-A new `bitcoin` command line tool has been added to make features more
-discoverable and convenient to use. The `bitcoin` tool just calls other
-executables and does not implement any functionality on its own.  Specifically
-`bitcoin node` is a synonym for `bitcoind`, `bitcoin gui` is a synonym for
-`bitcoin-qt`, and `bitcoin rpc` is a synonym for `bitcoin-cli -named`. Other
-commands and options can be listed with `bitcoin help`. The new tool does not
-replace other tools, so all existing commands should continue working and there
-are no plans to deprecate them.

src/CMakeLists.txt

@@ -322,7 +322,7 @@ if(ENABLE_IPC AND BUILD_DAEMON)
     bitcoin_ipc
     $<TARGET_NAME_IF_EXISTS:bitcoin_wallet>
   )
-  install_binary_component(bitcoin-node)
+  install_binary_component(bitcoin-node INTERNAL)
 endif()
 
 if(ENABLE_IPC AND BUILD_TESTS)
@@ -424,6 +424,7 @@ if(BUILD_UTIL_CHAINSTATE)
       core_interface
       bitcoinkernel
   )
+  install_binary_component(bitcoin-chainstate INTERNAL)
 endif()
 
 

src/bench/CMakeLists.txt

@@ -85,4 +85,4 @@ add_test(NAME bench_sanity_check
   COMMAND bench_bitcoin -sanity-check
 )
 
-install_binary_component(bench_bitcoin)
+install_binary_component(bench_bitcoin INTERNAL)

src/bitcoin.cpp

@@ -23,7 +23,7 @@ Options:
   -m, --multiprocess     Run multiprocess binaries bitcoin-node, bitcoin-gui.
   -M, --monolithic       Run monolithic binaries bitcoind, bitcoin-qt. (Default behavior)
   -v, --version          Show version information
-  -h, --help             Show this help message
+  -h, --help             Show full help message
 
 Commands:
   gui [ARGS]     Start GUI, equivalent to running 'bitcoin-qt [ARGS]' or 'bitcoin-gui [ARGS]'.
@@ -31,10 +31,10 @@ Commands:
   rpc [ARGS]     Call RPC method, equivalent to running 'bitcoin-cli -named [ARGS]'.
   wallet [ARGS]  Call wallet command, equivalent to running 'bitcoin-wallet [ARGS]'.
   tx [ARGS]      Manipulate hex-encoded transactions, equivalent to running 'bitcoin-tx [ARGS]'.
-  help [-a]      Show this help message. Include -a or --all to show additional commands.
+  help           Show full help message.
 )";
 
-static constexpr auto HELP_EXTRA = R"(
+static constexpr auto HELP_FULL = R"(
 Additional less commonly used commands:
   bench [ARGS]      Run bench command, equivalent to running 'bench_bitcoin [ARGS]'.
   chainstate [ARGS] Run bitcoin kernel chainstate util, equivalent to running 'bitcoin-chainstate [ARGS]'.
@@ -42,11 +42,14 @@ Additional less commonly used commands:
   test-gui [ARGS]   Run GUI unit tests, equivalent to running 'test_bitcoin-qt [ARGS]'.
 )";
 
+static constexpr auto HELP_SHORT = R"(
+Run '%s help' to see additional commands (e.g. for testing and debugging).
+)";
+
 struct CommandLine {
     bool use_multiprocess{false};
     bool show_version{false};
     bool show_help{false};
-    bool show_help_all{false};
     std::string_view command;
     std::vector<const char*> args;
 };
@@ -63,11 +66,17 @@ int main(int argc, char* argv[])
             return EXIT_SUCCESS;
         }
 
+        std::string exe_name{fs::PathToString(fs::PathFromString(argv[0]).filename())};
         std::vector<const char*> args;
         if (cmd.show_help || cmd.command.empty()) {
-            tfm::format(std::cout, HELP_USAGE, argv[0]);
+            tfm::format(std::cout, HELP_USAGE, exe_name);
-            if (cmd.show_help_all) tfm::format(std::cout, HELP_EXTRA);
+            if (cmd.show_help) {
-            return cmd.show_help ? EXIT_SUCCESS : EXIT_FAILURE;
+                tfm::format(std::cout, HELP_FULL);
+                return EXIT_SUCCESS;
+            } else {
+                tfm::format(std::cout, HELP_SHORT, exe_name);
+                return EXIT_FAILURE;
+            }
         } else if (cmd.command == "gui") {
             args.emplace_back(cmd.use_multiprocess ? "bitcoin-gui" : "bitcoin-qt");
         } else if (cmd.command == "node") {
@@ -125,8 +134,6 @@ CommandLine ParseCommandLine(int argc, char* argv[])
             cmd.show_version = true;
         } else if (arg == "-h" || arg == "--help" || arg == "help") {
             cmd.show_help = true;
-        } else if (cmd.show_help && (arg == "-a" || arg == "--all")) {
-            cmd.show_help_all = true;
         } else if (arg.starts_with("-")) {
             throw std::runtime_error(strprintf("Unknown option: %s", arg));
         } else if (!arg.empty()) {

src/qt/CMakeLists.txt

@@ -279,7 +279,7 @@ if(ENABLE_IPC)
     bitcoin_ipc
   )
   import_plugins(bitcoin-gui)
-  install_binary_component(bitcoin-gui)
+  install_binary_component(bitcoin-gui INTERNAL)
   if(WIN32)
     set_target_properties(bitcoin-gui PROPERTIES WIN32_EXECUTABLE TRUE)
   endif()

src/qt/test/CMakeLists.txt

@@ -45,4 +45,4 @@ if(WIN32 AND VCPKG_TARGET_TRIPLET)
   )
 endif()
 
-install_binary_component(test_bitcoin-qt)
+install_binary_component(test_bitcoin-qt INTERNAL)

src/test/CMakeLists.txt

@@ -214,4 +214,4 @@ endfunction()
 
 add_all_test_targets()
 
-install_binary_component(test_bitcoin)
+install_binary_component(test_bitcoin INTERNAL)