Merge bitcoin/bitcoin#33322: Update libmultiprocess subtree to improve build and logs

a334bbe9b7 Squashed 'src/ipc/libmultiprocess/' changes from 1b8d4a6f1e54..13424cf2ecc1 (Ryan Ofsky)

Pull request description:

  Includes:

  - https://github.com/bitcoin-core/libmultiprocess/pull/197
  - https://github.com/bitcoin-core/libmultiprocess/pull/202
  - https://github.com/bitcoin-core/libmultiprocess/pull/203
  - https://github.com/bitcoin-core/libmultiprocess/pull/200
  - https://github.com/bitcoin-core/libmultiprocess/pull/205

  These changes should give better feedback when there are build errors, and also make IPC logs more readable.

  The changes can be verified by running `test/lint/git-subtree-check.sh src/ipc/libmultiprocess` as described in [developer notes](https://github.com/bitcoin/bitcoin/blob/master/doc/developer-notes.md#subtrees) and [lint instructions](https://github.com/bitcoin/bitcoin/tree/master/test/lint#git-subtree-checksh)

ACKs for top commit:
  Sjors:
    ACK a4ee70e5b6

Tree-SHA512: ddddd0ed44522ade98a5b94f44b57210794d64f8c378a00438082b8c377f41e9b86c0c5ed29add45472549758f7478ab220af8e268b90b30f57a236c639497d3

src/ipc/libmultiprocess/.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
         uses: vmactions/openbsd-vm@v1
         with:
           prepare: |
-            pkg_add -v cmake ninja git python bash
+            pkg_add -v cmake ninja git bash
           run: |
             git clone --depth=1 https://codeberg.org/OpenBSD/ports.git /usr/ports
           sync: 'rsync'
@@ -34,6 +34,45 @@ jobs:
           cd ${{ github.workspace }}
           CI_CONFIG="ci/configs/openbsd.bash" bash ci/scripts/ci.sh
 
+  build-freebsd:
+    runs-on: ubuntu-latest
+    name: build • freebsd
+    defaults:
+      run:
+        shell: freebsd {0}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Start FreeBSD VM
+        uses: vmactions/freebsd-vm@v1
+        with:
+          prepare: |
+            pkg install -y cmake ninja bash capnproto
+          sync: 'rsync'
+          copyback: false
+
+      - name: Run CI script
+        run: |
+          cd ${{ github.workspace }}
+          CI_CONFIG="ci/configs/freebsd.bash" bash ci/scripts/ci.sh
+
+  build-macos:
+    runs-on: macos-latest
+    name: build • macos
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install dependencies
+        env:
+          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
+        run: |
+          brew install --quiet ninja capnp
+
+      - name: Run CI script
+        run: |
+          CI_CONFIG="ci/configs/macos.bash" bash ci/scripts/ci.sh
+
   build:
     runs-on: ubuntu-latest
 

src/ipc/libmultiprocess/CMakeLists.txt

@@ -13,7 +13,20 @@ endif()
 include("cmake/compat_find.cmake")
 
 find_package(Threads REQUIRED)
-find_package(CapnProto 0.7 REQUIRED)
+find_package(CapnProto 0.7 QUIET NO_MODULE)
+ if(NOT CapnProto_FOUND)
+   message(FATAL_ERROR
+     "Cap'n Proto is required but was not found.\n"
+     "To resolve, choose one of the following:\n"
+     "  - Install Cap'n Proto (version 1.0+ recommended)\n"
+     "  - For Bitcoin Core compilation build with -DENABLE_IPC=OFF to disable multiprocess support\n"
+   )
+ endif()
+
+# Cap'n Proto compatibility checks
+set(CAPNPROTO_ISSUES "")
+set(CAPNPROTO_CVE_AFFECTED FALSE)
+set(CAPNPROTO_CLANG_INCOMPATIBLE FALSE)
 
 # Check for list-of-pointers memory access bug from Nov 2022
 # https://nvd.nist.gov/vuln/detail/CVE-2022-46149
@@ -29,11 +42,43 @@ if(CapnProto_VERSION STREQUAL "0.7.0"
    OR CapnProto_VERSION STREQUAL "0.10.0"
    OR CapnProto_VERSION STREQUAL "0.10.1"
    OR CapnProto_VERSION STREQUAL "0.10.2")
+  set(CAPNPROTO_CVE_AFFECTED TRUE)
+  string(APPEND CAPNPROTO_ISSUES "- CVE-2022-46149 security vulnerability (details: https://github.com/advisories/GHSA-qqff-4vw4-f6hx)\n")
+endif()
+
+# Check for Cap'n Proto / Clang / C++20 incompatibility
+# Cap'n Proto 0.9.x and 0.10.x are incompatible with Clang 16+ when using C++20
+# due to P2468R2 implementation. This was fixed in Cap'n Proto 1.0+.
+# See: https://github.com/bitcoin-core/libmultiprocess/issues/199
+if((CapnProto_VERSION VERSION_GREATER_EQUAL "0.9.0") AND
+   (CapnProto_VERSION VERSION_LESS "1.0.0") AND
+   (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") AND
+   (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "16") AND
+   (CMAKE_CXX_STANDARD EQUAL 20))
+  set(CAPNPROTO_CLANG_INCOMPATIBLE TRUE)
+  string(APPEND CAPNPROTO_ISSUES "- Incompatible with Clang ${CMAKE_CXX_COMPILER_VERSION} when using C++20\n")
+endif()
+
+if(CAPNPROTO_CVE_AFFECTED OR CAPNPROTO_CLANG_INCOMPATIBLE)
+  set(RESOLUTION_OPTIONS "")
+
+  # Fixes both issues
+  string(APPEND RESOLUTION_OPTIONS "  - Upgrade to Cap'n Proto version 1.0 or newer (recommended)\n")
+
+  if(CAPNPROTO_CVE_AFFECTED AND NOT CAPNPROTO_CLANG_INCOMPATIBLE)
+    string(APPEND RESOLUTION_OPTIONS "  - Upgrade to a patched minor version (0.7.1, 0.8.1, 0.9.2, 0.10.3, or later)\n")
+  elseif(CAPNPROTO_CLANG_INCOMPATIBLE AND NOT CAPNPROTO_CVE_AFFECTED)
+    string(APPEND RESOLUTION_OPTIONS "  - Use GCC instead of Clang\n")
+  endif()
+
+  string(APPEND RESOLUTION_OPTIONS "  - For Bitcoin Core compilation build with -DENABLE_IPC=OFF to disable multiprocess support\n")
+
   message(FATAL_ERROR
-    "Cap'n Proto ${CapnProto_VERSION} is affected by CVE-2022-46149.\n"
+    "The version of Cap'n Proto detected: ${CapnProto_VERSION} has known compatibility issues:\n"
-    "Please install an updated package.\n"
+    "${CAPNPROTO_ISSUES}"
-    "Details: https://github.com/advisories/GHSA-qqff-4vw4-f6hx
+    "To resolve, choose one of the following:\n"
-  ")
+    "${RESOLUTION_OPTIONS}"
+  )
 endif()
 
 set(MPGEN_EXECUTABLE "" CACHE FILEPATH "If specified, should be full path to an external mpgen binary to use rather than the one built internally.")

src/ipc/libmultiprocess/ci/configs/freebsd.bash

@@ -0,0 +1,5 @@
+CI_DESC="CI config for FreeBSD"
+CI_DIR=build-freebsd
+export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter"
+CMAKE_ARGS=(-G Ninja)
+BUILD_ARGS=(-k 0)

src/ipc/libmultiprocess/ci/configs/macos.bash

@@ -0,0 +1,5 @@
+CI_DESC="CI config for macOS"
+CI_DIR=build-macos
+export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter"
+CMAKE_ARGS=(-G Ninja)
+BUILD_ARGS=(-k 0)

src/ipc/libmultiprocess/doc/usage.md

@@ -19,6 +19,9 @@ A simple interface description can be found at [test/mp/test/foo.capnp](../test/
 A more complete example can be found in [example](../example/) and run with:
 
 ```sh
-make -C build example
+mkdir build
-build/example/mpexample
+cd build
+cmake ..
+make mpexamples
+example/mpexample
 ```

src/ipc/libmultiprocess/include/mp/proxy-io.h

@@ -130,6 +130,16 @@ public:
     std::ostringstream m_buffer;
 };
 
+struct LogOptions {
+
+    //! External logging callback.
+    LogFn log_fn;
+
+    //! Maximum number of characters to use when representing
+    //! request and response structs as strings.
+    size_t max_chars{200};
+};
+
 std::string LongThreadName(const char* exe_name);
 
 //! Event loop implementation.
@@ -204,12 +214,12 @@ public:
 
     Logger log()
     {
-        Logger logger(false, m_log_fn);
+        Logger logger(false, m_log_opts.log_fn);
         logger << "{" << LongThreadName(m_exe_name) << "} ";
         return logger;
     }
-    Logger logPlain() { return {false, m_log_fn}; }
+    Logger logPlain() { return {false, m_log_opts.log_fn}; }
-    Logger raise() { return {true, m_log_fn}; }
+    Logger raise() { return {true, m_log_opts.log_fn}; }
 
     //! Process name included in thread names so combined debug output from
     //! multiple processes is easier to understand.
@@ -255,8 +265,8 @@ public:
     //! List of connections.
     std::list<Connection> m_incoming_connections;
 
-    //! External logging callback.
+    //! Logging options
-    LogFn m_log_fn;
+    LogOptions m_log_opts;
 
     //! External context pointer.
     void* m_context;

src/ipc/libmultiprocess/include/mp/proxy-types.h

@@ -631,13 +631,13 @@ void clientInvoke(ProxyClient& proxy_client, const GetRequest& get_request, Fiel
         IterateFields().handleChain(*invoke_context, request, FieldList(), typename FieldObjs::BuildParams{&fields}...);
         proxy_client.m_context.loop->logPlain()
             << "{" << thread_context.thread_name << "} IPC client send "
-            << TypeName<typename Request::Params>() << " " << LogEscape(request.toString());
+            << TypeName<typename Request::Params>() << " " << LogEscape(request.toString(), proxy_client.m_context.loop->m_log_opts.max_chars);
 
         proxy_client.m_context.loop->m_task_set->add(request.send().then(
             [&](::capnp::Response<typename Request::Results>&& response) {
                 proxy_client.m_context.loop->logPlain()
                     << "{" << thread_context.thread_name << "} IPC client recv "
-                    << TypeName<typename Request::Results>() << " " << LogEscape(response.toString());
+                    << TypeName<typename Request::Results>() << " " << LogEscape(response.toString(), proxy_client.m_context.loop->m_log_opts.max_chars);
                 try {
                     IterateFields().handleChain(
                         *invoke_context, response, FieldList(), typename FieldObjs::ReadResults{&fields}...);
@@ -701,7 +701,7 @@ kj::Promise<void> serverInvoke(Server& server, CallContext& call_context, Fn fn)
 
     int req = ++server_reqs;
     server.m_context.loop->log() << "IPC server recv request  #" << req << " "
-                                     << TypeName<typename Params::Reads>() << " " << LogEscape(params.toString());
+                                     << TypeName<typename Params::Reads>() << " " << LogEscape(params.toString(), server.m_context.loop->m_log_opts.max_chars);
 
     try {
         using ServerContext = ServerInvokeContext<Server, CallContext>;
@@ -718,7 +718,7 @@ kj::Promise<void> serverInvoke(Server& server, CallContext& call_context, Fn fn)
             [&]() { return kj::Promise<CallContext>(kj::mv(call_context)); })
             .then([&server, req](CallContext call_context) {
                 server.m_context.loop->log() << "IPC server send response #" << req << " " << TypeName<Results>()
-                                                 << " " << LogEscape(call_context.getResults().toString());
+                                                 << " " << LogEscape(call_context.getResults().toString(), server.m_context.loop->m_log_opts.max_chars);
             });
     } catch (const std::exception& e) {
         server.m_context.loop->log() << "IPC server unhandled exception: " << e.what();

src/ipc/libmultiprocess/include/mp/util.h

@@ -203,7 +203,7 @@ std::string ThreadName(const char* exe_name);
 
 //! Escape binary string for use in log so it doesn't trigger unicode decode
 //! errors in python unit tests.
-std::string LogEscape(const kj::StringTree& string);
+std::string LogEscape(const kj::StringTree& string, size_t max_size);
 
 //! Callback type used by SpawnProcess below.
 using FdToArgsFn = std::function<std::vector<std::string>(int fd)>;

src/ipc/libmultiprocess/src/mp/proxy.cpp

@@ -187,9 +187,9 @@ EventLoop::EventLoop(const char* exe_name, LogFn log_fn, void* context)
     : m_exe_name(exe_name),
       m_io_context(kj::setupAsyncIo()),
       m_task_set(new kj::TaskSet(m_error_handler)),
-      m_log_fn(std::move(log_fn)),
       m_context(context)
 {
+    m_log_opts.log_fn = log_fn;
     int fds[2];
     KJ_SYSCALL(socketpair(AF_UNIX, SOCK_STREAM, 0, fds));
     m_wait_fd = fds[0];

src/ipc/libmultiprocess/src/mp/util.cpp

@@ -76,12 +76,11 @@ std::string ThreadName(const char* exe_name)
     return std::move(buffer).str();
 }
 
-std::string LogEscape(const kj::StringTree& string)
+std::string LogEscape(const kj::StringTree& string, size_t max_size)
 {
-    const int MAX_SIZE = 1000;
     std::string result;
     string.visit([&](const kj::ArrayPtr<const char>& piece) {
-        if (result.size() > MAX_SIZE) return;
+        if (result.size() > max_size) return;
         for (const char c : piece) {
             if (c == '\\') {
                 result.append("\\\\");
@@ -92,7 +91,7 @@ std::string LogEscape(const kj::StringTree& string)
             } else {
                 result.push_back(c);
             }
-            if (result.size() > MAX_SIZE) {
+            if (result.size() > max_size) {
                 result += "...";
                 break;
             }