From a2f28e4be96e92079a219567cf20214996aefc53 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Wed, 2 Apr 2025 21:41:16 +0800 Subject: [PATCH] Squashed 'src/ipc/libmultiprocess/' content from commit 35944ffd23fa git-subtree-dir: src/ipc/libmultiprocess git-subtree-split: 35944ffd23fa26652b82210351d50e896ce16c8f --- .clang-tidy | 41 ++ CMakeLists.txt | 178 ++++++++ COPYING | 22 + README.md | 8 + cmake/Config.cmake.in | 33 ++ cmake/TargetCapnpSources.cmake | 113 +++++ cmake/compat_config.cmake | 59 +++ cmake/compat_find.cmake | 20 + cmake/pthread_checks.cmake | 41 ++ doc/design.md | 44 ++ doc/install.md | 52 +++ doc/usage.md | 24 ++ example/CMakeLists.txt | 29 ++ example/calculator.capnp | 16 + example/calculator.cpp | 58 +++ example/calculator.h | 17 + example/example.cpp | 72 ++++ example/init.capnp | 22 + example/init.h | 20 + example/printer.capnp | 16 + example/printer.cpp | 51 +++ example/printer.h | 17 + example/types.h | 14 + include/mp/config.h.in | 16 + include/mp/proxy-io.h | 641 ++++++++++++++++++++++++++++ include/mp/proxy-types.h | 727 ++++++++++++++++++++++++++++++++ include/mp/proxy.capnp | 65 +++ include/mp/proxy.h | 299 +++++++++++++ include/mp/type-char.h | 36 ++ include/mp/type-chrono.h | 34 ++ include/mp/type-context.h | 173 ++++++++ include/mp/type-data.h | 46 ++ include/mp/type-decay.h | 38 ++ include/mp/type-exception.h | 22 + include/mp/type-function.h | 67 +++ include/mp/type-interface.h | 112 +++++ include/mp/type-map.h | 52 +++ include/mp/type-message.h | 64 +++ include/mp/type-number.h | 87 ++++ include/mp/type-optional.h | 48 +++ include/mp/type-pair.h | 50 +++ include/mp/type-pointer.h | 113 +++++ include/mp/type-set.h | 48 +++ include/mp/type-string.h | 34 ++ include/mp/type-struct.h | 85 ++++ include/mp/type-threadmap.h | 41 ++ include/mp/type-tuple.h | 45 ++ include/mp/type-vector.h | 71 ++++ include/mp/type-void.h | 23 + include/mp/util.h | 220 ++++++++++ include/mpgen.mk | 4 + pkgconfig/libmultiprocess.pc.in | 12 + src/mp/gen.cpp | 671 +++++++++++++++++++++++++++++ src/mp/proxy.cpp | 399 ++++++++++++++++++ src/mp/util.cpp | 154 +++++++ test/CMakeLists.txt | 39 ++ test/mp/test/foo-types.h | 83 ++++ test/mp/test/foo.capnp | 67 +++ test/mp/test/foo.h | 84 ++++ test/mp/test/test.cpp | 141 +++++++ 60 files changed, 5878 insertions(+) create mode 100644 .clang-tidy create mode 100644 CMakeLists.txt create mode 100644 COPYING create mode 100644 README.md create mode 100644 cmake/Config.cmake.in create mode 100644 cmake/TargetCapnpSources.cmake create mode 100644 cmake/compat_config.cmake create mode 100644 cmake/compat_find.cmake create mode 100644 cmake/pthread_checks.cmake create mode 100644 doc/design.md create mode 100644 doc/install.md create mode 100644 doc/usage.md create mode 100644 example/CMakeLists.txt create mode 100644 example/calculator.capnp create mode 100644 example/calculator.cpp create mode 100644 example/calculator.h create mode 100644 example/example.cpp create mode 100644 example/init.capnp create mode 100644 example/init.h create mode 100644 example/printer.capnp create mode 100644 example/printer.cpp create mode 100644 example/printer.h create mode 100644 example/types.h create mode 100644 include/mp/config.h.in create mode 100644 include/mp/proxy-io.h create mode 100644 include/mp/proxy-types.h create mode 100644 include/mp/proxy.capnp create mode 100644 include/mp/proxy.h create mode 100644 include/mp/type-char.h create mode 100644 include/mp/type-chrono.h create mode 100644 include/mp/type-context.h create mode 100644 include/mp/type-data.h create mode 100644 include/mp/type-decay.h create mode 100644 include/mp/type-exception.h create mode 100644 include/mp/type-function.h create mode 100644 include/mp/type-interface.h create mode 100644 include/mp/type-map.h create mode 100644 include/mp/type-message.h create mode 100644 include/mp/type-number.h create mode 100644 include/mp/type-optional.h create mode 100644 include/mp/type-pair.h create mode 100644 include/mp/type-pointer.h create mode 100644 include/mp/type-set.h create mode 100644 include/mp/type-string.h create mode 100644 include/mp/type-struct.h create mode 100644 include/mp/type-threadmap.h create mode 100644 include/mp/type-tuple.h create mode 100644 include/mp/type-vector.h create mode 100644 include/mp/type-void.h create mode 100644 include/mp/util.h create mode 100644 include/mpgen.mk create mode 100644 pkgconfig/libmultiprocess.pc.in create mode 100644 src/mp/gen.cpp create mode 100644 src/mp/proxy.cpp create mode 100644 src/mp/util.cpp create mode 100644 test/CMakeLists.txt create mode 100644 test/mp/test/foo-types.h create mode 100644 test/mp/test/foo.capnp create mode 100644 test/mp/test/foo.h create mode 100644 test/mp/test/test.cpp diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000000..2d29f120ae4 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,41 @@ +Checks: ' +-*, +bugprone-*, +-bugprone-easily-swappable-parameters, +-bugprone-exception-escape, +-bugprone-move-forwarding-reference, +-bugprone-narrowing-conversions, +-bugprone-reserved-identifier, +misc-*, +-misc-non-private-member-variables-in-classes, +-misc-no-recursion, +-misc-unconventional-assign-operator, +-misc-unused-parameters, +-misc-use-anonymous-namespace, +modernize-*, +-modernize-avoid-c-arrays, +-modernize-concat-nested-namespaces, +-modernize-deprecated-headers, +-modernize-use-nodiscard, +-modernize-use-trailing-return-type, +-modernize-use-using, +performance-*, +-performance-avoid-endl, +-performance-noexcept-move-constructor, +readability-*, +-readability-braces-around-statements, +-readability-convert-member-functions-to-static, +-readability-else-after-return, +-readability-function-cognitive-complexity, +-readability-identifier-length, +-readability-implicit-bool-conversion, +-readability-inconsistent-declaration-parameter-name, +-readability-magic-numbers, +-readability-named-parameter, +-readability-uppercase-literal-suffix, +-readability-use-anyofallof, +' +CheckOptions: + - key: modernize-use-override.IgnoreDestructors + value: true +HeaderFilterRegex: 'example/calculator.h|example/init.h|example/printer.h|include/mp/proxy-io.h|include/mp/proxy-types.h|include/mp/proxy.h|include/mp/util.h|test/mp/test/foo-types.h|test/mp/test/foo.h' diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000000..5ade99338ce --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,178 @@ +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +cmake_minimum_required(VERSION 3.12) + +project("Libmultiprocess" CXX) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(CMAKE_CXX_STANDARD 20) + set(CMAKE_CXX_STANDARD_REQUIRED YES) +endif() + +include("cmake/compat_find.cmake") + +find_package(CapnProto REQUIRED) +find_package(Threads REQUIRED) + +option(Libmultiprocess_ENABLE_CLANG_TIDY "Run clang-tidy with the compiler." OFF) +if(Libmultiprocess_ENABLE_CLANG_TIDY) + find_program(CLANG_TIDY_EXECUTABLE NAMES clang-tidy) + if(NOT CLANG_TIDY_EXECUTABLE) + message(FATAL_ERROR "Libmultiprocess_ENABLE_CLANG_TIDY is ON but clang-tidy is not found.") + endif() + set(CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_EXECUTABLE}") +endif() + +set(MPGEN_EXECUTABLE "" CACHE FILEPATH "If specified, should be full path to an external mpgen binary to use rather than the one built internally.") + +include("cmake/compat_config.cmake") +include("cmake/pthread_checks.cmake") +include(GNUInstallDirs) + +# Set MP_INCLUDE_DIR as a global property so target_capnp_sources function can +# use it, and its callers don't need to specify the include directory manually +# to avoid "error: Import failed: /mp/proxy.capnp" failures from capnproto. +set_property(GLOBAL PROPERTY MP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") + +# Set a convenience variable for subdirectories. +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(MP_STANDALONE TRUE) + include(CTest) +else() + set(MP_STANDALONE FALSE) +endif() + +# Prevent include directories from parent project from leaking into this one. +set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "") + +# Generated C++ preprocessor defines +configure_file(include/mp/config.h.in "${CMAKE_CURRENT_BINARY_DIR}/include/mp/config.h") + +# Generated C++ Capn'Proto schema files +capnp_generate_cpp(MP_PROXY_SRCS MP_PROXY_HDRS include/mp/proxy.capnp) + +# util library +add_library(mputil OBJECT src/mp/util.cpp) +target_include_directories(mputil PRIVATE + $ + $) +target_link_libraries(mputil PUBLIC CapnProto::kj) + +# libmultiprocess.a runtime library +set(MP_PUBLIC_HEADERS + ${MP_PROXY_HDRS} + include/mp/proxy-io.h + include/mp/proxy-types.h + include/mp/proxy.h + include/mp/type-char.h + include/mp/type-chrono.h + include/mp/type-context.h + include/mp/type-data.h + include/mp/type-decay.h + include/mp/type-exception.h + include/mp/type-function.h + include/mp/type-interface.h + include/mp/type-map.h + include/mp/type-message.h + include/mp/type-number.h + include/mp/type-optional.h + include/mp/type-pair.h + include/mp/type-pointer.h + include/mp/type-set.h + include/mp/type-string.h + include/mp/type-struct.h + include/mp/type-threadmap.h + include/mp/type-tuple.h + include/mp/type-vector.h + include/mp/type-void.h + include/mp/util.h) +add_library(multiprocess STATIC + ${MP_PROXY_SRCS} + ${MP_PUBLIC_HEADERS} + src/mp/proxy.cpp + $) +add_library(Libmultiprocess::multiprocess ALIAS multiprocess) +target_include_directories(multiprocess PUBLIC + $ + $ + $) +target_link_libraries(multiprocess PUBLIC CapnProto::capnp) +target_link_libraries(multiprocess PUBLIC CapnProto::capnp-rpc) +target_link_libraries(multiprocess PUBLIC CapnProto::kj) +target_link_libraries(multiprocess PUBLIC CapnProto::kj-async) +set_target_properties(multiprocess PROPERTIES + PUBLIC_HEADER "${MP_PUBLIC_HEADERS}") +install(TARGETS multiprocess EXPORT LibTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT lib + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/mp COMPONENT lib) + +# mpgen code generator +add_executable(mpgen src/mp/gen.cpp $) +add_executable(Libmultiprocess::mpgen ALIAS mpgen) +target_include_directories(mpgen PRIVATE $) +target_include_directories(mpgen PUBLIC $ $) +target_link_libraries(mpgen PRIVATE CapnProto::capnp) +target_link_libraries(mpgen PRIVATE CapnProto::capnp-rpc) +target_link_libraries(mpgen PRIVATE CapnProto::capnpc) +target_link_libraries(mpgen PRIVATE CapnProto::kj) +target_link_libraries(mpgen PRIVATE Threads::Threads) +set_target_properties(mpgen PROPERTIES + INSTALL_RPATH_USE_LINK_PATH TRUE) +set_target_properties(mpgen PROPERTIES + PUBLIC_HEADER include/mp/proxy.capnp) +install(TARGETS mpgen EXPORT BinTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT bin + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/mp COMPONENT bin) + +# makefile include to invoke mpgen code generator, for downstream Make projects +install(FILES "include/mpgen.mk" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT bin) + +# pkg-config module to build against libmultiprocess library, for downstream autoconf projects +configure_file(pkgconfig/libmultiprocess.pc.in "${CMAKE_CURRENT_BINARY_DIR}/pkgconfig/libmultiprocess.pc" @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pkgconfig/libmultiprocess.pc" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig COMPONENT lib) + +# cmake include to invoke mpgen code generator, for downstream CMake projects +install( + FILES + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/TargetCapnpSources.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess COMPONENT bin) + +# CMake target import files, for downstream CMake projects +install(EXPORT BinTargets + NAMESPACE Libmultiprocess:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess COMPONENT bin) +install(EXPORT LibTargets + NAMESPACE Libmultiprocess:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess COMPONENT lib) + +# CMake find_package config file, for downstream CMake projects +include(CMakePackageConfigHelpers) +configure_package_config_file( + ${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in + LibmultiprocessConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess + NO_SET_AND_CHECK_MACRO) +install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/LibmultiprocessConfig.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess + COMPONENT common) + +# Makefile targets to support "make install-bin" "make install-lib" +add_custom_target(install-bin + COMMAND ${CMAKE_COMMAND} -DCOMPONENT=bin -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake + COMMAND ${CMAKE_COMMAND} -DCOMPONENT=common -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake + VERBATIM) +add_dependencies(install-bin mpgen) +add_custom_target(install-lib + COMMAND ${CMAKE_COMMAND} -DCOMPONENT=lib -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake + COMMAND ${CMAKE_COMMAND} -DCOMPONENT=common -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake + VERBATIM) +add_dependencies(install-lib multiprocess) + +# Example and test subdirectories +add_subdirectory(example EXCLUDE_FROM_ALL) +add_subdirectory(test EXCLUDE_FROM_ALL) diff --git a/COPYING b/COPYING new file mode 100644 index 00000000000..9d54ecbde12 --- /dev/null +++ b/COPYING @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2009-2019 The Bitcoin Core developers +Copyright (c) 2009-2019 Bitcoin Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000000..9a6e65795b3 --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# libmultiprocess + +`libmultiprocess` is a C++ library and code generator making it easy to call functions and reference objects in different processes. + +For more information see the [usage instructions](doc/usage.md), [installation instructions](doc/install.md), or [design documentation](doc/design.md). + +If you have any questions, comments, or feedback, please submit an [issue](https://github.com/bitcoin-core/libmultiprocess/issues/new). +Duplicate issues are perfectly fine and all discussion about the project is welcome, since there isn't another discussion forum currently. diff --git a/cmake/Config.cmake.in b/cmake/Config.cmake.in new file mode 100644 index 00000000000..edff7d143be --- /dev/null +++ b/cmake/Config.cmake.in @@ -0,0 +1,33 @@ +@PACKAGE_INIT@ + +# CMake find_package compatible package file, for downstream CMake projects +# +# Based on https://cmake.org/cmake/help/latest/guide/importing-exporting/index.html#adding-components + +set(_Libmultiprocess_supported_components Bin Lib) + +# If no components specified, include all components. +list(LENGTH ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS_len) +if(${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS_len EQUAL 0) + set(${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS ${_Libmultiprocess_supported_components}) +endif() + +if ("Bin" IN_LIST ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + include("${CMAKE_CURRENT_LIST_DIR}/TargetCapnpSources.cmake") +endif() + +if ("Lib" IN_LIST ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + # Setting FOUND_LIBATOMIC is needed on debian & ubuntu systems to work around bug in + # their capnproto packages. See compat_find.cmake for a more complete explanation. + set(FOUND_LIBATOMIC TRUE) + include(CMakeFindDependencyMacro) + find_dependency(CapnProto) +endif() + +foreach(_comp ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS}) + if (NOT _comp IN_LIST _Libmultiprocess_supported_components) + set(${CMAKE_FIND_PACKAGE_NAME}_FOUND False) + set(${CMAKE_FIND_PACKAGE_NAME}_NOT_FOUND_MESSAGE "Unsupported component: ${_comp}") + endif() + include("${CMAKE_CURRENT_LIST_DIR}/${_comp}Targets.cmake") +endforeach() diff --git a/cmake/TargetCapnpSources.cmake b/cmake/TargetCapnpSources.cmake new file mode 100644 index 00000000000..cf7d20feb96 --- /dev/null +++ b/cmake/TargetCapnpSources.cmake @@ -0,0 +1,113 @@ +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +#[=[ + +target_capnp_sources +-------------------- + +This function adds build steps to generate C++ files from Cap'n Proto files +and build them as part of a specified target. + +Arguments: + + target: The name of the CMake target (e.g., a library or executable) to + which the generated source files will be added. This target must already + be defined elsewhere in the CMake scripts. + + include_prefix: Absolute path indicating what portion of capnp source paths + should be used in relative #include statements in the generated C++ + files. For example, if the .capnp path is /home/src/lib/schema.capnp + and include_prefix is /home/src, generated includes look like: + + #include + + And if include_prefix is /home/src/lib, generated includes look like: + + #include + + The specified include_prefix should be ${CMAKE_SOURCE_DIR} or a + subdirectory of it to include files relative to the project root. It can + be ${CMAKE_CURRENT_SOURCE_DIR} to include files relative to the current + source directory. + +Additional Unnamed Arguments: + + After `target` and `include_prefix`, all unnamed arguments are treated as + paths to `.capnp` schema files. These should be paths relative to + ${CMAKE_CURRENT_SOURCE_DIR}. + +Optional Keyword Arguments: + + IMPORT_PATHS: Specifies additional directories to search for imported + `.capnp` files. + +Example: + # Assuming `my_library` is a target and `lib/` contains `.capnp` schema + # files with imports from `include/`. + target_capnp_sources(my_library "${CMAKE_SOURCE_DIR}" + lib/schema1.capnp lib/schema2.capnp + IMPORT_PATHS ${CMAKE_SOURCE_DIR}/include) + +#]=] + +function(target_capnp_sources target include_prefix) + cmake_parse_arguments(PARSE_ARGV 2 + "TCS" # prefix + "" # options + "" # one_value_keywords + "IMPORT_PATHS" # multi_value_keywords + ) + + set(MPGEN_BINARY "") + if(MPGEN_EXECUTABLE) + set(MPGEN_BINARY "${MPGEN_EXECUTABLE}") + if(NOT EXISTS "${MPGEN_BINARY}") + message(FATAL_ERROR "MPGEN_EXECUTABLE: \"${MPGEN_BINARY}\" does not exist.") + endif() + elseif(TARGET Libmultiprocess::multiprocess) + set(MPGEN_BINARY Libmultiprocess::mpgen) + else() + message(FATAL_ERROR "No usable mpgen. Set MPGEN_EXECUTABLE or enable the internal target.") + endif() + + get_property(mp_include_dir GLOBAL PROPERTY MP_INCLUDE_DIR) + set(generated_headers "") + foreach(capnp_file IN LISTS TCS_UNPARSED_ARGUMENTS) + add_custom_command( + OUTPUT ${capnp_file}.c++ ${capnp_file}.h ${capnp_file}.proxy-client.c++ ${capnp_file}.proxy-types.h ${capnp_file}.proxy-server.c++ ${capnp_file}.proxy-types.c++ ${capnp_file}.proxy.h + COMMAND ${MPGEN_BINARY} ${CMAKE_CURRENT_SOURCE_DIR} ${include_prefix} ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_file} ${TCS_IMPORT_PATHS} ${mp_include_dir} + DEPENDS ${capnp_file} + VERBATIM + ) + target_sources(${target} PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}/${capnp_file}.c++ + ${CMAKE_CURRENT_BINARY_DIR}/${capnp_file}.proxy-client.c++ + ${CMAKE_CURRENT_BINARY_DIR}/${capnp_file}.proxy-server.c++ + ${CMAKE_CURRENT_BINARY_DIR}/${capnp_file}.proxy-types.c++ + ) + + list(APPEND generated_headers ${capnp_file}.h) + endforeach() + + # Translate include_prefix from a source path to a binary path and add it as a + # target include directory. + set(build_include_prefix ${CMAKE_BINARY_DIR}) + file(RELATIVE_PATH relative_path ${CMAKE_SOURCE_DIR} ${include_prefix}) + if(relative_path) + string(APPEND build_include_prefix "/" "${relative_path}") + endif() + target_include_directories(${target} PUBLIC $ ${mp_include_dir}) + + if(TARGET Libmultiprocess::multiprocess) + target_link_libraries(${target} PRIVATE Libmultiprocess::multiprocess) + endif() + + # Add a custom target that can be specified as a dependency of c++ targets + # that include generated headers. It can be necessary to specify these + # dependencies explicitly because while cmake detect dependencies of non + # generated files on generated headers, it does not reliably detect + # dependencies of generated headers on other generated headers. + add_custom_target("${target}_headers" DEPENDS ${generated_headers}) +endfunction() diff --git a/cmake/compat_config.cmake b/cmake/compat_config.cmake new file mode 100644 index 00000000000..283cd38c49e --- /dev/null +++ b/cmake/compat_config.cmake @@ -0,0 +1,59 @@ +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# compat_config.cmake -- compatibility workarounds meant to be included after +# cmake find_package() calls are made, before configuring the ebuild + +# Define capnp_PREFIX if not defined to avoid issue on macos +# https://github.com/bitcoin-core/libmultiprocess/issues/26 + +if (NOT DEFINED capnp_PREFIX AND DEFINED CAPNP_INCLUDE_DIRS) + get_filename_component(capnp_PREFIX "${CAPNP_INCLUDE_DIRS}" DIRECTORY) +endif() + +if (NOT DEFINED CAPNPC_OUTPUT_DIR) + set(CAPNPC_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}") +endif() + +# CMake target definitions for backwards compatibility with Ubuntu bionic +# capnproto 0.6.1 package (https://packages.ubuntu.com/bionic/libcapnp-dev) +# https://github.com/bitcoin-core/libmultiprocess/issues/27 + +if (NOT DEFINED CAPNP_LIB_CAPNPC AND DEFINED CAPNP_LIB_CAPNP-RPC) + string(REPLACE "-rpc" "c" CAPNP_LIB_CAPNPC "${CAPNP_LIB_CAPNP-RPC}") +endif() + +if (NOT DEFINED CapnProto_capnpc_IMPORTED_LOCATION AND DEFINED CapnProto_capnp-rpc_IMPORTED_LOCATION) + string(REPLACE "-rpc" "c" CapnProto_capnpc_IMPORTED_LOCATION "${CapnProto_capnp-rpc_IMPORTED_LOCATION}") +endif() + +if (NOT TARGET CapnProto::capnp AND DEFINED CAPNP_LIB_CAPNP) + add_library(CapnProto::capnp SHARED IMPORTED) + set_target_properties(CapnProto::capnp PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNP}") +endif() + +if (NOT TARGET CapnProto::capnpc AND DEFINED CAPNP_LIB_CAPNPC) + add_library(CapnProto::capnpc SHARED IMPORTED) + set_target_properties(CapnProto::capnpc PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNPC}") +endif() + +if (NOT TARGET CapnProto::capnpc AND DEFINED CapnProto_capnpc_IMPORTED_LOCATION) + add_library(CapnProto::capnpc SHARED IMPORTED) + set_target_properties(CapnProto::capnpc PROPERTIES IMPORTED_LOCATION "${CapnProto_capnpc_IMPORTED_LOCATION}") +endif() + +if (NOT TARGET CapnProto::capnp-rpc AND DEFINED CAPNP_LIB_CAPNP-RPC) + add_library(CapnProto::capnp-rpc SHARED IMPORTED) + set_target_properties(CapnProto::capnp-rpc PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNP-RPC}") +endif() + +if (NOT TARGET CapnProto::kj AND DEFINED CAPNP_LIB_KJ) + add_library(CapnProto::kj SHARED IMPORTED) + set_target_properties(CapnProto::kj PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_KJ}") +endif() + +if (NOT TARGET CapnProto::kj-async AND DEFINED CAPNP_LIB_KJ-ASYNC) + add_library(CapnProto::kj-async SHARED IMPORTED) + set_target_properties(CapnProto::kj-async PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_KJ-ASYNC}") +endif() diff --git a/cmake/compat_find.cmake b/cmake/compat_find.cmake new file mode 100644 index 00000000000..e1d4f7d427a --- /dev/null +++ b/cmake/compat_find.cmake @@ -0,0 +1,20 @@ +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# compat_find.cmake -- compatibility workarounds meant to be included before +# cmake find_package() calls are made + +# Set FOUND_LIBATOMIC to work around bug in debian capnproto package that is +# debian-specific and does not happpen upstream. Debian includes a patch +# https://sources.debian.org/patches/capnproto/1.0.1-4/07_libatomic.patch/ which +# uses check_library_exists(atomic __atomic_load_8 ...) and it fails because the +# symbol name conflicts with a compiler instrinsic as described +# https://github.com/bitcoin-core/libmultiprocess/issues/68#issuecomment-1135150171. +# This could be fixed by improving the check_library_exists function as +# described in the github comment, or by changing the debian patch to check for +# the symbol a different way, but simplest thing to do is work around the +# problem by setting FOUND_LIBATOMIC. This problem has probably not +# been noticed upstream because it only affects CMake packages depending on +# capnproto, not autoconf packages. +set(FOUND_LIBATOMIC TRUE) diff --git a/cmake/pthread_checks.cmake b/cmake/pthread_checks.cmake new file mode 100644 index 00000000000..b54c0b45b8d --- /dev/null +++ b/cmake/pthread_checks.cmake @@ -0,0 +1,41 @@ +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# Define HAVE_PTHREAD_* variables depending on what pthread functions are +# available. + +include(CMakePushCheckState) +include(CheckCXXSourceCompiles) + +cmake_push_check_state() +set(CMAKE_REQUIRED_LIBRARIES Threads::Threads) +check_cxx_source_compiles(" + #include + int main(int argc, char** argv) + { + char thread_name[16]; + return pthread_getname_np(pthread_self(), thread_name, sizeof(thread_name)); + }" + HAVE_PTHREAD_GETNAME_NP) + +check_cxx_source_compiles(" + #include + #include + int main(int argc, char** argv) + { + uint64_t tid; + pthread_threadid_np(NULL, &tid); + return 0; + }" + HAVE_PTHREAD_THREADID_NP) + +check_cxx_source_compiles(" + #include + #include + int main(int argc, char** argv) + { + return pthread_getthreadid_np(); + }" + HAVE_PTHREAD_GETTHREADID_NP) +cmake_pop_check_state() diff --git a/doc/design.md b/doc/design.md new file mode 100644 index 00000000000..ff4b30d063c --- /dev/null +++ b/doc/design.md @@ -0,0 +1,44 @@ +# libmultiprocess Design + +Given an interface description of an object with one or more methods, libmultiprocess generates: + +* A C++ `ProxyClient` class with an implementation of each interface method that sends a request over a socket, waits for a response, and returns the result. +* A C++ `ProxyServer` class that listens for requests over a socket and calls a wrapped C++ object implementing the same interface to actually execute the requests. + +The function call ⇆ request translation supports input and output arguments, standard types like `unique_ptr`, `vector`, `map`, and `optional`, and bidirectional calls between processes through interface pointer and `std::function` arguments. + +If the wrapped C++ object inherits from an abstract base class declaring virtual methods, the generated `ProxyClient` objects can inherit from the same class, allowing interprocess calls to replace local calls without changes to existing code. + +There is also optional support for thread mapping, so each thread making interprocess calls can have a dedicated thread processing requests from it, and callbacks from processing threads are executed on corresponding request threads (so recursive mutexes and thread names function as expected in callbacks). + +Libmultiprocess acts as a pure wrapper or layer over the underlying protocol. Clients and servers written in other languages, but using a shared capnproto schema can communicate with interprocess counterparties using libmultiprocess without having to use libmultiprocess themselves or having to know about the implementation details of libmultiprocess. + +### Internals + +The `ProxyClient` and `ProxyServer` generated classes are not directly exposed to the user, as described in [usage.md](usage.md). Instead, they wrap c++ interfaces and appear to the user as pointers to an interface. They are first instantiated when calling `ConnectStream` and `ServeStream` respectively for creating the `InitInterface`. These methods establish connections through sockets, internally creating `Connection` objects wrapping a `capnp::RpcSystem` configured for client and server mode respectively. + +The `InitInterface` interface will typically have methods which return other interfaces, giving the connecting process the ability to call other functions in the serving process. Interfaces can also have methods accepting other interfaces as parameters, giving serving processes the ability to call back and invoke functions in connecting processes. Creating new interfaces does not create new connections, and typically many interface objects will share the same connection. + +Both `ConnectStream` and `ServeStream` also require an instantiation of the `EventLoop`. The `EventLoop` owns pending requests, notifies on request dispatch, allows clients from multiple threads to make synchronous calls, and handles some cleanup routines on exit. It must be run in a separate thread so it is always active and can process incoming requests from local clients and remote connections. + +When a generated method on the `ProxyClient` is called, it calls `clientInvoke` with the capnp-translated types. `clientInvoke` creates a self-executing promise (`kj::TaskSet`) that drives the execution of the request and gives ownership of it to the `EventLoop`. `clientInvoke` blocks until a response is received, or until there is a call from the server that needs to run on the same client thread, using a `Waiter` object. + +On the server side, the `capnp::RpcSystem` receives the capnp request and invokes the corresponding c++ method through the corresponding `ProxyServer` and the heavily templated `serverInvoke` triggering a `ServerCall`. Its return values from the actual c++ methods are copied into capnp responses by `ServerRet` and exceptions are caught and copied by `ServerExcept`. The two are connected through `ServerField`. The main method driving execution of a request is `PassField`, which is invoked through `ServerField`. Instantiated interfaces, or capabilities in capnp speak, are tracked and owned by the server's `capnp::RpcSystem`. + +## Interface descriptions + +As explained in the [usage](usage.md) document, interface descriptions need to be consumed both by the _libmultiprocess_ code generator, and by C++ code that calls and implements the interfaces. The C++ code only needs to know about C++ arguments and return types, while the code generator only needs to know about capnp arguments and return types, but both need to know class and method names, so the corresponding `.h` and `.capnp` source files contain some of the same information, and have to be kept in sync manually when methods or parameters change. Despite the redundancy, reconciling the interface definitions is designed to be _straightforward_ and _safe_. _Straightforward_ because there is no need to write manual serialization code or use awkward intermediate types like [`UniValue`](https://github.com/bitcoin/bitcoin/blob/master/src/univalue/include/univalue.h) instead of native types. _Safe_ because if there are any inconsistencies between API and data definitions (even minor ones like using a narrow int data type for a wider int API input), there are errors at build time instead of errors or bugs at runtime. + +In the future, it would be possible to combine API and data definitions together using [C++ attributes](https://en.cppreference.com/w/cpp/language/attributes). To do this we would add attributes to the API definition files, and then generate the data definitions from the API definitions and attributes. I didn't take this approach mostly because it would be extra work, but also because until c++ standardizes reflection, this would require either hooking into compiler APIs like https://github.com/RosettaCommons/binder, or parsing c++ code manually like http://www.swig.org/. + +## What is `kj`? + +KJ is a concurrency framework [bundled with +capnproto](https://capnproto.org/cxxrpc.html#kj-concurrency-framework); it is used as a +basis in this library to construct the event-loop necessary to service IPC requests. + +## Future directions + +_libmultiprocess_ uses the [Cap'n Proto](https://capnproto.org) interface description language and protocol, but it could be extended or changed to use a different IDL/protocol like [gRPC](https://grpc.io). The nice thing about _Cap'n Proto_ compared to _gRPC_ and most other lower level protocols is that it allows interface pointers (_Services_ in gRPC parlance) to be passed as method arguments and return values, so object references and bidirectional requests work out of the box. Supporting a lower-level protocol would require writing adding maps and tracking code to proxy objects. + +_libmultiprocess_ is currently compatible with sandboxing but could add platform-specific sandboxing support or integration with a sandboxing library like [SAPI](https://github.com/google/sandboxed-api). diff --git a/doc/install.md b/doc/install.md new file mode 100644 index 00000000000..3fe7c7b29a4 --- /dev/null +++ b/doc/install.md @@ -0,0 +1,52 @@ +# libmultiprocess Installation + +Installation currently requires Cap'n Proto: + +```sh +apt install libcapnp-dev capnproto +brew install capnp cmake +dnf install capnproto +``` + +Installation steps are: + +```sh +mkdir build +cd build +cmake .. +make +make check # Optionally build and run tests +make install +``` + +To build with libmultiprocess in a CMake project can specify: + +```cmake +find_package(Libmultiprocess) +target_capnp_sources(mytarget ${CMAKE_CURRENT_SOURCE_DIR} myschema.capnp) +``` + +Which will locate the libmultiprocess cmake package, and call the +`target_capnp_sources` function to generate C++ files and link them into a +library or executable target. See `example/CMakeLists.txt` for a complete +example. + +To build with libmultiprocess in a non-CMake project can use installed +`/include/mpgen.mk` Makefile rule to generate C++ files, and +`/lib/pkgconfig/libmultiprocess.pc` pkg-config definition to link +against the runtime library. + +For cross-compilation, it may be useful to build the runtime library and code +generation binaries separately, which can be done with: + +```sh +make install-bin # install bin/mpgen and related files +make install-lib # install lib/libmultiprocess.a and related files +``` + +It is also possible to import CMake targets separately with: + +```cmake +find_package(Libmultiprocess COMPONENTS Bin) +find_package(Libmultiprocess COMPONENTS Lib) +``` diff --git a/doc/usage.md b/doc/usage.md new file mode 100644 index 00000000000..1421542d961 --- /dev/null +++ b/doc/usage.md @@ -0,0 +1,24 @@ +# libmultiprocess Usage + +## Overview + +_libmultiprocess_ is a library and code generator that allows calling C++ class interfaces across different processes. For an interface to be available from other processes, it needs two definitions: + +- An **API definition** declaring how the interface is called. Included examples: [calculator.h](https://github.com/bitcoin-core/libmultiprocess/blob/master/example/calculator.h), [printer.h](https://github.com/bitcoin-core/libmultiprocess/blob/master/example/printer.h), [init.h](https://github.com/bitcoin-core/libmultiprocess/blob/master/example/init.h). Bitcoin examples: [node.h](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/interfaces/node.h), [wallet.h](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/interfaces/wallet.h), [echo.h](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/interfaces/echo.h), [init.h](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/interfaces/init.h). + +- A **data definition** declaring how interface calls get sent across the wire. Included examples: [calculator.capnp](https://github.com/bitcoin-core/libmultiprocess/blob/master/example/calculator.capnp), [printer.capnp](https://github.com/bitcoin-core/libmultiprocess/blob/master/example/printer.capnp), [init.capnp](https://github.com/bitcoin-core/libmultiprocess/blob/master/example/init.capnp). Bitcoin examples: [node.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/node.capnp), [wallet.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/wallet.capnp), [echo.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/echo.capnp), [init.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/init.capnp). + +The `*.capnp` data definition files are consumed by the _libmultiprocess_ code generator and each `X.capnp` file generates `X.capnp.c++`, `X.capnp.h`, `X.capnp.proxy-client.c++`, `X.capnp.proxy-server.c++`, `X.capnp.proxy-types.c++`, `X.capnp.proxy-types.h`, and `X.capnp.proxy.h` output files. The generated files include `mp::ProxyClient` and `mp::ProxyServer` class specializations for all the interfaces in the `.capnp` files. These allow methods on C++ objects in one process to be called from other processes over IPC sockets. + +The `ProxyServer` objects help translate IPC requests from a socket to method calls on a local object. The `ProxyServer` objects are just used internally by the `mp::ServeStream(loop, socket, wrapped_object)` and `mp::ListenConnections(loop, socket, wrapped_object)` functions, and aren't exposed externally. The `ProxyClient` classes are exposed, and returned from the `mp::ConnectStream(loop, socket)` function and meant to be used directly. The classes implement methods described in `.capnp` definitions, and whenever any method is called, a request with the method arguments is sent over the associated IPC connection, and the corresponding `wrapped_object` method on the other end of the connection is called, with the `ProxyClient` method blocking until it returns and forwarding back any return value to the `ProxyClient` method caller. + +## Example + +A simple interface description can be found at [test/mp/test/foo.capnp](../test/mp/test/foo.capnp), implementation in [test/mp/test/foo.h](../test/mp/test/foo.h), and usage in [test/mp/test/test.cpp](../test/mp/test/test.cpp). + +A more complete example can be found in [example](../example/) and run with: + +```sh +make -C build example +build/example/mpexample +``` diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt new file mode 100644 index 00000000000..333462b8249 --- /dev/null +++ b/example/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +include(${PROJECT_SOURCE_DIR}/cmake/TargetCapnpSources.cmake) + +add_executable(mpcalculator + calculator.cpp +) +target_capnp_sources(mpcalculator ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) +target_include_directories(mpcalculator PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(mpcalculator PRIVATE Threads::Threads) + +add_executable(mpprinter + printer.cpp +) +target_capnp_sources(mpprinter ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) +target_include_directories(mpprinter PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(mpprinter PRIVATE Threads::Threads) + +add_executable(mpexample + example.cpp +) +target_capnp_sources(mpexample ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) +target_include_directories(mpexample PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(mpexample PRIVATE Threads::Threads) +target_link_libraries(mpexample PRIVATE stdc++fs) + +add_custom_target(mpexamples DEPENDS mpexample mpcalculator mpprinter) diff --git a/example/calculator.capnp b/example/calculator.capnp new file mode 100644 index 00000000000..8f546552f70 --- /dev/null +++ b/example/calculator.capnp @@ -0,0 +1,16 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xb67dbf34061180a9; + +using Cxx = import "/capnp/c++.capnp"; +using Proxy = import "/mp/proxy.capnp"; + +$Proxy.include("calculator.h"); +$Proxy.includeTypes("types.h"); + +interface CalculatorInterface $Proxy.wrap("Calculator") { + destroy @0 (context :Proxy.Context) -> (); + solveEquation @1 (context :Proxy.Context, eqn: Text) -> (); +} diff --git a/example/calculator.cpp b/example/calculator.cpp new file mode 100644 index 00000000000..ae69ce8a626 --- /dev/null +++ b/example/calculator.cpp @@ -0,0 +1,58 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include // NOLINT(misc-include-cleaner) +#include +#include +#include +#include +#include +#include +#include +#include + +class CalculatorImpl : public Calculator +{ +public: + CalculatorImpl(std::unique_ptr printer) : m_printer(std::move(printer)) {} + void solveEquation(const std::string& eqn) override { m_printer->print("Wow " + eqn + ", that's a tough one.\n"); } + std::unique_ptr m_printer; +}; + +class InitImpl : public Init +{ +public: + std::unique_ptr makeCalculator(std::unique_ptr printer) override + { + return std::make_unique(std::move(printer)); + } +}; + +static void LogPrint(bool raise, const std::string& message) +{ + if (raise) throw std::runtime_error(message); + std::ofstream("debug.log", std::ios_base::app) << message << std::endl; +} + +int main(int argc, char** argv) +{ + if (argc != 2) { + std::cout << "Usage: mpcalculator \n"; + return 1; + } + int fd; + if (std::from_chars(argv[1], argv[1] + strlen(argv[1]), fd).ec != std::errc{}) { + std::cerr << argv[1] << " is not a number or is larger than an int\n"; + return 1; + } + mp::EventLoop loop("mpcalculator", LogPrint); + std::unique_ptr init = std::make_unique(); + mp::ServeStream(loop, fd, *init); + loop.loop(); + return 0; +} diff --git a/example/calculator.h b/example/calculator.h new file mode 100644 index 00000000000..749e435547d --- /dev/null +++ b/example/calculator.h @@ -0,0 +1,17 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef EXAMPLE_CALCULATOR_H +#define EXAMPLE_CALCULATOR_H + +#include + +class Calculator +{ +public: + virtual ~Calculator() = default; + virtual void solveEquation(const std::string& eqn) = 0; +}; + +#endif // EXAMPLE_CALCULATOR_H diff --git a/example/example.cpp b/example/example.cpp new file mode 100644 index 00000000000..a4f84c55a75 --- /dev/null +++ b/example/example.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static auto Spawn(mp::EventLoop& loop, const std::string& process_argv0, const std::string& new_exe_name) +{ + int pid; + const int fd = mp::SpawnProcess(pid, [&](int fd) -> std::vector { + fs::path path = process_argv0; + path.remove_filename(); + path.append(new_exe_name); + return {path.string(), std::to_string(fd)}; + }); + return std::make_tuple(mp::ConnectStream(loop, fd), pid); +} + +static void LogPrint(bool raise, const std::string& message) +{ + if (raise) throw std::runtime_error(message); + std::ofstream("debug.log", std::ios_base::app) << message << std::endl; +} + +int main(int argc, char** argv) +{ + if (argc != 1) { + std::cout << "Usage: mpexample\n"; + return 1; + } + + std::promise promise; + std::thread loop_thread([&] { + mp::EventLoop loop("mpexample", LogPrint); + promise.set_value(&loop); + loop.loop(); + }); + mp::EventLoop* loop = promise.get_future().get(); + + auto [printer_init, printer_pid] = Spawn(*loop, argv[0], "mpprinter"); + auto [calc_init, calc_pid] = Spawn(*loop, argv[0], "mpcalculator"); + auto calc = calc_init->makeCalculator(printer_init->makePrinter()); + while (true) { + std::string eqn; + std::cout << "Enter the equation, or \"exit\" to quit: "; + std::getline(std::cin, eqn); + if (eqn == "exit") break; + calc->solveEquation(eqn); + } + calc.reset(); + calc_init.reset(); + mp::WaitProcess(calc_pid); + printer_init.reset(); + mp::WaitProcess(printer_pid); + loop_thread.join(); + std::cout << "Bye!" << std::endl; + return 0; +} diff --git a/example/init.capnp b/example/init.capnp new file mode 100644 index 00000000000..2b0b5113972 --- /dev/null +++ b/example/init.capnp @@ -0,0 +1,22 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xba5a7448664901b1; + +using Cxx = import "/capnp/c++.capnp"; +using Proxy = import "/mp/proxy.capnp"; +using Calculator = import "calculator.capnp"; +using Printer = import "printer.capnp"; + +$Proxy.include("calculator.h"); +$Proxy.include("init.h"); +$Proxy.include("printer.h"); +$Proxy.includeTypes("calculator.capnp.proxy-types.h"); +$Proxy.includeTypes("printer.capnp.proxy-types.h"); + +interface InitInterface $Proxy.wrap("Init") { + construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap :Proxy.ThreadMap); + makeCalculator @1 (context :Proxy.Context, print :Printer.PrinterInterface) -> (result :Calculator.CalculatorInterface); + makePrinter @2 (context :Proxy.Context) -> (result :Printer.PrinterInterface); +} diff --git a/example/init.h b/example/init.h new file mode 100644 index 00000000000..314d5d7f238 --- /dev/null +++ b/example/init.h @@ -0,0 +1,20 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef EXAMPLE_INIT_H +#define EXAMPLE_INIT_H + +#include +#include +#include + +class Init +{ +public: + virtual ~Init() = default; + virtual std::unique_ptr makePrinter() { return nullptr; } + virtual std::unique_ptr makeCalculator(std::unique_ptr printer) { return nullptr; } +}; + +#endif // EXAMPLE_INIT_H diff --git a/example/printer.capnp b/example/printer.capnp new file mode 100644 index 00000000000..e27ce412048 --- /dev/null +++ b/example/printer.capnp @@ -0,0 +1,16 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0x893db95f456ed0e3; + +using Cxx = import "/capnp/c++.capnp"; +using Proxy = import "/mp/proxy.capnp"; + +$Proxy.include("printer.h"); +$Proxy.includeTypes("types.h"); + +interface PrinterInterface $Proxy.wrap("Printer") { + destroy @0 (context :Proxy.Context) -> (); + print @1 (context :Proxy.Context, text: Text) -> (); +} diff --git a/example/printer.cpp b/example/printer.cpp new file mode 100644 index 00000000000..9f85d450b66 --- /dev/null +++ b/example/printer.cpp @@ -0,0 +1,51 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include // NOLINT(misc-include-cleaner) +#include +#include +#include +#include +#include +#include +#include + +class PrinterImpl : public Printer +{ +public: + void print(const std::string& message) override { std::cout << "mpprinter: " << message << std::endl; } +}; + +class InitImpl : public Init +{ +public: + std::unique_ptr makePrinter() override { return std::make_unique(); } +}; + +static void LogPrint(bool raise, const std::string& message) +{ + if (raise) throw std::runtime_error(message); + std::ofstream("debug.log", std::ios_base::app) << message << std::endl; +} + +int main(int argc, char** argv) +{ + if (argc != 2) { + std::cout << "Usage: mpprinter \n"; + return 1; + } + int fd; + if (std::from_chars(argv[1], argv[1] + strlen(argv[1]), fd).ec != std::errc{}) { + std::cerr << argv[1] << " is not a number or is larger than an int\n"; + return 1; + } + mp::EventLoop loop("mpprinter", LogPrint); + std::unique_ptr init = std::make_unique(); + mp::ServeStream(loop, fd, *init); + loop.loop(); + return 0; +} diff --git a/example/printer.h b/example/printer.h new file mode 100644 index 00000000000..066facf1e26 --- /dev/null +++ b/example/printer.h @@ -0,0 +1,17 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef EXAMPLE_PRINTER_H +#define EXAMPLE_PRINTER_H + +#include + +class Printer +{ +public: + virtual ~Printer() = default; + virtual void print(const std::string& message) = 0; +}; + +#endif // EXAMPLE_PRINTER_H diff --git a/example/types.h b/example/types.h new file mode 100644 index 00000000000..0c0bd9342b9 --- /dev/null +++ b/example/types.h @@ -0,0 +1,14 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef EXAMPLE_TYPES_H +#define EXAMPLE_TYPES_H + +#include +#include +#include +#include +#include + +#endif // EXAMPLE_TYPES_H diff --git a/include/mp/config.h.in b/include/mp/config.h.in new file mode 100644 index 00000000000..79ebc4790b8 --- /dev/null +++ b/include/mp/config.h.in @@ -0,0 +1,16 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_CONFIG_H +#define MP_CONFIG_H + +#cmakedefine CMAKE_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@" +#cmakedefine capnp_PREFIX "@capnp_PREFIX@" +#cmakedefine HAVE_KJ_FILESYSTEM + +#cmakedefine HAVE_PTHREAD_GETNAME_NP @HAVE_PTHREAD_GETNAME_NP@ +#cmakedefine HAVE_PTHREAD_THREADID_NP @HAVE_PTHREAD_THREADID_NP@ +#cmakedefine HAVE_PTHREAD_GETTHREADID_NP @HAVE_PTHREAD_GETTHREADID_NP@ + +#endif // MP_CONFIG_H diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h new file mode 100644 index 00000000000..4eb27fae7c4 --- /dev/null +++ b/include/mp/proxy-io.h @@ -0,0 +1,641 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_IO_H +#define MP_PROXY_IO_H + +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace mp { +struct ThreadContext; + +struct InvokeContext +{ + Connection& connection; +}; + +struct ClientInvokeContext : InvokeContext +{ + ThreadContext& thread_context; + ClientInvokeContext(Connection& conn, ThreadContext& thread_context) + : InvokeContext{conn}, thread_context{thread_context} + { + } +}; + +template +struct ServerInvokeContext : InvokeContext +{ + using CallContext = CallContext_; + + ProxyServer& proxy_server; + CallContext& call_context; + int req; + + ServerInvokeContext(ProxyServer& proxy_server, CallContext& call_context, int req) + : InvokeContext{*proxy_server.m_context.connection}, proxy_server{proxy_server}, call_context{call_context}, req{req} + { + } +}; + +template +using ServerContext = ServerInvokeContext, ::capnp::CallContext>; + +template <> +struct ProxyClient : public ProxyClientBase +{ + using ProxyClientBase::ProxyClientBase; + // https://stackoverflow.com/questions/22357887/comparing-two-mapiterators-why-does-it-need-the-copy-constructor-of-stdpair + ProxyClient(const ProxyClient&) = delete; + ~ProxyClient(); + + void setCleanup(const std::function& fn); + + //! Cleanup function to run when the connection is closed. If the Connection + //! gets destroyed before this ProxyClient object, this cleanup + //! callback lets it destroy this object and remove its entry in the + //! thread's request_threads or callback_threads map (after resetting + //! m_cleanup_it so the destructor does not try to access it). But if this + //! object gets destroyed before the Connection, there's no need to run the + //! cleanup function and the destructor will unregister it. + std::optional m_cleanup_it; +}; + +template <> +struct ProxyServer final : public Thread::Server +{ +public: + ProxyServer(ThreadContext& thread_context, std::thread&& thread); + ~ProxyServer(); + kj::Promise getName(GetNameContext context) override; + ThreadContext& m_thread_context; + std::thread m_thread; +}; + +//! Handler for kj::TaskSet failed task events. +class LoggingErrorHandler : public kj::TaskSet::ErrorHandler +{ +public: + LoggingErrorHandler(EventLoop& loop) : m_loop(loop) {} + void taskFailed(kj::Exception&& exception) override; + EventLoop& m_loop; +}; + +using LogFn = std::function; + +class Logger +{ +public: + Logger(bool raise, LogFn& fn) : m_raise(raise), m_fn(fn) {} + Logger(Logger&& logger) : m_raise(logger.m_raise), m_fn(logger.m_fn), m_buffer(std::move(logger.m_buffer)) {} + ~Logger() noexcept(false) + { + if (m_fn) m_fn(m_raise, m_buffer.str()); + } + + template + friend Logger& operator<<(Logger& logger, T&& value) + { + if (logger.m_fn) logger.m_buffer << std::forward(value); + return logger; + } + + template + friend Logger& operator<<(Logger&& logger, T&& value) + { + return logger << std::forward(value); + } + + bool m_raise; + LogFn& m_fn; + std::ostringstream m_buffer; +}; + +std::string LongThreadName(const char* exe_name); + +//! Event loop implementation. +//! +//! Based on https://groups.google.com/d/msg/capnproto/TuQFF1eH2-M/g81sHaTAAQAJ +class EventLoop +{ +public: + //! Construct event loop object. + EventLoop(const char* exe_name, LogFn log_fn, void* context = nullptr); + ~EventLoop(); + + //! Run event loop. Does not return until shutdown. This should only be + //! called once from the m_thread_id thread. This will block until + //! the m_num_clients reference count is 0. + void loop(); + + //! Run function on event loop thread. Does not return until function completes. + //! Must be called while the loop() function is active. + void post(const std::function& fn); + + //! Wrapper around EventLoop::post that takes advantage of the + //! fact that callable will not go out of scope to avoid requirement that it + //! be copyable. + template + void sync(Callable&& callable) + { + post(std::ref(callable)); + } + + //! Start asynchronous worker thread if necessary. This is only done if + //! there are ProxyServerBase::m_impl objects that need to be destroyed + //! asynchronously, without tying up the event loop thread. This can happen + //! when an interface does not declare a destroy() method that would allow + //! the client to wait for the destructor to finish and run it on a + //! dedicated thread. It can also happen whenever this is a broken + //! connection and the client is no longer around to call the destructors + //! and the server objects need to be garbage collected. In both cases, it + //! is important that ProxyServer::m_impl destructors do not run on the + //! eventloop thread because they may need it to do I/O if they perform + //! other IPC calls. + void startAsyncThread(std::unique_lock& lock); + + //! Add/remove remote client reference counts. + void addClient(std::unique_lock& lock); + bool removeClient(std::unique_lock& lock); + //! Check if loop should exit. + bool done(std::unique_lock& lock); + + Logger log() + { + Logger logger(false, m_log_fn); + logger << "{" << LongThreadName(m_exe_name) << "} "; + return logger; + } + Logger logPlain() { return {false, m_log_fn}; } + Logger raise() { return {true, m_log_fn}; } + + //! Process name included in thread names so combined debug output from + //! multiple processes is easier to understand. + const char* m_exe_name; + + //! ID of the event loop thread + std::thread::id m_thread_id = std::this_thread::get_id(); + + //! Handle of an async worker thread. Joined on destruction. Unset if async + //! method has not been called. + std::thread m_async_thread; + + //! Callback function to run on event loop thread during post() or sync() call. + const std::function* m_post_fn = nullptr; + + //! Callback functions to run on async thread. + CleanupList m_async_fns; + + //! Pipe read handle used to wake up the event loop thread. + int m_wait_fd = -1; + + //! Pipe write handle used to wake up the event loop thread. + int m_post_fd = -1; + + //! Number of clients holding references to ProxyServerBase objects that + //! reference this event loop. + int m_num_clients = 0; + + //! Mutex and condition variable used to post tasks to event loop and async + //! thread. + std::mutex m_mutex; + std::condition_variable m_cv; + + //! Capnp IO context. + kj::AsyncIoContext m_io_context; + + //! Capnp error handler. Needs to outlive m_task_set. + LoggingErrorHandler m_error_handler{*this}; + + //! Capnp list of pending promises. + std::unique_ptr m_task_set; + + //! List of connections. + std::list m_incoming_connections; + + //! External logging callback. + LogFn m_log_fn; + + //! External context pointer. + void* m_context; +}; + +//! Single element task queue used to handle recursive capnp calls. (If server +//! makes an callback into the client in the middle of a request, while client +//! thread is blocked waiting for server response, this is what allows the +//! client to run the request in the same thread, the same way code would run in +//! single process, with the callback sharing same thread stack as the original +//! call. +struct Waiter +{ + Waiter() = default; + + template + void post(Fn&& fn) + { + const std::unique_lock lock(m_mutex); + assert(!m_fn); + m_fn = std::move(fn); + m_cv.notify_all(); + } + + template + void wait(std::unique_lock& lock, Predicate pred) + { + m_cv.wait(lock, [&] { + // Important for this to be "while (m_fn)", not "if (m_fn)" to avoid + // a lost-wakeup bug. A new m_fn and m_cv notification might be sent + // after the fn() call and before the lock.lock() call in this loop + // in the case where a capnp response is sent and a brand new + // request is immediately received. + while (m_fn) { + auto fn = std::move(m_fn); + m_fn = nullptr; + lock.unlock(); + fn(); + lock.lock(); + } + const bool done = pred(); + return done; + }); + } + + std::mutex m_mutex; + std::condition_variable m_cv; + std::function m_fn; +}; + +//! Object holding network & rpc state associated with either an incoming server +//! connection, or an outgoing client connection. It must be created and destroyed +//! on the event loop thread. +//! In addition to Cap'n Proto state, it also holds lists of callbacks to run +//! when the connection is closed. +class Connection +{ +public: + Connection(EventLoop& loop, kj::Own&& stream_) + : m_loop(loop), m_stream(kj::mv(stream_)), + m_network(*m_stream, ::capnp::rpc::twoparty::Side::CLIENT, ::capnp::ReaderOptions()), + m_rpc_system(::capnp::makeRpcClient(m_network)) + { + std::unique_lock lock(m_loop.m_mutex); + m_loop.addClient(lock); + } + Connection(EventLoop& loop, + kj::Own&& stream_, + const std::function<::capnp::Capability::Client(Connection&)>& make_client) + : m_loop(loop), m_stream(kj::mv(stream_)), + m_network(*m_stream, ::capnp::rpc::twoparty::Side::SERVER, ::capnp::ReaderOptions()), + m_rpc_system(::capnp::makeRpcServer(m_network, make_client(*this))) + { + std::unique_lock lock(m_loop.m_mutex); + m_loop.addClient(lock); + } + + //! Run cleanup functions. Must be called from the event loop thread. First + //! calls synchronous cleanup functions while blocked (to free capnp + //! Capability::Client handles owned by ProxyClient objects), then schedules + //! asynchronous cleanup functions to run in a worker thread (to run + //! destructors of m_impl instances owned by ProxyServer objects). + ~Connection(); + + //! Register synchronous cleanup function to run on event loop thread (with + //! access to capnp thread local variables) when disconnect() is called. + //! any new i/o. + CleanupIt addSyncCleanup(std::function fn); + void removeSyncCleanup(CleanupIt it); + + //! Register asynchronous cleanup function to run on worker thread when + //! disconnect() is called. + void addAsyncCleanup(std::function fn); + + //! Add disconnect handler. + template + void onDisconnect(F&& f) + { + // Add disconnect handler to local TaskSet to ensure it is cancelled and + // will never run after connection object is destroyed. But when disconnect + // handler fires, do not call the function f right away, instead add it + // to the EventLoop TaskSet to avoid "Promise callback destroyed itself" + // error in cases where f deletes this Connection object. + m_on_disconnect.add(m_network.onDisconnect().then( + [f = std::move(f), this]() mutable { m_loop.m_task_set->add(kj::evalLater(kj::mv(f))); })); + } + + EventLoop& m_loop; + kj::Own m_stream; + LoggingErrorHandler m_error_handler{m_loop}; + kj::TaskSet m_on_disconnect{m_error_handler}; + ::capnp::TwoPartyVatNetwork m_network; + std::optional<::capnp::RpcSystem<::capnp::rpc::twoparty::VatId>> m_rpc_system; + + // ThreadMap interface client, used to create a remote server thread when an + // client IPC call is being made for the first time from a new thread. + ThreadMap::Client m_thread_map{nullptr}; + + //! Collection of server-side IPC worker threads (ProxyServer objects previously returned by + //! ThreadMap.makeThread) used to service requests to clients. + ::capnp::CapabilityServerSet m_threads; + + //! Cleanup functions to run if connection is broken unexpectedly. + //! Lists will be empty if all ProxyClient and ProxyServer objects are + //! destroyed cleanly before the connection is destroyed. + CleanupList m_sync_cleanup_fns; + CleanupList m_async_cleanup_fns; +}; + +//! Vat id for server side of connection. Required argument to RpcSystem::bootStrap() +//! +//! "Vat" is Cap'n Proto nomenclature for a host of various objects that facilitates +//! bidirectional communication with other vats; it is often but not always 1-1 with +//! processes. Cap'n Proto doesn't reference clients or servers per se; instead everything +//! is just a vat. +//! +//! See also: https://github.com/capnproto/capnproto/blob/9021f0c722b36cb11e3690b0860939255ebad39c/c%2B%2B/src/capnp/rpc.capnp#L42-L56 +struct ServerVatId +{ + ::capnp::word scratch[4]{}; + ::capnp::MallocMessageBuilder message{scratch}; + ::capnp::rpc::twoparty::VatId::Builder vat_id{message.getRoot<::capnp::rpc::twoparty::VatId>()}; + ServerVatId() { vat_id.setSide(::capnp::rpc::twoparty::Side::SERVER); } +}; + +template +ProxyClientBase::ProxyClientBase(typename Interface::Client client, + Connection* connection, + bool destroy_connection) + : m_client(std::move(client)), m_context(connection) + +{ + { + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.addClient(lock); + } + + // Handler for the connection getting destroyed before this client object. + auto cleanup_it = m_context.connection->addSyncCleanup([this]() { + // Release client capability by move-assigning to temporary. + { + typename Interface::Client(std::move(m_client)); + } + { + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.removeClient(lock); + } + m_context.connection = nullptr; + }); + + // Two shutdown sequences are supported: + // + // - A normal sequence where client proxy objects are deleted by external + // code that no longer needs them + // + // - A garbage collection sequence where the connection or event loop shuts + // down while external code is still holding client references. + // + // The first case is handled here when m_context.connection is not null. The + // second case is handled by the cleanup function, which sets m_context.connection to + // null so nothing happens here. + m_context.cleanup_fns.emplace_front([this, destroy_connection, cleanup_it]{ + if (m_context.connection) { + // Remove cleanup callback so it doesn't run and try to access + // this object after it's already destroyed. + m_context.connection->removeSyncCleanup(cleanup_it); + + // If the capnp interface defines a destroy method, call it to destroy + // the remote object, waiting for it to be deleted server side. If the + // capnp interface does not define a destroy method, this will just call + // an empty stub defined in the ProxyClientBase class and do nothing. + Sub::destroy(*this); + + // FIXME: Could just invoke removed addCleanup fn here instead of duplicating code + m_context.connection->m_loop.sync([&]() { + // Release client capability by move-assigning to temporary. + { + typename Interface::Client(std::move(m_client)); + } + { + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.removeClient(lock); + } + + if (destroy_connection) { + delete m_context.connection; + m_context.connection = nullptr; + } + }); + } + }); + Sub::construct(*this); +} + +template +ProxyClientBase::~ProxyClientBase() noexcept +{ + CleanupRun(m_context.cleanup_fns); +} + +template +ProxyServerBase::ProxyServerBase(std::shared_ptr impl, Connection& connection) + : m_impl(std::move(impl)), m_context(&connection) +{ + assert(m_impl); + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.addClient(lock); +} + +//! ProxyServer destructor, called from the EventLoop thread by Cap'n Proto +//! garbage collection code after there are no more references to this object. +template +ProxyServerBase::~ProxyServerBase() +{ + if (m_impl) { + // If impl is non-null at this point, it means no client is waiting for + // the m_impl server object to be destroyed synchronously. This can + // happen either if the interface did not define a "destroy" method (see + // invokeDestroy method below), or if a destroy method was defined, but + // the connection was broken before it could be called. + // + // In either case, be conservative and run the cleanup on an + // asynchronous thread, to avoid destructors or cleanup functions + // blocking or deadlocking the current EventLoop thread, since they + // could be making IPC calls. + // + // Technically this is a little too conservative since if the interface + // defines a "destroy" method, but the destroy method does not accept a + // Context parameter specifying a worker thread, the cleanup method + // would run on the EventLoop thread normally (when connection is + // unbroken), but will not run on the EventLoop thread now (when + // connection is broken). Probably some refactoring of the destructor + // and invokeDestroy function is possible to make this cleaner and more + // consistent. + m_context.connection->addAsyncCleanup([impl=std::move(m_impl), fns=std::move(m_context.cleanup_fns)]() mutable { + impl.reset(); + CleanupRun(fns); + }); + } + assert(m_context.cleanup_fns.empty()); + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.removeClient(lock); +} + +//! If the capnp interface defined a special "destroy" method, as described the +//! ProxyClientBase class, this method will be called and synchronously destroy +//! m_impl before returning to the client. +//! +//! If the capnp interface does not define a "destroy" method, this will never +//! be called and the ~ProxyServerBase destructor will be responsible for +//! deleting m_impl asynchronously, whenever the ProxyServer object gets garbage +//! collected by Cap'n Proto. +//! +//! This method is called in the same way other proxy server methods are called, +//! via the serverInvoke function. Basically serverInvoke just calls this as a +//! substitute for a non-existent m_impl->destroy() method. If the destroy +//! method has any parameters or return values they will be handled in the +//! normal way by PassField/ReadField/BuildField functions. Particularly if a +//! Context.thread parameter was passed, this method will run on the worker +//! thread specified by the client. Otherwise it will run on the EventLoop +//! thread, like other server methods without an assigned thread. +template +void ProxyServerBase::invokeDestroy() +{ + m_impl.reset(); + CleanupRun(m_context.cleanup_fns); +} + +using ConnThreads = std::map>; +using ConnThread = ConnThreads::iterator; + +// Retrieve ProxyClient object associated with this connection from a +// map, or create a new one and insert it into the map. Return map iterator and +// inserted bool. +std::tuple SetThread(ConnThreads& threads, std::mutex& mutex, Connection* connection, const std::function& make_thread); + +struct ThreadContext +{ + //! Identifying string for debug. + std::string thread_name; + + //! Waiter object used to allow client threads blocked waiting for a server + //! response to execute callbacks made from the client's corresponding + //! server thread. + std::unique_ptr waiter = nullptr; + + //! When client is making a request to a server, this is the + //! `callbackThread` argument it passes in the request, used by the server + //! in case it needs to make callbacks into the client that need to execute + //! while the client is waiting. This will be set to a local thread object. + ConnThreads callback_threads; + + //! When client is making a request to a server, this is the `thread` + //! argument it passes in the request, used to control which thread on + //! server will be responsible for executing it. If client call is being + //! made from a local thread, this will be a remote thread object returned + //! by makeThread. If a client call is being made from a thread currently + //! handling a server request, this will be set to the `callbackThread` + //! request thread argument passed in that request. + ConnThreads request_threads; + + //! Whether this thread is a capnp event loop thread. Not really used except + //! to assert false if there's an attempt to execute a blocking operation + //! which could deadlock the thread. + bool loop_thread = false; +}; + +//! Given stream file descriptor, make a new ProxyClient object to send requests +//! over the stream. Also create a new Connection object embedded in the +//! client that is freed when the client is closed. +template +std::unique_ptr> ConnectStream(EventLoop& loop, int fd) +{ + typename InitInterface::Client init_client(nullptr); + std::unique_ptr connection; + loop.sync([&] { + auto stream = + loop.m_io_context.lowLevelProvider->wrapSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP); + connection = std::make_unique(loop, kj::mv(stream)); + init_client = connection->m_rpc_system->bootstrap(ServerVatId().vat_id).castAs(); + Connection* connection_ptr = connection.get(); + connection->onDisconnect([&loop, connection_ptr] { + loop.log() << "IPC client: unexpected network disconnect."; + delete connection_ptr; + }); + }); + return std::make_unique>( + kj::mv(init_client), connection.release(), /* destroy_connection= */ true); +} + +//! Given stream and init objects, construct a new ProxyServer object that +//! handles requests from the stream by calling the init object. Embed the +//! ProxyServer in a Connection object that is stored and erased if +//! disconnected. This should be called from the event loop thread. +template +void _Serve(EventLoop& loop, kj::Own&& stream, InitImpl& init) +{ + loop.m_incoming_connections.emplace_front(loop, kj::mv(stream), [&](Connection& connection) { + // Disable deleter so proxy server object doesn't attempt to delete the + // init implementation when the proxy client is destroyed or + // disconnected. + return kj::heap>(std::shared_ptr(&init, [](InitImpl*){}), connection); + }); + auto it = loop.m_incoming_connections.begin(); + it->onDisconnect([&loop, it] { + loop.log() << "IPC server: socket disconnected."; + loop.m_incoming_connections.erase(it); + }); +} + +//! Given connection receiver and an init object, handle incoming connections by +//! calling _Serve, to create ProxyServer objects and forward requests to the +//! init object. +template +void _Listen(EventLoop& loop, kj::Own&& listener, InitImpl& init) +{ + auto* ptr = listener.get(); + loop.m_task_set->add(ptr->accept().then( + [&loop, &init, listener = kj::mv(listener)](kj::Own&& stream) mutable { + _Serve(loop, kj::mv(stream), init); + _Listen(loop, kj::mv(listener), init); + })); +} + +//! Given stream file descriptor and an init object, handle requests on the +//! stream by calling methods on the Init object. +template +void ServeStream(EventLoop& loop, int fd, InitImpl& init) +{ + _Serve( + loop, loop.m_io_context.lowLevelProvider->wrapSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP), init); +} + +//! Given listening socket file descriptor and an init object, handle incoming +//! connections and requests by calling methods on the Init object. +template +void ListenConnections(EventLoop& loop, int fd, InitImpl& init) +{ + loop.sync([&]() { + _Listen(loop, + loop.m_io_context.lowLevelProvider->wrapListenSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP), + init); + }); +} + +extern thread_local ThreadContext g_thread_context; + +} // namespace mp + +#endif // MP_PROXY_IO_H diff --git a/include/mp/proxy-types.h b/include/mp/proxy-types.h new file mode 100644 index 00000000000..1a519efde02 --- /dev/null +++ b/include/mp/proxy-types.h @@ -0,0 +1,727 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPES_H +#define MP_PROXY_TYPES_H + +#include + +#include +#include +#include +#include +#include + +namespace mp { + +template +class ValueField +{ +public: + ValueField(Value& value) : m_value(value) {} + ValueField(Value&& value) : m_value(value) {} + Value& m_value; + + Value& get() { return m_value; } + Value& init() { return m_value; } + bool has() { return true; } +}; + +template +struct StructField +{ + template + StructField(S& struct_) : m_struct(struct_) + { + } + Struct& m_struct; + + // clang-format off + template auto get() const -> decltype(A::get(this->m_struct)) { return A::get(this->m_struct); } + template auto has() const -> std::enable_if_t { return A::getHas(m_struct); } + template auto has() const -> std::enable_if_t { return A::has(m_struct); } + template auto has() const -> std::enable_if_t { return true; } + template auto want() const -> std::enable_if_t { return A::getWant(m_struct); } + template auto want() const -> std::enable_if_t { return true; } + template decltype(auto) set(Args&&... args) const { return A::set(this->m_struct, std::forward(args)...); } + template decltype(auto) init(Args&&... args) const { return A::init(this->m_struct, std::forward(args)...); } + template auto setHas() const -> std::enable_if_t { return A::setHas(m_struct); } + template auto setHas() const -> std::enable_if_t { } + template auto setWant() const -> std::enable_if_t { return A::setWant(m_struct); } + template auto setWant() const -> std::enable_if_t { } + // clang-format on +}; + + + +// Destination parameter type that can be passed to ReadField function as an +// alternative to ReadDestUpdate. It allows the ReadField implementation to call +// the provided emplace_fn function with constructor arguments, so it only needs +// to determine the arguments, and can let the emplace function decide how to +// actually construct the read destination object. For example, if a std::string +// is being read, the ReadField call will call the custom emplace_fn with char* +// and size_t arguments, and the emplace function can decide whether to call the +// constructor via the operator or make_shared or emplace or just return a +// temporary string that is moved from. +template +struct ReadDestEmplace +{ + ReadDestEmplace(TypeList, EmplaceFn&& emplace_fn) : m_emplace_fn(emplace_fn) {} + + //! Simple case. If ReadField impementation calls this construct() method + //! with constructor arguments, just pass them on to the emplace function. + template + decltype(auto) construct(Args&&... args) + { + return m_emplace_fn(std::forward(args)...); + } + + //! More complicated case. If ReadField implementation works by calling this + //! update() method, adapt it call construct() instead. This requires + //! LocalType to have a default constructor to create new object that can be + //! passed to update() + template + decltype(auto) update(UpdateFn&& update_fn) + { + if constexpr (std::is_const_v>>) { + // If destination type is const, default construct temporary + // to pass to update, then call move constructor via construct() to + // move from that temporary. + std::remove_cv_t temp; + update_fn(temp); + return construct(std::move(temp)); + } else { + // Default construct object and pass it to update_fn. + decltype(auto) temp = construct(); + update_fn(temp); + return temp; + } + } + EmplaceFn& m_emplace_fn; +}; + +//! Helper function to create a ReadDestEmplace object that constructs a +//! temporary, ReadField can return. +template +auto ReadDestTemp() +{ + return ReadDestEmplace{TypeList(), [&](auto&&... args) -> decltype(auto) { + return LocalType{std::forward(args)...}; + }}; +} + +//! Destination parameter type that can be passed to ReadField function as an +//! alternative to ReadDestEmplace. Instead of requiring an emplace callback to +//! construct a new value, it just takes a reference to an existing value and +//! assigns a new value to it. +template +struct ReadDestUpdate +{ + ReadDestUpdate(Value& value) : m_value(value) {} + + //! Simple case. If ReadField works by calling update() just forward arguments to update_fn. + template + Value& update(UpdateFn&& update_fn) + { + update_fn(m_value); + return m_value; + } + + //! More complicated case. If ReadField works by calling construct(), need + //! to reconstruct m_value in place. + template + Value& construct(Args&&... args) + { + m_value.~Value(); + new (&m_value) Value(std::forward(args)...); + return m_value; + } + + Value& m_value; +}; + +template +decltype(auto) ReadField(TypeList, Args&&... args) +{ + return CustomReadField(TypeList...>(), Priority<2>(), std::forward(args)...); +} + +template +void ThrowField(TypeList, InvokeContext& invoke_context, Input&& input) +{ + ReadField( + TypeList(), invoke_context, input, ReadDestEmplace(TypeList(), + [](auto&& ...args) -> const LocalType& { throw LocalType{std::forward(args)...}; })); +} + +//! Special case for generic std::exception. It's an abstract type so it can't +//! be created directly. Rethrow as std::runtime_error so callers expecting it +//! will still catch it. +template +void ThrowField(TypeList, InvokeContext& invoke_context, Input&& input) +{ + auto data = input.get(); + throw std::runtime_error(std::string(CharCast(data.begin()), data.size())); +} + +template +bool CustomHasValue(InvokeContext& invoke_context, Values&&... value) +{ + return true; +} + +template +void BuildField(TypeList, Context& context, Output&& output, Values&&... values) +{ + if (CustomHasValue(context, std::forward(values)...)) { + CustomBuildField(TypeList(), Priority<3>(), context, std::forward(values)..., + std::forward(output)); + } +} + +// Adapter to let BuildField overloads methods work set & init list elements as +// if they were fields of a struct. If BuildField is changed to use some kind of +// accessor class instead of calling method pointers, then then maybe this could +// go away or be simplified, because would no longer be a need to return +// ListOutput method pointers emulating capnp struct method pointers.. +template +struct ListOutput; + +template +struct ListOutput<::capnp::List> +{ + using Builder = typename ::capnp::List::Builder; + + ListOutput(Builder& builder, size_t index) : m_builder(builder), m_index(index) {} + Builder& m_builder; + size_t m_index; + + // clang-format off + decltype(auto) get() const { return this->m_builder[this->m_index]; } + decltype(auto) init() const { return this->m_builder[this->m_index]; } + template decltype(auto) set(Arg&& arg) const { return static_cast(this->m_builder).set(m_index, std::forward(arg)); } + template decltype(auto) init(Arg&& arg) const { return static_cast(this->m_builder).init(m_index, std::forward(arg)); } + // clang-format on +}; + +template +void CustomBuildField(TypeList, Priority<0>, InvokeContext& invoke_context, Value&& value, Output&& output) +{ + output.set(BuildPrimitive(invoke_context, std::forward(value), TypeList())); +} + +//! PassField override for callable interface reference arguments. +template +auto PassField(Priority<1>, TypeList, ServerContext& server_context, Fn&& fn, Args&&... args) + -> Require +{ + // Just create a temporary ProxyClient if argument is a reference to an + // interface client. If argument needs to have a longer lifetime and not be + // destroyed after this call, a CustomPassField overload can be implemented + // to bypass this code, and a custom ProxyServerMethodTraits overload can be + // implemented in order to read the capability pointer out of params and + // construct a ProxyClient with a longer lifetime. + const auto& params = server_context.call_context.getParams(); + const auto& input = Make(params); + using Interface = typename Decay::Calls; + auto param = std::make_unique>(input.get(), server_context.proxy_server.m_context.connection, false); + fn.invoke(server_context, std::forward(args)..., *param); +} + +template +void MaybeBuildField(std::true_type, Args&&... args) +{ + BuildField(std::forward(args)...); +} +template +void MaybeBuildField(std::false_type, Args&&...) +{ +} +template +void MaybeReadField(std::true_type, Args&&... args) +{ + ReadField(std::forward(args)...); +} +template +void MaybeReadField(std::false_type, Args&&...) +{ +} + +template +void MaybeSetWant(TypeList, Priority<1>, Value&& value, Output&& output) +{ + if (value) { + output.setWant(); + } +} + +template +void MaybeSetWant(LocalTypes, Priority<0>, Args&&...) +{ +} + +//! Default PassField implementation calling MaybeReadField/MaybeBuildField. +template +void PassField(Priority<0>, TypeList, ServerContext& server_context, Fn&& fn, Args&&... args) +{ + InvokeContext& invoke_context = server_context; + using ArgType = RemoveCvRef; + std::optional param; + const auto& params = server_context.call_context.getParams(); + MaybeReadField(std::integral_constant(), TypeList(), invoke_context, + Make(params), ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + param.emplace(std::forward(args)...); + return *param; + })); + if constexpr (Accessor::in) { + assert(param); + } else { + if (!param) param.emplace(); + } + fn.invoke(server_context, std::forward(args)..., static_cast(*param)); + auto&& results = server_context.call_context.getResults(); + MaybeBuildField(std::integral_constant(), TypeList(), invoke_context, + Make(results), *param); +} + +//! Default PassField implementation for count(0) arguments, calling ReadField/BuildField +template +void PassField(Priority<0>, TypeList<>, ServerContext& server_context, const Fn& fn, Args&&... args) +{ + const auto& params = server_context.call_context.getParams(); + const auto& input = Make(params); + ReadField(TypeList<>(), server_context, input); + fn.invoke(server_context, std::forward(args)...); + auto&& results = server_context.call_context.getResults(); + BuildField(TypeList<>(), server_context, Make(results)); +} + +template +struct IterateFieldsHelper +{ + template + void handleChain(Arg1&& arg1, Arg2&& arg2, ParamList, NextFn&& next_fn, NextFnArgs&&... next_fn_args) + { + using S = Split; + handleChain(std::forward(arg1), std::forward(arg2), typename S::First()); + next_fn.handleChain(std::forward(arg1), std::forward(arg2), typename S::Second(), + std::forward(next_fn_args)...); + } + + template + void handleChain(Arg1&& arg1, Arg2&& arg2, ParamList) + { + static_cast(this)->handleField(std::forward(arg1), std::forward(arg2), ParamList()); + } +private: + IterateFieldsHelper() = default; + friend Derived; +}; + +struct IterateFields : IterateFieldsHelper +{ + template + void handleField(Arg1&&, Arg2&&, ParamList) + { + } +}; + +template +struct ClientException +{ + struct BuildParams : IterateFieldsHelper + { + template + void handleField(InvokeContext& invoke_context, Params& params, ParamList) + { + } + + BuildParams(ClientException* client_exception) : m_client_exception(client_exception) {} + ClientException* m_client_exception; + }; + + struct ReadResults : IterateFieldsHelper + { + template + void handleField(InvokeContext& invoke_context, Results& results, ParamList) + { + StructField input(results); + if (input.has()) { + ThrowField(TypeList(), invoke_context, input); + } + } + + ReadResults(ClientException* client_exception) : m_client_exception(client_exception) {} + ClientException* m_client_exception; + }; +}; + +template +struct ClientParam +{ + ClientParam(Types&&... values) : m_values(values...) {} + + struct BuildParams : IterateFieldsHelper + { + template + void handleField(Args&&... args) + { + callBuild<0>(std::forward(args)...); + } + + // TODO Possible optimization to speed up compile time: + // https://stackoverflow.com/a/7858971 Using enable_if below to check + // position when unpacking tuple might be slower than pattern matching + // approach in the stack overflow solution + template + auto callBuild(Args&&... args) -> std::enable_if_t<(I < sizeof...(Types))> + { + callBuild(std::forward(args)..., std::get(m_client_param->m_values)); + } + + template + auto callBuild(ClientInvokeContext& invoke_context, Params& params, ParamList, Values&&... values) -> + std::enable_if_t<(I == sizeof...(Types))> + { + MaybeBuildField(std::integral_constant(), ParamList(), invoke_context, + Make(params), std::forward(values)...); + MaybeSetWant( + ParamList(), Priority<1>(), std::forward(values)..., Make(params)); + } + + BuildParams(ClientParam* client_param) : m_client_param(client_param) {} + ClientParam* m_client_param; + }; + + struct ReadResults : IterateFieldsHelper + { + template + void handleField(Args&&... args) + { + callRead<0>(std::forward(args)...); + } + + template + auto callRead(Args&&... args) -> std::enable_if_t<(I < sizeof...(Types))> + { + callRead(std::forward(args)..., std::get(m_client_param->m_values)); + } + + template + auto callRead(ClientInvokeContext& invoke_context, Results& results, TypeList, Values&&... values) + -> std::enable_if_t + { + MaybeReadField(std::integral_constant(), TypeList...>(), invoke_context, + Make(results), ReadDestUpdate(values)...); + } + + ReadResults(ClientParam* client_param) : m_client_param(client_param) {} + ClientParam* m_client_param; + }; + + std::tuple m_values; +}; + +template +ClientParam MakeClientParam(Types&&... values) +{ + return {std::forward(values)...}; +} + +struct ServerCall +{ + // FIXME: maybe call call_context.releaseParams() + template + decltype(auto) invoke(ServerContext& server_context, TypeList<>, Args&&... args) const + { + return ProxyServerMethodTraits::invoke( + server_context, + std::forward(args)...); + } +}; + +struct ServerDestroy +{ + template + void invoke(ServerContext& server_context, TypeList<>, Args&&... args) const + { + server_context.proxy_server.invokeDestroy(std::forward(args)...); + } +}; + +template +struct ServerRet : Parent +{ + ServerRet(Parent parent) : Parent(parent) {} + + template + void invoke(ServerContext& server_context, TypeList<>, Args&&... args) const + { + auto&& result = Parent::invoke(server_context, TypeList<>(), std::forward(args)...); + auto&& results = server_context.call_context.getResults(); + InvokeContext& invoke_context = server_context; + BuildField(TypeList(), invoke_context, Make(results), + std::forward(result)); + } +}; + +template +struct ServerExcept : Parent +{ + ServerExcept(Parent parent) : Parent(parent) {} + + template + void invoke(ServerContext& server_context, TypeList<>, Args&&... args) const + { + try { + return Parent::invoke(server_context, TypeList<>(), std::forward(args)...); + } catch (const Exception& exception) { + auto&& results = server_context.call_context.getResults(); + BuildField(TypeList(), server_context, Make(results), exception); + } + } +}; + +//! Helper for CustomPassField below. Call Accessor::get method if it has one, +//! otherwise return capnp::Void. +template +decltype(auto) MaybeGet(Message&& message, decltype(Accessor::get(message))* enable = nullptr) +{ + return Accessor::get(message); +} + +template +::capnp::Void MaybeGet(...) +{ + return {}; +} + +template +void CustomPassField(); + +//! PassField override calling CustomPassField function, if it exists. +//! Defining a CustomPassField or CustomPassMessage overload is useful for +//! input/output parameters. If an overload is not defined these parameters will +//! just be deserialized on the server side with ReadField into a temporary +//! variable, then the server method will be called passing the temporary +//! variable as a parameter, then the temporary variable will be serialized and +//! sent back to the client with BuildField. But if a PassField or PassMessage +//! overload is defined, the overload is called with a callback to invoke and +//! pass parameters to the server side function, and run arbitrary code before +//! and after invoking the function. +template +auto PassField(Priority<2>, Args&&... args) -> decltype(CustomPassField(std::forward(args)...)) +{ + return CustomPassField(std::forward(args)...); +}; + +template +struct ServerField : Parent +{ + ServerField(Parent parent) : Parent(parent) {} + + const Parent& parent() const { return *this; } + + template + decltype(auto) invoke(ServerContext& server_context, ArgTypes, Args&&... args) const + { + return PassField(Priority<2>(), + typename Split::First(), + server_context, + this->parent(), + typename Split::Second(), + std::forward(args)...); + } +}; + +template +ServerField MakeServerField(Parent parent) +{ + return {parent}; +} + +template +struct CapRequestTraits; + +template +struct CapRequestTraits<::capnp::Request<_Params, _Results>> +{ + using Params = _Params; + using Results = _Results; +}; + +//! Entry point called by all generated ProxyClient destructors. This only logs +//! the object destruction. The actual cleanup happens in the ProxyClient base +//! destructor. +template +void clientDestroy(Client& client) +{ + if (client.m_context.connection) { + client.m_context.connection->m_loop.log() << "IPC client destroy " << typeid(client).name(); + } else { + KJ_LOG(INFO, "IPC interrupted client destroy", typeid(client).name()); + } +} + +template +void serverDestroy(Server& server) +{ + server.m_context.connection->m_loop.log() << "IPC server destroy " << typeid(server).name(); +} + +//! Entry point called by generated client code that looks like: +//! +//! ProxyClient::M0::Result ProxyClient::methodName(M0::Param<0> arg0, M0::Param<1> arg1) { +//! typename M0::Result result; +//! clientInvoke(*this, &InterfaceName::Client::methodNameRequest, MakeClientParam<...>(arg0), MakeClientParam<...>(arg1), MakeClientParam<...>(result)); +//! return result; +//! } +//! +//! Ellipses above are where generated Accessor<> type declarations are inserted. +template +void clientInvoke(ProxyClient& proxy_client, const GetRequest& get_request, FieldObjs&&... fields) +{ + if (!proxy_client.m_context.connection) { + throw std::logic_error("clientInvoke call made after disconnect"); + } + if (!g_thread_context.waiter) { + assert(g_thread_context.thread_name.empty()); + g_thread_context.thread_name = ThreadName(proxy_client.m_context.connection->m_loop.m_exe_name); + // If next assert triggers, it means clientInvoke is being called from + // the capnp event loop thread. This can happen when a ProxyServer + // method implementation that runs synchronously on the event loop + // thread tries to make a blocking callback to the client. Any server + // method that makes a blocking callback or blocks in general needs to + // run asynchronously off the event loop thread. This is easy to fix by + // just adding a 'context :Proxy.Context' argument to the capnp method + // declaration so the server method runs in a dedicated thread. + assert(!g_thread_context.loop_thread); + g_thread_context.waiter = std::make_unique(); + proxy_client.m_context.connection->m_loop.logPlain() + << "{" << g_thread_context.thread_name + << "} IPC client first request from current thread, constructing waiter"; + } + ClientInvokeContext invoke_context{*proxy_client.m_context.connection, g_thread_context}; + std::exception_ptr exception; + std::string kj_exception; + bool done = false; + proxy_client.m_context.connection->m_loop.sync([&]() { + auto request = (proxy_client.m_client.*get_request)(nullptr); + using Request = CapRequestTraits; + using FieldList = typename ProxyClientMethodTraits::Fields; + IterateFields().handleChain(invoke_context, request, FieldList(), typename FieldObjs::BuildParams{&fields}...); + proxy_client.m_context.connection->m_loop.logPlain() + << "{" << invoke_context.thread_context.thread_name << "} IPC client send " + << TypeName() << " " << LogEscape(request.toString()); + + proxy_client.m_context.connection->m_loop.m_task_set->add(request.send().then( + [&](::capnp::Response&& response) { + proxy_client.m_context.connection->m_loop.logPlain() + << "{" << invoke_context.thread_context.thread_name << "} IPC client recv " + << TypeName() << " " << LogEscape(response.toString()); + try { + IterateFields().handleChain( + invoke_context, response, FieldList(), typename FieldObjs::ReadResults{&fields}...); + } catch (...) { + exception = std::current_exception(); + } + const std::unique_lock lock(invoke_context.thread_context.waiter->m_mutex); + done = true; + invoke_context.thread_context.waiter->m_cv.notify_all(); + }, + [&](const ::kj::Exception& e) { + kj_exception = kj::str("kj::Exception: ", e).cStr(); + proxy_client.m_context.connection->m_loop.logPlain() + << "{" << invoke_context.thread_context.thread_name << "} IPC client exception " << kj_exception; + const std::unique_lock lock(invoke_context.thread_context.waiter->m_mutex); + done = true; + invoke_context.thread_context.waiter->m_cv.notify_all(); + })); + }); + + std::unique_lock lock(invoke_context.thread_context.waiter->m_mutex); + invoke_context.thread_context.waiter->wait(lock, [&done]() { return done; }); + if (exception) std::rethrow_exception(exception); + if (!kj_exception.empty()) proxy_client.m_context.connection->m_loop.raise() << kj_exception; +} + +//! Invoke callable `fn()` that may return void. If it does return void, replace +//! return value with value of `ret()`. This is useful for avoiding code +//! duplication and branching in generic code that forwards calls to functions. +template +auto ReplaceVoid(Fn&& fn, Ret&& ret) -> + std::enable_if_t, decltype(ret())> +{ + fn(); + return ret(); +} + +//! Overload of above for non-void `fn()` case. +template +auto ReplaceVoid(Fn&& fn, Ret&& ret) -> + std::enable_if_t, decltype(fn())> +{ + return fn(); +} + +extern std::atomic server_reqs; + +//! Entry point called by generated server code that looks like: +//! +//! kj::Promise ProxyServer::methodName(CallContext call_context) { +//! return serverInvoke(*this, call_context, MakeServerField<0, ...>(MakeServerField<1, ...>(Make(ServerCall())))); +//! } +//! +//! Ellipses above are where generated Accessor<> type declarations are inserted. +template +kj::Promise serverInvoke(Server& server, CallContext& call_context, Fn fn) +{ + auto params = call_context.getParams(); + using Params = decltype(params); + using Results = typename decltype(call_context.getResults())::Builds; + + int req = ++server_reqs; + server.m_context.connection->m_loop.log() << "IPC server recv request #" << req << " " + << TypeName() << " " << LogEscape(params.toString()); + + try { + using ServerContext = ServerInvokeContext; + using ArgList = typename ProxyClientMethodTraits::Params; + ServerContext server_context{server, call_context, req}; + // ReplaceVoid is used to support fn.invoke implementations that + // execute asynchronously and return promises, as well as + // implementations that execute synchronously and return void. The + // invoke function will be synchronous by default, but asynchronous if + // an mp.Context argument is passed, and the mp.Context PassField + // overload returns a promise executing the request in a worker thread + // and waiting for it to complete. + return ReplaceVoid([&]() { return fn.invoke(server_context, ArgList()); }, + [&]() { return kj::Promise(kj::mv(call_context)); }) + .then([&server, req](CallContext call_context) { + server.m_context.connection->m_loop.log() << "IPC server send response #" << req << " " << TypeName() + << " " << LogEscape(call_context.getResults().toString()); + }); + } catch (const std::exception& e) { + server.m_context.connection->m_loop.log() << "IPC server unhandled exception: " << e.what(); + throw; + } catch (...) { + server.m_context.connection->m_loop.log() << "IPC server unhandled exception"; + throw; + } +} + +//! Map to convert client interface pointers to ProxyContext struct references +//! at runtime using typeids. +struct ProxyTypeRegister { + template + ProxyTypeRegister(TypeList) { + types().emplace(typeid(Interface), [](void* iface) -> ProxyContext& { return static_cast::Client&>(*static_cast(iface)).m_context; }); + } + using Types = std::map; + static Types& types() { static Types types; return types; } +}; + +} // namespace mp + +#endif // MP_PROXY_TYPES_H diff --git a/include/mp/proxy.capnp b/include/mp/proxy.capnp new file mode 100644 index 00000000000..abd02e437fc --- /dev/null +++ b/include/mp/proxy.capnp @@ -0,0 +1,65 @@ +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xcc316e3f71a040fb; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("mp"); + +annotation include(file): Text; +annotation includeTypes(file): Text; +# Extra include paths to add to generated files. + +annotation wrap(interface, struct): Text; +# Wrap capnp interface generating ProxyClient / ProxyServer C++ classes that +# forward calls to a C++ interface with same methods and parameters. Text +# string should be the name of the C++ interface. +# If applied to struct rather than an interface, this will generate a ProxyType +# struct with get methods for introspection and copying fields between C++ and +# capnp structs. + +annotation count(param, struct, interface): Int32; +# Indicate how many C++ method parameters there are corresponding to one capnp +# parameter (default is 1). If not 1, multiple C++ method arguments will be +# condensed into a single capnp parameter by the client and then expanded by +# the server by CustomReadField/CustomBuildField overloads which need to be +# provided separately. An example would be a capnp Text parameter initialized +# from C++ char* and size arguments. Can be 0 to fill an implicit capnp +# parameter from client or server side context. If annotation is applied to an +# interface or struct type it will apply to all parameters of that type. + +annotation exception(param): Text; +# Indicate that a result parameter corresponds to a C++ exception. Text string +# should be the name of a C++ exception type that the generated server class +# will catch and the client class will rethrow. + +annotation name(field, method): Text; +# Name of the C++ method or field corresponding to a capnp method or field. + +annotation skip(field): Void; +# Synonym for count(0). + +interface ThreadMap $count(0) { + # Interface letting clients control which thread a method call should + # execute on. Clients create and name threads and pass the thread handle as + # a call parameter. + makeThread @0 (name :Text) -> (result :Thread); +} + +interface Thread { + # Thread handle returned by makeThread corresponding to one server thread. + + getName @0 () -> (result: Text); +} + +struct Context $count(0) { + # Execution context passed as a parameter from the client class to the server class. + + thread @0 : Thread; + # Handle of the server thread the current method call should execute on. + + callbackThread @1 : Thread; + # Handle of the client thread that is calling the current method, and that + # any callbacks made by the server thread should be made on. +} diff --git a/include/mp/proxy.h b/include/mp/proxy.h new file mode 100644 index 00000000000..76be0992109 --- /dev/null +++ b/include/mp/proxy.h @@ -0,0 +1,299 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_H +#define MP_PROXY_H + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace mp { +class Connection; +class EventLoop; +//! Mapping from capnp interface type to proxy client implementation (specializations are generated by +//! proxy-codegen.cpp). +template +struct ProxyClient; +//! Mapping from capnp interface type to proxy server implementation (specializations are generated by +//! proxy-codegen.cpp). +template +struct ProxyServer; +//! Mapping from capnp method params type to method traits (specializations are generated by proxy-codegen.cpp). +template +struct ProxyMethod; +//! Mapping from capnp struct type to struct traits (specializations are generated by proxy-codegen.cpp). +template +struct ProxyStruct; +//! Mapping from local c++ type to capnp type and traits (specializations are generated by proxy-codegen.cpp). +template +struct ProxyType; + +using CleanupList = std::list>; +using CleanupIt = typename CleanupList::iterator; + +inline void CleanupRun(CleanupList& fns) { + while (!fns.empty()) { + auto fn = std::move(fns.front()); + fns.pop_front(); + fn(); + } +} + +//! Context data associated with proxy client and server classes. +struct ProxyContext +{ + Connection* connection; + CleanupList cleanup_fns; + + ProxyContext(Connection* connection) : connection(connection) {} +}; + +//! Base class for generated ProxyClient classes that implement a C++ interface +//! and forward calls to a capnp interface. +template +class ProxyClientBase : public Impl_ +{ +public: + using Interface = Interface_; + using Impl = Impl_; + using Sub = ProxyClient; + using Super = ProxyClientBase; + + ProxyClientBase(typename Interface::Client client, Connection* connection, bool destroy_connection); + ~ProxyClientBase() noexcept; + + // construct/destroy methods called during client construction/destruction + // that can optionally be defined in capnp interfaces to invoke code on the + // server when proxy client objects are created and destroyed. + // + // The construct() method is not generally very useful, but can be used to + // run custom code on the server automatically when a ProxyClient client is + // constructed. The only current use is adding a construct method to Init + // interfaces that is called automatically on construction, so client and + // server exchange ThreadMap references and set Connection::m_thread_map + // values as soon as the Init client is created. + // + // construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap: Proxy.ThreadMap); + // + // But construct() is not necessary for this, thread maps could be passed + // through a normal method that is just called explicitly rather than + // implicitly. + // + // The destroy() method is more generally useful than construct(), because + // it ensures that the server object will be destroyed synchronously before + // the client destructor returns, instead of asynchronously at some + // unpredictable time after the client object is already destroyed and + // client code has moved on. If the destroy method accepts a Context + // parameter like: + // + // destroy @0 (context: Proxy.Context) -> (); + // + // then it will also ensure that the destructor runs on the same thread the + // client used to make other RPC calls, instead of running on the server + // EventLoop thread and possibly blocking it. + static void construct(Super&) {} + static void destroy(Super&) {} + + typename Interface::Client m_client; + ProxyContext m_context; +}; + +//! Customizable (through template specialization) base class used in generated ProxyClient implementations from +//! proxy-codegen.cpp. +template +class ProxyClientCustom : public ProxyClientBase +{ + using ProxyClientBase::ProxyClientBase; +}; + +//! Base class for generated ProxyServer classes that implement capnp server +//! methods and forward calls to a wrapped c++ implementation class. +template +struct ProxyServerBase : public virtual Interface_::Server +{ +public: + using Interface = Interface_; + using Impl = Impl_; + + ProxyServerBase(std::shared_ptr impl, Connection& connection); + virtual ~ProxyServerBase(); + void invokeDestroy(); + + /** + * Implementation pointer that may or may not be owned and deleted when this + * capnp server goes out of scope. It is owned for servers created to wrap + * unique_ptr method arguments, but unowned for servers created to + * wrap Impl& method arguments. + * + * In the case of Impl& arguments, custom code is required on other side of + * the connection to delete the capnp client & server objects since native + * code on that side of the connection will just be taking a plain reference + * rather than a pointer, so won't be able to do its own cleanup. Right now + * this is implemented with addCloseHook callbacks to delete clients at + * appropriate times depending on semantics of the particular method being + * wrapped. */ + std::shared_ptr m_impl; + ProxyContext m_context; +}; + +//! Customizable (through template specialization) base class which ProxyServer +//! classes produced by generated code will inherit from. The default +//! specialization of this class just inherits from ProxyServerBase, but custom +//! specializations can be defined to control ProxyServer behavior. +//! +//! Specifically, it can be useful to specialize this class to add additional +//! state to ProxyServer classes, for example to cache state between IPC calls. +//! If this is done, however, care should be taken to ensure that the extra +//! state can be destroyed without blocking, because ProxyServer destructors are +//! called from the EventLoop thread, and if they block, it could deadlock the +//! program. One way to do avoid blocking is to clean up the state by pushing +//! cleanup callbacks to the m_context.cleanup_fns list, which run after the server +//! m_impl object is destroyed on the same thread destroying it (which will +//! either be an IPC worker thread if the ProxyServer is being explicitly +//! destroyed by a client calling a destroy() method with a Context argument and +//! Context.thread value set, or the temporary EventLoop::m_async_thread used to +//! run destructors without blocking the event loop when no-longer used server +//! objects are garbage collected by Cap'n Proto.) Alternately, if cleanup needs +//! to run before m_impl is destroyed, the specialization can override +//! invokeDestroy and destructor methods to do that. +template +struct ProxyServerCustom : public ProxyServerBase +{ + using ProxyServerBase::ProxyServerBase; +}; + +//! Function traits class used to get method parameter and result types, used in +//! generated ProxyClient and ProxyServer classes produced by gen.cpp to get C++ +//! method type information. The generated code accesses these traits via +//! intermediate ProxyClientMethodTraits and ProxyServerMethodTraits classes, +//! which it is possible to specialize to change the way method arguments and +//! return values are handled. +//! +//! Fields of the trait class are: +//! +//! Params - TypeList of C++ ClassName::methodName parameter types +//! Result - Return type of ClassName::method +//! Param - helper to access individual parameters by index number. +//! Fields - helper alias that appends Result type to the Params typelist if +//! it not void. +template +struct FunctionTraits; + +//! Specialization of above extracting result and params types assuming the +//! template argument is a pointer-to-method type, +//! decltype(&ClassName::methodName) +template +struct FunctionTraits<_Result (_Class::*const)(_Params...)> +{ + using Params = TypeList<_Params...>; + using Result = _Result; + template + using Param = typename std::tuple_element>::type; + using Fields = + std::conditional_t, Params, TypeList<_Params..., _Result>>; +}; + +//! Traits class for a proxy method, providing the same +//! Params/Result/Param/Fields described in the FunctionTraits class above, plus +//! an additional invoke() method that calls the C++ method which is being +//! proxied, forwarding any arguments. +//! +//! The template argument should be the InterfaceName::MethodNameParams class +//! (generated by Cap'n Proto) associated with the method. +//! +//! Note: The class definition here is just the fallback definition used when +//! the other specialization below doesn't match. The fallback is only used for +//! capnp methods which do not have corresponding C++ methods, which in practice +//! is just the two special construct() and destroy() methods described in \ref +//! ProxyClientBase. These methods don't have any C++ parameters or return +//! types, so the trait information below reflects that. +template +struct ProxyMethodTraits +{ + using Params = TypeList<>; + using Result = void; + using Fields = Params; + + template + static void invoke(ServerContext&) + { + } +}; + +//! Specialization of above for proxy methods that have a +//! ProxyMethod::impl pointer-to-method +//! constant defined by generated code. This includes all functions defined in +//! the capnp interface except any construct() or destroy() methods, that are +//! assumed not to correspond to real member functions in the C++ class, and +//! will use the fallback traits definition above. The generated code this +//! specialization relies on looks like: +//! +//! struct ProxyMethod +//! { +//! static constexpr auto impl = &ClassName::methodName; +//! }; +template +struct ProxyMethodTraits::impl)>> + : public FunctionTraits::impl)> +{ + template + static decltype(auto) invoke(ServerContext& server_context, Args&&... args) + { + return (server_context.proxy_server.m_impl.get()->*ProxyMethod::impl)(std::forward(args)...); + } +}; + +//! Customizable (through template specialization) traits class used in generated ProxyClient implementations from +//! proxy-codegen.cpp. +template +struct ProxyClientMethodTraits : public ProxyMethodTraits +{ +}; + +//! Customizable (through template specialization) traits class used in generated ProxyServer implementations from +//! proxy-codegen.cpp. +template +struct ProxyServerMethodTraits : public ProxyMethodTraits +{ +}; + +static constexpr int FIELD_IN = 1; +static constexpr int FIELD_OUT = 2; +static constexpr int FIELD_OPTIONAL = 4; +static constexpr int FIELD_REQUESTED = 8; +static constexpr int FIELD_BOXED = 16; + +//! Accessor type holding flags that determine how to access a message field. +template +struct Accessor : public Field +{ + static const bool in = flags & FIELD_IN; + static const bool out = flags & FIELD_OUT; + static const bool optional = flags & FIELD_OPTIONAL; + static const bool requested = flags & FIELD_REQUESTED; + static const bool boxed = flags & FIELD_BOXED; +}; + +//! Wrapper around std::function for passing std::function objects between client and servers. +template +class ProxyCallback; + +//! Specialization of above to separate Result and Arg types. +template +class ProxyCallback> +{ +public: + virtual Result call(Args&&... args) = 0; +}; + +} // namespace mp + +#endif // MP_PROXY_H diff --git a/include/mp/type-char.h b/include/mp/type-char.h new file mode 100644 index 00000000000..d1d27b62414 --- /dev/null +++ b/include/mp/type-char.h @@ -0,0 +1,36 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_CHAR_H +#define MP_PROXY_TYPE_CHAR_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, + Priority<3>, + InvokeContext& invoke_context, + const unsigned char (&value)[size], + Output&& output) +{ + auto result = output.init(size); + memcpy(result.begin(), value, size); +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + memcpy(value, data.begin(), size); + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_CHAR_H diff --git a/include/mp/type-chrono.h b/include/mp/type-chrono.h new file mode 100644 index 00000000000..a17d9a994bf --- /dev/null +++ b/include/mp/type-chrono.h @@ -0,0 +1,34 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_CHRONO_H +#define MP_PROXY_TYPE_CHRONO_H + +#include + +#include + +namespace mp { +//! Overload CustomBuildField and CustomReadField to serialize std::chrono +//! parameters and return values as numbers. +template +void CustomBuildField(TypeList>, Priority<1>, InvokeContext& invoke_context, Value&& value, + Output&& output) +{ + static_assert(std::numeric_limits::lowest() <= std::numeric_limits::lowest(), + "capnp type does not have enough range to hold lowest std::chrono::duration value"); + static_assert(std::numeric_limits::max() >= std::numeric_limits::max(), + "capnp type does not have enough range to hold highest std::chrono::duration value"); + output.set(value.count()); +} + +template +decltype(auto) CustomReadField(TypeList>, Priority<1>, InvokeContext& invoke_context, + Input&& input, ReadDest&& read_dest) +{ + return read_dest.construct(input.get()); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_CHRONO_H diff --git a/include/mp/type-context.h b/include/mp/type-context.h new file mode 100644 index 00000000000..7c12afe2ff0 --- /dev/null +++ b/include/mp/type-context.h @@ -0,0 +1,173 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_CONTEXT_H +#define MP_PROXY_TYPE_CONTEXT_H + +#include +#include + +namespace mp { +template +void CustomBuildField(TypeList<>, + Priority<1>, + ClientInvokeContext& invoke_context, + Output&& output, + typename std::enable_if::value>::type* enable = nullptr) +{ + auto& connection = invoke_context.connection; + auto& thread_context = invoke_context.thread_context; + + // Create local Thread::Server object corresponding to the current thread + // and pass a Thread::Client reference to it in the Context.callbackThread + // field so the function being called can make callbacks to this thread. + // Also store the Thread::Client reference in the callback_threads map so + // future calls over this connection can reuse it. + auto [callback_thread, _]{SetThread( + thread_context.callback_threads, thread_context.waiter->m_mutex, &connection, + [&] { return connection.m_threads.add(kj::heap>(thread_context, std::thread{})); })}; + + // Call remote ThreadMap.makeThread function so server will create a + // dedicated worker thread to run function calls from this thread. Store the + // Thread::Client reference it returns in the request_threads map. + auto make_request_thread{[&]{ + // This code will only run if an IPC client call is being made for the + // first time on this thread. After the first call, subsequent calls + // will use the existing request thread. This code will also never run at + // all if the current thread is a request thread created for a different + // IPC client, because in that case PassField code (below) will have set + // request_thread to point to the calling thread. + auto request = connection.m_thread_map.makeThreadRequest(); + request.setName(thread_context.thread_name); + return request.send().getResult(); // Nonblocking due to capnp request pipelining. + }}; + auto [request_thread, _1]{SetThread( + thread_context.request_threads, thread_context.waiter->m_mutex, + &connection, make_request_thread)}; + + auto context = output.init(); + context.setThread(request_thread->second.m_client); + context.setCallbackThread(callback_thread->second.m_client); +} + +//! PassField override for mp.Context arguments. Return asynchronously and call +//! function on other thread found in context. +template +auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& fn, Args&&... args) -> + typename std::enable_if< + std::is_same::value, + kj::Promise>::type +{ + const auto& params = server_context.call_context.getParams(); + Context::Reader context_arg = Accessor::get(params); + auto future = kj::newPromiseAndFulfiller(); + auto& server = server_context.proxy_server; + int req = server_context.req; + auto invoke = MakeAsyncCallable( + [fulfiller = kj::mv(future.fulfiller), + call_context = kj::mv(server_context.call_context), &server, req, fn, args...]() mutable { + const auto& params = call_context.getParams(); + Context::Reader context_arg = Accessor::get(params); + ServerContext server_context{server, call_context, req}; + bool disconnected{false}; + { + // Before invoking the function, store a reference to the + // callbackThread provided by the client in the + // thread_local.request_threads map. This way, if this + // server thread needs to execute any RPCs that call back to + // the client, they will happen on the same client thread + // that is waiting for this function, just like what would + // happen if this were a normal function call made on the + // local stack. + // + // If the request_threads map already has an entry for this + // connection, it will be left unchanged, and it indicates + // that the current thread is an RPC client thread which is + // in the middle of an RPC call, and the current RPC call is + // a nested call from the remote thread handling that RPC + // call. In this case, the callbackThread value should point + // to the same thread already in the map, so there is no + // need to update the map. + auto& thread_context = g_thread_context; + auto& request_threads = thread_context.request_threads; + auto [request_thread, inserted]{SetThread( + request_threads, thread_context.waiter->m_mutex, + server.m_context.connection, + [&] { return context_arg.getCallbackThread(); })}; + + // If an entry was inserted into the requests_threads map, + // remove it after calling fn.invoke. If an entry was not + // inserted, one already existed, meaning this must be a + // recursive call (IPC call calling back to the caller which + // makes another IPC call), so avoid modifying the map. + const bool erase_thread{inserted}; + KJ_DEFER({ + std::unique_lock lock(thread_context.waiter->m_mutex); + // Call erase here with a Connection* argument instead + // of an iterator argument, because the `request_thread` + // iterator may be invalid if the connection is closed + // during this function call. More specifically, the + // iterator may be invalid because SetThread adds a + // cleanup callback to the Connection destructor that + // erases the thread from the map, and also because the + // ProxyServer destructor calls + // request_threads.clear(). + if (erase_thread) { + disconnected = !request_threads.erase(server.m_context.connection); + } else { + disconnected = !request_threads.count(server.m_context.connection); + } + }); + fn.invoke(server_context, args...); + } + if (disconnected) { + // If disconnected is true, the Connection object was + // destroyed during the method call. Deal with this by + // returning without ever fulfilling the promise, which will + // cause the ProxyServer object to leak. This is not ideal, + // but fixing the leak will require nontrivial code changes + // because there is a lot of code assuming ProxyServer + // objects are destroyed before Connection objects. + return; + } + KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() { + server.m_context.connection->m_loop.sync([&] { + auto fulfiller_dispose = kj::mv(fulfiller); + fulfiller_dispose->fulfill(kj::mv(call_context)); + }); + })) + { + server.m_context.connection->m_loop.sync([&]() { + auto fulfiller_dispose = kj::mv(fulfiller); + fulfiller_dispose->reject(kj::mv(*exception)); + }); + } + }); + + // Lookup Thread object specified by the client. The specified thread should + // be a local Thread::Server object, but it needs to be looked up + // asynchronously with getLocalServer(). + auto thread_client = context_arg.getThread(); + return server.m_context.connection->m_threads.getLocalServer(thread_client) + .then([&server, invoke, req](const kj::Maybe& perhaps) { + // Assuming the thread object is found, pass it a pointer to the + // `invoke` lambda above which will invoke the function on that + // thread. + KJ_IF_MAYBE (thread_server, perhaps) { + const auto& thread = static_cast&>(*thread_server); + server.m_context.connection->m_loop.log() + << "IPC server post request #" << req << " {" << thread.m_thread_context.thread_name << "}"; + thread.m_thread_context.waiter->post(std::move(invoke)); + } else { + server.m_context.connection->m_loop.log() + << "IPC server error request #" << req << ", missing thread to execute request"; + throw std::runtime_error("invalid thread handle"); + } + }) + // Wait for the invocation to finish before returning to the caller. + .then([invoke_wait = kj::mv(future.promise)]() mutable { return kj::mv(invoke_wait); }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_CONTEXT_H diff --git a/include/mp/type-data.h b/include/mp/type-data.h new file mode 100644 index 00000000000..46a2b2fc725 --- /dev/null +++ b/include/mp/type-data.h @@ -0,0 +1,46 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_DATA_H +#define MP_PROXY_TYPE_DATA_H + +#include + +namespace mp { +template +concept IsSpanOf = + std::convertible_to> && + std::constructible_from; + +template +concept IsByteSpan = + IsSpanOf || + IsSpanOf || + IsSpanOf || + IsSpanOf; + +//! Generic ::capnp::Data field builder for any C++ type that can be converted +//! to a span of bytes, like std::vector or std::array, or custom +//! blob types like uint256 or PKHash with data() and size() methods pointing to +//! bytes. +template +void CustomBuildField(TypeList, Priority<2>, InvokeContext& invoke_context, Value&& value, Output&& output) +requires (std::is_same_v && IsByteSpan) +{ + auto data = std::span{value}; + auto result = output.init(data.size()); + memcpy(result.begin(), data.data(), data.size()); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<2>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest) +requires (std::is_same_v && IsByteSpan) +{ + using ByteType = decltype(std::span{std::declval().begin(), std::declval().end()})::element_type; + const kj::byte *begin{input.get().begin()}, *end{input.get().end()}; + return read_dest.construct(reinterpret_cast(begin), reinterpret_cast(end)); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_DATA_H diff --git a/include/mp/type-decay.h b/include/mp/type-decay.h new file mode 100644 index 00000000000..7b203c8628c --- /dev/null +++ b/include/mp/type-decay.h @@ -0,0 +1,38 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_DECAY_H +#define MP_PROXY_TYPE_DECAY_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, + Priority<0>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + BuildField(TypeList(), invoke_context, output, std::forward(value)); +} + +template +void CustomBuildField(TypeList, Priority<0>, InvokeContext& invoke_context, Value&& value, Output&& output) +{ + BuildField(TypeList(), invoke_context, output, std::forward(value)); +} + +template +void CustomBuildField(TypeList, + Priority<0>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + BuildField(TypeList(), invoke_context, output, std::forward(value)); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_DECAY_H diff --git a/include/mp/type-exception.h b/include/mp/type-exception.h new file mode 100644 index 00000000000..3e2fcac2737 --- /dev/null +++ b/include/mp/type-exception.h @@ -0,0 +1,22 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_EXCEPTION_H +#define MP_PROXY_TYPE_EXCEPTION_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + const std::exception& value, + Output&& output) +{ + BuildField(TypeList(), invoke_context, output, std::string(value.what())); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_EXCEPTION_H diff --git a/include/mp/type-function.h b/include/mp/type-function.h new file mode 100644 index 00000000000..bf00c581197 --- /dev/null +++ b/include/mp/type-function.h @@ -0,0 +1,67 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_FUNCTION_H +#define MP_PROXY_TYPE_FUNCTION_H + +#include + +namespace mp { +//! Adapter to convert ProxyCallback object call to function object call. +template +class ProxyCallbackImpl final : public ProxyCallback> +{ + using Fn = std::function; + Fn m_fn; + +public: + ProxyCallbackImpl(Fn fn) : m_fn(std::move(fn)) {} + Result call(Args&&... args) override { return m_fn(std::forward(args)...); } +}; + +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value& value, + Output&& output) +{ + if (value) { + using Interface = typename decltype(output.get())::Calls; + using Callback = ProxyCallbackImpl; + output.set(kj::heap>( + std::make_shared(std::forward(value)), invoke_context.connection)); + } +} + +// ProxyCallFn class is needed because c++11 doesn't support auto lambda parameters. +// It's equivalent c++14: [invoke_context](auto&& params) { +// invoke_context->call(std::forward(params)...) +template +struct ProxyCallFn +{ + InvokeContext m_proxy; + + template + decltype(auto) operator()(CallParams&&... params) { return this->m_proxy->call(std::forward(params)...); } +}; + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + if (input.has()) { + using Interface = typename Decay::Calls; + auto client = std::make_shared>( + input.get(), &invoke_context.connection, /* destroy_connection= */ false); + return read_dest.construct(ProxyCallFn{std::move(client)}); + } + return read_dest.construct(); +}; +} // namespace mp + +#endif // MP_PROXY_TYPE_FUNCTION_H diff --git a/include/mp/type-interface.h b/include/mp/type-interface.h new file mode 100644 index 00000000000..99adf2ab94e --- /dev/null +++ b/include/mp/type-interface.h @@ -0,0 +1,112 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_INTERFACE_H +#define MP_PROXY_TYPE_INTERFACE_H + +#include + +namespace mp { +template +kj::Own MakeProxyServer(InvokeContext& context, std::shared_ptr impl) +{ + return kj::heap>(std::move(impl), context.connection); +} + +template +kj::Own CustomMakeProxyServer(InvokeContext& context, std::shared_ptr&& impl) +{ + return MakeProxyServer(context, std::move(impl)); +} + +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output, + typename Decay::Calls* enable = nullptr) +{ + if (value) { + using Interface = typename decltype(output.get())::Calls; + output.set(CustomMakeProxyServer(invoke_context, std::shared_ptr(value.release()))); + } +} + +template +void CustomBuildField(TypeList>, + Priority<2>, + InvokeContext& invoke_context, + Value&& value, + Output&& output, + typename Decay::Calls* enable = nullptr) +{ + if (value) { + using Interface = typename decltype(output.get())::Calls; + output.set(CustomMakeProxyServer(invoke_context, std::move(value))); + } +} + +template +void CustomBuildField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Impl& value, + Output&& output, + typename decltype(output.get())::Calls* enable = nullptr) +{ + // Disable deleter so proxy server object doesn't attempt to delete the + // wrapped implementation when the proxy client is destroyed or + // disconnected. + using Interface = typename decltype(output.get())::Calls; + output.set(CustomMakeProxyServer(invoke_context, std::shared_ptr(&value, [](Impl*){}))); +} + +template +std::unique_ptr MakeProxyClient(InvokeContext& context, typename Interface::Client&& client) +{ + return std::make_unique>( + std::move(client), &context.connection, /* destroy_connection= */ false); +} + +template +std::unique_ptr CustomMakeProxyClient(InvokeContext& context, typename Interface::Client&& client) +{ + return MakeProxyClient(context, kj::mv(client)); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename Decay::Calls* enable = nullptr) +{ + using Interface = typename Decay::Calls; + if (input.has()) { + return read_dest.construct( + CustomMakeProxyClient(invoke_context, std::move(input.get()))); + } + return read_dest.construct(); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename Decay::Calls* enable = nullptr) +{ + using Interface = typename Decay::Calls; + if (input.has()) { + return read_dest.construct( + CustomMakeProxyClient(invoke_context, std::move(input.get()))); + } + return read_dest.construct(); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_INTERFACE_H diff --git a/include/mp/type-map.h b/include/mp/type-map.h new file mode 100644 index 00000000000..bc1b22769df --- /dev/null +++ b/include/mp/type-map.h @@ -0,0 +1,52 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_MAP_H +#define MP_PROXY_TYPE_MAP_H + +#include +#include +#include + +namespace mp { +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + // FIXME dededup with vector handler above + auto list = output.init(value.size()); + size_t i = 0; + for (const auto& elem : value) { + BuildField(TypeList>(), invoke_context, + ListOutput(list, i), elem); + ++i; + } +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.clear(); + for (auto item : data) { + ReadField(TypeList>(), invoke_context, + Make(item), + ReadDestEmplace( + TypeList>(), [&](auto&&... args) -> auto& { + return *value.emplace(std::forward(args)...).first; + })); + } + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_MAP_H diff --git a/include/mp/type-message.h b/include/mp/type-message.h new file mode 100644 index 00000000000..d80f43c8d37 --- /dev/null +++ b/include/mp/type-message.h @@ -0,0 +1,64 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_MESSAGE_H +#define MP_PROXY_TYPE_MESSAGE_H + +#include + +namespace mp { +//! Overload CustomBuildField to serialize objects that have CustomBuildMessage +//! overloads. Defining a CustomBuildMessage overload is simpler than defining a +//! CustomBuildField overload because it only requires defining a normal +//! function, not a template function, but less flexible. +template +void CustomBuildField(TypeList, Priority<2>, InvokeContext& invoke_context, Value&& value, Output&& output, + decltype(CustomBuildMessage(invoke_context, value, std::move(output.get())))* enable = nullptr) +{ + CustomBuildMessage(invoke_context, value, std::move(output.init())); +} + +//! Overload CustomReadField to serialize objects that have CustomReadMessage +//! overloads. Defining a CustomReadMessage overload is simpler than defining a +//! CustomReadField overload because it only requires defining a normal +//! function, not a template function, but less flexible. +template +decltype(auto) CustomReadField(TypeList, Priority<2>, InvokeContext& invoke_context, Reader&& reader, + ReadDest&& read_dest, + decltype(CustomReadMessage(invoke_context, reader.get(), + std::declval()))* enable = nullptr) +{ + return read_dest.update([&](auto& value) { if (reader.has()) CustomReadMessage(invoke_context, reader.get(), value); }); +} + +//! Helper for CustomPassField below. Call Accessor::init method if it has one, +//! otherwise do nothing. +template +decltype(auto) MaybeInit(Message&& message, decltype(Accessor::get(message))* enable = nullptr) +{ + return Accessor::init(message); +} + +template +::capnp::Void MaybeInit(...) +{ + return {}; +} + +//! Overload CustomPassField to serialize objects that have CustomPassMessage +//! overloads. Defining a CustomPassMessage overload is simpler than defining a +//! CustomPassField overload because it only requires defining a normal +//! function, not a template function, but less flexible. +template +auto CustomPassField(TypeList, ServerContext& server_context, Fn&& fn, Args&&... args) + -> decltype(CustomPassMessage(server_context, MaybeGet(server_context.call_context.getParams()), + MaybeGet(server_context.call_context.getResults()), nullptr)) +{ + CustomPassMessage(server_context, MaybeGet(server_context.call_context.getParams()), + MaybeInit(server_context.call_context.getResults()), + [&](LocalTypes... param) { fn.invoke(server_context, std::forward(args)..., param...); }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_MESSAGE_H diff --git a/include/mp/type-number.h b/include/mp/type-number.h new file mode 100644 index 00000000000..9d269be60a8 --- /dev/null +++ b/include/mp/type-number.h @@ -0,0 +1,87 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_NUMBER_H +#define MP_PROXY_TYPE_NUMBER_H + +#include + +namespace mp { +template +LocalType BuildPrimitive(InvokeContext& invoke_context, + const Value& value, + TypeList, + typename std::enable_if::value>::type* enable = nullptr) +{ + using E = std::make_unsigned_t>; + using T = std::make_unsigned_t; + static_assert(std::numeric_limits::max() >= std::numeric_limits::max(), "mismatched integral/enum types"); + return static_cast(value); +} + +template +LocalType BuildPrimitive(InvokeContext& invoke_context, + const Value& value, + TypeList, + typename std::enable_if::value, int>::type* enable = nullptr) +{ + static_assert( + std::numeric_limits::lowest() <= std::numeric_limits::lowest(), "mismatched integral types"); + static_assert( + std::numeric_limits::max() >= std::numeric_limits::max(), "mismatched integral types"); + return value; +} + +template +LocalType BuildPrimitive(InvokeContext& invoke_context, + const Value& value, + TypeList, + typename std::enable_if::value>::type* enable = nullptr) +{ + static_assert(std::is_same::value, + "mismatched floating point types. please fix message.capnp type declaration to match wrapped interface"); + return value; +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename std::enable_if::value>::type* enable = 0) +{ + return read_dest.construct(static_cast(input.get())); +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename std::enable_if::value>::type* enable = nullptr) +{ + auto value = input.get(); + if (value < std::numeric_limits::min() || value > std::numeric_limits::max()) { + throw std::range_error("out of bound int received"); + } + return read_dest.construct(static_cast(value)); +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename std::enable_if::value>::type* enable = 0) +{ + auto value = input.get(); + static_assert(std::is_same::value, "floating point type mismatch"); + return read_dest.construct(value); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_NUMBER_H diff --git a/include/mp/type-optional.h b/include/mp/type-optional.h new file mode 100644 index 00000000000..822508d5533 --- /dev/null +++ b/include/mp/type-optional.h @@ -0,0 +1,48 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_OPTIONAL_H +#define MP_PROXY_TYPE_OPTIONAL_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + if (value) { + output.setHas(); + // FIXME: should std::move value if destvalue is rref? + BuildField(TypeList(), invoke_context, output, *value); + } +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + if (!input.has()) { + value.reset(); + } else if (value) { + ReadField(TypeList(), invoke_context, input, ReadDestUpdate(*value)); + } else { + ReadField(TypeList(), invoke_context, input, + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + value.emplace(std::forward(args)...); + return *value; + })); + } + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_OPTIONAL_H diff --git a/include/mp/type-pair.h b/include/mp/type-pair.h new file mode 100644 index 00000000000..3af9c9313de --- /dev/null +++ b/include/mp/type-pair.h @@ -0,0 +1,50 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_PAIR_H +#define MP_PROXY_TYPE_PAIR_H + +#include + +namespace mp { +// FIXME: Overload on output type instead of value type and switch to std::get and merge with next overload +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + auto pair = output.init(); + using Accessors = typename ProxyStruct::Accessors; + BuildField(TypeList(), invoke_context, Make>(pair), value.first); + BuildField(TypeList(), invoke_context, Make>(pair), value.second); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + const auto& pair = input.get(); + using Accessors = typename ProxyStruct::Reads>::Accessors; + + ReadField(TypeList(), invoke_context, Make>(pair), + ReadDestEmplace(TypeList(), [&](auto&&... key_args) -> auto& { + KeyLocalType* key = nullptr; + ReadField(TypeList(), invoke_context, Make>(pair), + ReadDestEmplace(TypeList(), [&](auto&&... value_args) -> auto& { + auto& ret = read_dest.construct(std::piecewise_construct, std::forward_as_tuple(key_args...), + std::forward_as_tuple(value_args...)); + key = &ret.first; + return ret.second; + })); + return *key; + })); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_PAIR_H diff --git a/include/mp/type-pointer.h b/include/mp/type-pointer.h new file mode 100644 index 00000000000..5c79e8d2e8a --- /dev/null +++ b/include/mp/type-pointer.h @@ -0,0 +1,113 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_POINTER_H +#define MP_PROXY_TYPE_POINTER_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, Priority<3>, InvokeContext& invoke_context, Value&& value, Output&& output) +{ + if (value) { + BuildField(TypeList(), invoke_context, output, *value); + } +} + +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + if (value) { + BuildField(TypeList(), invoke_context, output, *value); + } +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + if (value) { + ReadField(TypeList(), invoke_context, std::forward(input), ReadDestUpdate(*value)); + } + }); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<0>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + if (!input.has()) { + value.reset(); + } else if (value) { + ReadField(TypeList(), invoke_context, input, ReadDestUpdate(*value)); + } else { + ReadField(TypeList(), invoke_context, input, + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + value = std::make_shared(std::forward(args)...); + return *value; + })); + } + }); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + if (!input.has()) { + value.reset(); + return; + } + ReadField(TypeList(), invoke_context, std::forward(input), + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + value = std::make_shared(std::forward(args)...); + return *value; + })); + }); +} + +//! PassField override for C++ pointer arguments. +template +void PassField(Priority<1>, TypeList, ServerContext& server_context, const Fn& fn, Args&&... args) +{ + const auto& params = server_context.call_context.getParams(); + const auto& input = Make(params); + + if (!input.want()) { + fn.invoke(server_context, std::forward(args)..., nullptr); + return; + } + + InvokeContext& invoke_context = server_context; + Decay param; + + MaybeReadField(std::integral_constant(), TypeList(), invoke_context, input, + ReadDestUpdate(param)); + + fn.invoke(server_context, std::forward(args)..., ¶m); + + auto&& results = server_context.call_context.getResults(); + MaybeBuildField(std::integral_constant(), TypeList(), invoke_context, + Make(results), param); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_POINTER_H diff --git a/include/mp/type-set.h b/include/mp/type-set.h new file mode 100644 index 00000000000..ea60dc4d1a0 --- /dev/null +++ b/include/mp/type-set.h @@ -0,0 +1,48 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_SET_H +#define MP_PROXY_TYPE_SET_H + +#include +#include + +namespace mp { +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + // FIXME dededup with vector handler above + auto list = output.init(value.size()); + size_t i = 0; + for (const auto& elem : value) { + BuildField(TypeList(), invoke_context, ListOutput(list, i), elem); + ++i; + } +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.clear(); + for (auto item : data) { + ReadField(TypeList(), invoke_context, Make(item), + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + return *value.emplace(std::forward(args)...).first; + })); + } + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_SET_H diff --git a/include/mp/type-string.h b/include/mp/type-string.h new file mode 100644 index 00000000000..77d04acb280 --- /dev/null +++ b/include/mp/type-string.h @@ -0,0 +1,34 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_STRING_H +#define MP_PROXY_TYPE_STRING_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + auto result = output.init(value.size()); + memcpy(result.begin(), value.data(), value.size()); +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + auto data = input.get(); + return read_dest.construct(CharCast(data.begin()), data.size()); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_STRING_H diff --git a/include/mp/type-struct.h b/include/mp/type-struct.h new file mode 100644 index 00000000000..d282e20e9a4 --- /dev/null +++ b/include/mp/type-struct.h @@ -0,0 +1,85 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_STRUCT_H +#define MP_PROXY_TYPE_STRUCT_H + +#include + +namespace mp { +template +void BuildOne(TypeList param, + InvokeContext& invoke_context, + Output&& output, + Value&& value, + typename std::enable_if < index::fields>::type * enable = nullptr) +{ + using Index = std::integral_constant; + using Struct = typename ProxyType::Struct; + using Accessor = typename std::tuple_element::Accessors>::type; + auto&& field_output = Make(output); + auto&& field_value = value.*ProxyType::get(Index()); + BuildField(TypeList>(), invoke_context, field_output, field_value); + BuildOne(param, invoke_context, output, value); +} + +template +void BuildOne(TypeList param, + InvokeContext& invoke_context, + Output&& output, + Value&& value, + typename std::enable_if::fields>::type* enable = nullptr) +{ +} + +template +void CustomBuildField(TypeList local_type, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output, + typename ProxyType::Struct* enable = nullptr) +{ + BuildOne<0>(local_type, invoke_context, output.init(), value); +} + +template +void ReadOne(TypeList param, + InvokeContext& invoke_context, + Input&& input, + Value&& value, + typename std::enable_if::fields>::type* enable = nullptr) +{ + using Index = std::integral_constant; + using Struct = typename ProxyType::Struct; + using Accessor = typename std::tuple_element::Accessors>::type; + const auto& struc = input.get(); + auto&& field_value = value.*ProxyType::get(Index()); + ReadField(TypeList>(), invoke_context, Make(struc), + ReadDestUpdate(field_value)); + ReadOne(param, invoke_context, input, value); +} + +template +void ReadOne(TypeList param, + InvokeContext& invoke_context, + Input& input, + Value& value, + typename std::enable_if::fields>::type* enable = nullptr) +{ +} + +template +decltype(auto) CustomReadField(TypeList param, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename ProxyType::Struct* enable = nullptr) +{ + return read_dest.update([&](auto& value) { ReadOne<0>(param, invoke_context, input, value); }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_STRUCT_H diff --git a/include/mp/type-threadmap.h b/include/mp/type-threadmap.h new file mode 100644 index 00000000000..683586fbc6e --- /dev/null +++ b/include/mp/type-threadmap.h @@ -0,0 +1,41 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_THREADMAP_H +#define MP_PROXY_TYPE_THREADMAP_H + +#include + +namespace mp { +template <> +struct ProxyServer final : public virtual ThreadMap::Server +{ +public: + ProxyServer(Connection& connection); + kj::Promise makeThread(MakeThreadContext context) override; + Connection& m_connection; +}; + +template +void CustomBuildField(TypeList<>, + Priority<1>, + InvokeContext& invoke_context, + Output&& output, + typename std::enable_if::value>::type* enable = nullptr) +{ + output.set(kj::heap>(invoke_context.connection)); +} + +template +decltype(auto) CustomReadField(TypeList<>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + typename std::enable_if::value>::type* enable = nullptr) +{ + invoke_context.connection.m_thread_map = input.get(); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_THREADMAP_H diff --git a/include/mp/type-tuple.h b/include/mp/type-tuple.h new file mode 100644 index 00000000000..5083887258f --- /dev/null +++ b/include/mp/type-tuple.h @@ -0,0 +1,45 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_TUPLE_H +#define MP_PROXY_TYPE_TUPLE_H + +#include + +namespace mp { +// TODO: Should generalize this to work with arbitrary length tuples, not just length 2-tuples. +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + auto pair = output.init(); + using Accessors = typename ProxyStruct::Accessors; + BuildField(TypeList(), invoke_context, Make>(pair), std::get<0>(value)); + BuildField(TypeList(), invoke_context, Make>(pair), std::get<1>(value)); +} + +// TODO: Should generalize this to work with arbitrary length tuples, not just length 2-tuples. +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + const auto& pair = input.get(); + using Struct = ProxyStruct::Reads>; + using Accessors = typename Struct::Accessors; + ReadField(TypeList(), invoke_context, Make>(pair), + ReadDestUpdate(std::get<0>(value))); + ReadField(TypeList(), invoke_context, Make>(pair), + ReadDestUpdate(std::get<1>(value))); + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_TUPLE_H diff --git a/include/mp/type-vector.h b/include/mp/type-vector.h new file mode 100644 index 00000000000..e4996e93043 --- /dev/null +++ b/include/mp/type-vector.h @@ -0,0 +1,71 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_VECTOR_H +#define MP_PROXY_TYPE_VECTOR_H + +#include +#include + +namespace mp { +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + // FIXME dedup with set handler below + auto list = output.init(value.size()); + size_t i = 0; + for (auto it = value.begin(); it != value.end(); ++it, ++i) { + BuildField(TypeList(), invoke_context, ListOutput(list, i), *it); + } +} + +inline static bool BuildPrimitive(InvokeContext& invoke_context, std::vector::const_reference value, TypeList) +{ + return value; +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.clear(); + value.reserve(data.size()); + for (auto item : data) { + ReadField(TypeList(), invoke_context, Make(item), + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + value.emplace_back(std::forward(args)...); + return value.back(); + })); + } + }); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.clear(); + value.reserve(data.size()); + for (auto item : data) { + value.push_back(ReadField(TypeList(), invoke_context, Make(item), ReadDestTemp())); + } + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_VECTOR_H diff --git a/include/mp/type-void.h b/include/mp/type-void.h new file mode 100644 index 00000000000..0a887680529 --- /dev/null +++ b/include/mp/type-void.h @@ -0,0 +1,23 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_VOID_H +#define MP_PROXY_TYPE_VOID_H + +#include + +namespace mp { +template +::capnp::Void BuildPrimitive(InvokeContext& invoke_context, Value&&, TypeList<::capnp::Void>) +{ + return {}; +} + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, ::capnp::Void, Output&& output) +{ +} +} // namespace mp + +#endif // MP_PROXY_TYPE_VOID_H diff --git a/include/mp/util.h b/include/mp/util.h new file mode 100644 index 00000000000..ebfc3b5e720 --- /dev/null +++ b/include/mp/util.h @@ -0,0 +1,220 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_UTIL_H +#define MP_UTIL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mp { + +//! Generic utility functions used by capnp code. + +//! Type holding a list of types. +//! +//! Example: +//! TypeList +template +struct TypeList +{ + static constexpr size_t size = sizeof...(Types); +}; + +//! Construct a template class value by deducing template arguments from the +//! types of constructor arguments, so they don't need to be specified manually. +//! +//! Uses of this can go away with class template deduction in C++17 +//! (https://en.cppreference.com/w/cpp/language/class_template_argument_deduction) +//! +//! Example: +//! Make(5, true) // Constructs std::pair(5, true); +template