highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2026-06-04 02:02:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Squashed 'src/ipc/libmultiprocess/' content from commit 35944ffd23fa

Browse Source 
git-subtree-dir: src/ipc/libmultiprocess
git-subtree-split: 35944ffd23fa26652b82210351d50e896ce16c8f
This commit is contained in:
Ryan Ofsky
2025-04-02 21:41:16 +08:00
commit a2f28e4be9
60 changed files with 5878 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

671
src/mp/gen.cpp Normal file
View File

@@ -0,0 +1,671 @@
// Copyright (c) 2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <mp/config.h>
#include <mp/util.h>
#include <algorithm>
#include <capnp/schema-parser.h>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <errno.h>
#include <fstream>
#include <functional>
#include <iostream>
#include <kj/array.h>
#include <kj/common.h>
#include <kj/filesystem.h>
#include <kj/memory.h>
#include <kj/string.h>
#include <map>
#include <set>
#include <sstream>
#include <stdexcept>
#include <string>
#include <system_error>
#include <unistd.h>
#include <vector>
#define PROXY_BIN "mpgen"
#define PROXY_DECL "mp/proxy.h"
#define PROXY_TYPES "mp/proxy-types.h"
constexpr uint64_t NAMESPACE_ANNOTATION_ID = 0xb9c6f99ebf805f2cull; // From c++.capnp
constexpr uint64_t INCLUDE_ANNOTATION_ID = 0xb899f3c154fdb458ull; // From proxy.capnp
constexpr uint64_t INCLUDE_TYPES_ANNOTATION_ID = 0xbcec15648e8a0cf1ull; // From proxy.capnp
constexpr uint64_t WRAP_ANNOTATION_ID = 0xe6f46079b7b1405eull; // From proxy.capnp
constexpr uint64_t COUNT_ANNOTATION_ID = 0xd02682b319f69b38ull; // From proxy.capnp
constexpr uint64_t EXCEPTION_ANNOTATION_ID = 0x996a183200992f88ull; // From proxy.capnp
constexpr uint64_t NAME_ANNOTATION_ID = 0xb594888f63f4dbb9ull; // From proxy.capnp
constexpr uint64_t SKIP_ANNOTATION_ID = 0x824c08b82695d8ddull; // From proxy.capnp
template <typename Reader>
static bool AnnotationExists(const Reader& reader, uint64_t id)
{
for (const auto annotation : reader.getAnnotations()) {
if (annotation.getId() == id) {
return true;
}
}
return false;
}
template <typename Reader>
static bool GetAnnotationText(const Reader& reader, uint64_t id, kj::StringPtr* result)
{
for (const auto annotation : reader.getAnnotations()) {
if (annotation.getId() == id) {
*result = annotation.getValue().getText();
return true;
}
}
return false;
}
template <typename Reader>
static bool GetAnnotationInt32(const Reader& reader, uint64_t id, int32_t* result)
{
for (const auto annotation : reader.getAnnotations()) {
if (annotation.getId() == id) {
*result = annotation.getValue().getInt32();
return true;
}
}
return false;
}
static void ForEachMethod(const capnp::InterfaceSchema& interface, const std::function<void(const capnp::InterfaceSchema& interface, const capnp::InterfaceSchema::Method)>& callback)
{
for (const auto super : interface.getSuperclasses()) {
ForEachMethod(super, callback);
}
for (const auto method : interface.getMethods()) {
callback(interface, method);
}
}
using CharSlice = kj::ArrayPtr<const char>;
// Overload for any type with a string .begin(), like kj::StringPtr and kj::ArrayPtr<char>.
template <class OutputStream, class Array, const char* Enable = decltype(std::declval<Array>().begin())()>
static OutputStream& operator<<(OutputStream& os, const Array& array)
{
os.write(array.begin(), array.size());
return os;
}
struct Format
{
template <typename Value>
Format& operator<<(Value&& value)
{
m_os << value;
return *this;
}
operator std::string() const { return m_os.str(); }
std::ostringstream m_os;
};
static std::string Cap(kj::StringPtr str)
{
std::string result = str;
if (!result.empty() && 'a' <= result[0] && result[0] <= 'z') result[0] -= 'a' - 'A';
return result;
}
static bool BoxedType(const ::capnp::Type& type)
{
return !(type.isVoid() || type.isBool() || type.isInt8() || type.isInt16() || type.isInt32() || type.isInt64() ||
type.isUInt8() || type.isUInt16() || type.isUInt32() || type.isUInt64() || type.isFloat32() ||
type.isFloat64() || type.isEnum());
}
// src_file is path to .capnp file to generate stub code from.
//
// src_prefix can be used to generate outputs in a different directory than the
// source directory. For example if src_file is "/a/b/c/d/file.canp", and
// src_prefix is "/a/b", then output files will be "c/d/file.capnp.h"
// "c/d/file.capnp.cxx" "c/d/file.capnp.proxy.h", etc. This is equivalent to
// the capnp "--src-prefix" option (see "capnp help compile").
//
// include_prefix can be used to control relative include paths used in
// generated files. For example if src_file is "/a/b/c/d/file.canp" and
// include_prefix is "/a/b/c" include lines like
// "#include <d/file.capnp.proxy.h>" "#include <d/file.capnp.proxy-types.h>"i
// will be generated.
static void Generate(kj::StringPtr src_prefix,
kj::StringPtr include_prefix,
kj::StringPtr src_file,
const std::vector<kj::StringPtr>& import_paths,
const kj::ReadableDirectory& src_dir,
const std::vector<kj::Own<const kj::ReadableDirectory>>& import_dirs)
{
std::string output_path;
if (src_prefix == ".") {
output_path = src_file;
} else if (!src_file.startsWith(src_prefix) || src_file.size() <= src_prefix.size() ||
src_file[src_prefix.size()] != '/') {
throw std::runtime_error("src_prefix is not src_file prefix");
} else {
output_path = src_file.slice(src_prefix.size() + 1);
}
std::string include_path;
if (include_prefix == ".") {
include_path = src_file;
} else if (!src_file.startsWith(include_prefix) || src_file.size() <= include_prefix.size() ||
src_file[include_prefix.size()] != '/') {
throw std::runtime_error("include_prefix is not src_file prefix");
} else {
include_path = src_file.slice(include_prefix.size() + 1);
}
std::string include_base = include_path;
const std::string::size_type p = include_base.rfind('.');
if (p != std::string::npos) include_base.erase(p);
std::vector<std::string> args;
args.emplace_back(capnp_PREFIX "/bin/capnp");
args.emplace_back("compile");
args.emplace_back("--src-prefix=");
args.back().append(src_prefix.cStr(), src_prefix.size());
for (const auto& import_path : import_paths) {
args.emplace_back("--import-path=");
args.back().append(import_path.cStr(), import_path.size());
}
args.emplace_back("--output=" capnp_PREFIX "/bin/capnpc-c++");
args.emplace_back(src_file);
const int pid = fork();
if (pid == -1) {
throw std::system_error(errno, std::system_category(), "fork");
}
if (!pid) {
mp::ExecProcess(args);
}
const int status = mp::WaitProcess(pid);
if (status) {
throw std::runtime_error("Invoking " capnp_PREFIX "/bin/capnp failed");
}
const capnp::SchemaParser parser;
auto directory_pointers = kj::heapArray<const kj::ReadableDirectory*>(import_dirs.size());
for (size_t i = 0; i < import_dirs.size(); ++i) {
directory_pointers[i] = import_dirs[i].get();
}
auto file_schema = parser.parseFromDirectory(src_dir, kj::Path::parse(output_path), directory_pointers);
std::ofstream cpp_server(output_path + ".proxy-server.c++");
cpp_server << "// Generated by " PROXY_BIN " from " << src_file << "\n\n";
cpp_server << "#include <" << include_path << ".proxy-types.h>\n";
cpp_server << "#include <" << PROXY_TYPES << ">\n\n";
cpp_server << "namespace mp {\n";
std::ofstream cpp_client(output_path + ".proxy-client.c++");
cpp_client << "// Generated by " PROXY_BIN " from " << src_file << "\n\n";
cpp_client << "#include <" << include_path << ".proxy-types.h>\n";
cpp_client << "#include <" << PROXY_TYPES << ">\n\n";
cpp_client << "namespace mp {\n";
std::ofstream cpp_types(output_path + ".proxy-types.c++");
cpp_types << "// Generated by " PROXY_BIN " from " << src_file << "\n\n";
cpp_types << "#include <" << include_path << ".proxy-types.h>\n";
cpp_types << "#include <" << PROXY_TYPES << ">\n\n";
cpp_types << "namespace mp {\n";
std::string guard = output_path;
std::transform(guard.begin(), guard.end(), guard.begin(), [](unsigned char c) -> unsigned char {
if ('0' <= c && c <= '9') return c;
if ('A' <= c && c <= 'Z') return c;
if ('a' <= c && c <= 'z') return c - 'a' + 'A';
return '_';
});
std::ofstream inl(output_path + ".proxy-types.h");
inl << "// Generated by " PROXY_BIN " from " << src_file << "\n\n";
inl << "#ifndef " << guard << "_PROXY_TYPES_H\n";
inl << "#define " << guard << "_PROXY_TYPES_H\n\n";
inl << "#include <" << include_path << ".proxy.h>\n";
for (const auto annotation : file_schema.getProto().getAnnotations()) {
if (annotation.getId() == INCLUDE_TYPES_ANNOTATION_ID) {
inl << "#include <" << annotation.getValue().getText() << ">\n";
}
}
inl << "namespace mp {\n";
std::ofstream h(output_path + ".proxy.h");
h << "// Generated by " PROXY_BIN " from " << src_file << "\n\n";
h << "#ifndef " << guard << "_PROXY_H\n";
h << "#define " << guard << "_PROXY_H\n\n";
h << "#include <" << include_path << ".h>\n";
for (const auto annotation : file_schema.getProto().getAnnotations()) {
if (annotation.getId() == INCLUDE_ANNOTATION_ID) {
h << "#include <" << annotation.getValue().getText() << ">\n";
}
}
h << "#include <" << PROXY_DECL << ">\n\n";
h << "#if defined(__GNUC__)\n";
h << "#pragma GCC diagnostic push\n";
h << "#if !defined(__has_warning)\n";
h << "#pragma GCC diagnostic ignored \"-Wsuggest-override\"\n";
h << "#elif __has_warning(\"-Wsuggest-override\")\n";
h << "#pragma GCC diagnostic ignored \"-Wsuggest-override\"\n";
h << "#endif\n";
h << "#endif\n";
h << "namespace mp {\n";
kj::StringPtr message_namespace;
GetAnnotationText(file_schema.getProto(), NAMESPACE_ANNOTATION_ID, &message_namespace);
std::string base_name = include_base;
const size_t output_slash = base_name.rfind('/');
if (output_slash != std::string::npos) {
base_name.erase(0, output_slash + 1);
}
std::ostringstream methods;
std::set<kj::StringPtr> accessors_done;
std::ostringstream accessors;
std::ostringstream dec;
std::ostringstream def_server;
std::ostringstream def_client;
std::ostringstream int_client;
std::ostringstream def_types;
auto add_accessor = [&](kj::StringPtr name) {
if (!accessors_done.insert(name).second) return;
const std::string cap = Cap(name);
accessors << "struct " << cap << "\n";
accessors << "{\n";
accessors << " template<typename S> static auto get(S&& s) -> decltype(s.get" << cap << "()) { return s.get" << cap << "(); }\n";
accessors << " template<typename S> static bool has(S&& s) { return s.has" << cap << "(); }\n";
accessors << " template<typename S, typename A> static void set(S&& s, A&& a) { s.set" << cap
<< "(std::forward<A>(a)); }\n";
accessors << " template<typename S, typename... A> static decltype(auto) init(S&& s, A&&... a) { return s.init"
<< cap << "(std::forward<A>(a)...); }\n";
accessors << " template<typename S> static bool getWant(S&& s) { return s.getWant" << cap << "(); }\n";
accessors << " template<typename S> static void setWant(S&& s) { s.setWant" << cap << "(true); }\n";
accessors << " template<typename S> static bool getHas(S&& s) { return s.getHas" << cap << "(); }\n";
accessors << " template<typename S> static void setHas(S&& s) { s.setHas" << cap << "(true); }\n";
accessors << "};\n";
};
for (const auto node_nested : file_schema.getProto().getNestedNodes()) {
kj::StringPtr node_name = node_nested.getName();
const auto& node = file_schema.getNested(node_name);
kj::StringPtr proxied_class_type;
GetAnnotationText(node.getProto(), WRAP_ANNOTATION_ID, &proxied_class_type);
if (node.getProto().isStruct()) {
const auto& struc = node.asStruct();
std::ostringstream generic_name;
generic_name << node_name;
dec << "template<";
bool first_param = true;
for (const auto param : node.getProto().getParameters()) {
if (first_param) {
first_param = false;
generic_name << "<";
} else {
dec << ", ";
generic_name << ", ";
}
dec << "typename " << param.getName();
generic_name << "" << param.getName();
}
if (!first_param) generic_name << ">";
dec << ">\n";
dec << "struct ProxyStruct<" << message_namespace << "::" << generic_name.str() << ">\n";
dec << "{\n";
dec << " using Struct = " << message_namespace << "::" << generic_name.str() << ";\n";
for (const auto field : struc.getFields()) {
auto field_name = field.getProto().getName();
add_accessor(field_name);
dec << " using " << Cap(field_name) << "Accessor = Accessor<" << base_name
<< "_fields::" << Cap(field_name) << ", FIELD_IN | FIELD_OUT";
if (BoxedType(field.getType())) dec << " | FIELD_BOXED";
dec << ">;\n";
}
dec << " using Accessors = std::tuple<";
size_t i = 0;
for (const auto field : struc.getFields()) {
if (AnnotationExists(field.getProto(), SKIP_ANNOTATION_ID)) {
continue;
}
if (i) dec << ", ";
dec << Cap(field.getProto().getName()) << "Accessor";
++i;
}
dec << ">;\n";
dec << " static constexpr size_t fields = " << i << ";\n";
dec << "};\n";
if (proxied_class_type.size()) {
inl << "template<>\n";
inl << "struct ProxyType<" << proxied_class_type << ">\n";
inl << "{\n";
inl << "public:\n";
inl << " using Struct = " << message_namespace << "::" << node_name << ";\n";
size_t i = 0;
for (const auto field : struc.getFields()) {
if (AnnotationExists(field.getProto(), SKIP_ANNOTATION_ID)) {
continue;
}
auto field_name = field.getProto().getName();
auto member_name = field_name;
GetAnnotationText(field.getProto(), NAME_ANNOTATION_ID, &member_name);
inl << " static decltype(auto) get(std::integral_constant<size_t, " << i << ">) { return "
<< "&" << proxied_class_type << "::" << member_name << "; }\n";
++i;
}
inl << " static constexpr size_t fields = " << i << ";\n";
inl << "};\n";
}
}
if (proxied_class_type.size() && node.getProto().isInterface()) {
const auto& interface = node.asInterface();
std::ostringstream client;
client << "template<>\nstruct ProxyClient<" << message_namespace << "::" << node_name << "> final : ";
client << "public ProxyClientCustom<" << message_namespace << "::" << node_name << ", "
<< proxied_class_type << ">\n{\n";
client << "public:\n";
client << " using ProxyClientCustom::ProxyClientCustom;\n";
client << " ~ProxyClient();\n";
std::ostringstream server;
server << "template<>\nstruct ProxyServer<" << message_namespace << "::" << node_name << "> : public "
<< "ProxyServerCustom<" << message_namespace << "::" << node_name << ", " << proxied_class_type
<< ">\n{\n";
server << "public:\n";
server << " using ProxyServerCustom::ProxyServerCustom;\n";
server << " ~ProxyServer();\n";
const std::ostringstream client_construct;
const std::ostringstream client_destroy;
int method_ordinal = 0;
ForEachMethod(interface, [&] (const capnp::InterfaceSchema& method_interface, const capnp::InterfaceSchema::Method& method) {
const kj::StringPtr method_name = method.getProto().getName();
kj::StringPtr proxied_method_name = method_name;
GetAnnotationText(method.getProto(), NAME_ANNOTATION_ID, &proxied_method_name);
const std::string method_prefix = Format() << message_namespace << "::" << method_interface.getShortDisplayName()
<< "::" << Cap(method_name);
const bool is_construct = method_name == "construct";
const bool is_destroy = method_name == "destroy";
struct Field
{
::capnp::StructSchema::Field param;
bool param_is_set = false;
::capnp::StructSchema::Field result;
bool result_is_set = false;
int args = 0;
bool retval = false;
bool optional = false;
bool requested = false;
bool skip = false;
kj::StringPtr exception;
};
std::vector<Field> fields;
std::map<kj::StringPtr, int> field_idx; // name -> args index
bool has_result = false;
auto add_field = [&](const ::capnp::StructSchema::Field& schema_field, bool param) {
if (AnnotationExists(schema_field.getProto(), SKIP_ANNOTATION_ID)) {
return;
}
auto field_name = schema_field.getProto().getName();
auto inserted = field_idx.emplace(field_name, fields.size());
if (inserted.second) {
fields.emplace_back();
}
auto& field = fields[inserted.first->second];
if (param) {
field.param = schema_field;
field.param_is_set = true;
} else {
field.result = schema_field;
field.result_is_set = true;
}
if (!param && field_name == "result") {
field.retval = true;
has_result = true;
}
GetAnnotationText(schema_field.getProto(), EXCEPTION_ANNOTATION_ID, &field.exception);
int32_t count = 1;
if (!GetAnnotationInt32(schema_field.getProto(), COUNT_ANNOTATION_ID, &count)) {
if (schema_field.getType().isStruct()) {
GetAnnotationInt32(schema_field.getType().asStruct().getProto(),
COUNT_ANNOTATION_ID, &count);
} else if (schema_field.getType().isInterface()) {
GetAnnotationInt32(schema_field.getType().asInterface().getProto(),
COUNT_ANNOTATION_ID, &count);
}
}
if (inserted.second && !field.retval && !field.exception.size()) {
field.args = count;
}
};
for (const auto schema_field : method.getParamType().getFields()) {
add_field(schema_field, true);
}
for (const auto schema_field : method.getResultType().getFields()) {
add_field(schema_field, false);
}
for (auto& field : field_idx) {
auto has_field = field_idx.find("has" + Cap(field.first));
if (has_field != field_idx.end()) {
fields[has_field->second].skip = true;
fields[field.second].optional = true;
}
auto want_field = field_idx.find("want" + Cap(field.first));
if (want_field != field_idx.end() && fields[want_field->second].param_is_set) {
fields[want_field->second].skip = true;
fields[field.second].requested = true;
}
}
if (!is_construct && !is_destroy && (&method_interface == &interface)) {
methods << "template<>\n";
methods << "struct ProxyMethod<" << method_prefix << "Params>\n";
methods << "{\n";
methods << " static constexpr auto impl = &" << proxied_class_type
<< "::" << proxied_method_name << ";\n";
methods << "};\n\n";
}
std::ostringstream client_args;
std::ostringstream client_invoke;
std::ostringstream server_invoke_start;
std::ostringstream server_invoke_end;
int argc = 0;
for (const auto& field : fields) {
if (field.skip) continue;
const auto& f = field.param_is_set ? field.param : field.result;
auto field_name = f.getProto().getName();
auto field_type = f.getType();
std::ostringstream field_flags;
if (!field.param_is_set) {
field_flags << "FIELD_OUT";
} else if (field.result_is_set) {
field_flags << "FIELD_IN | FIELD_OUT";
} else {
field_flags << "FIELD_IN";
}
if (field.optional) field_flags << " | FIELD_OPTIONAL";
if (field.requested) field_flags << " | FIELD_REQUESTED";
if (BoxedType(field_type)) field_flags << " | FIELD_BOXED";
add_accessor(field_name);
for (int i = 0; i < field.args; ++i) {
if (argc > 0) client_args << ",";
client_args << "M" << method_ordinal << "::Param<" << argc << "> " << field_name;
if (field.args > 1) client_args << i;
++argc;
}
client_invoke << ", ";
if (field.exception.size()) {
client_invoke << "ClientException<" << field.exception << ", ";
} else {
client_invoke << "MakeClientParam<";
}
client_invoke << "Accessor<" << base_name << "_fields::" << Cap(field_name) << ", "
<< field_flags.str() << ">>(";
if (field.retval || field.args == 1) {
client_invoke << field_name;
} else {
for (int i = 0; i < field.args; ++i) {
if (i > 0) client_invoke << ", ";
client_invoke << field_name << i;
}
}
client_invoke << ")";
if (field.exception.size()) {
server_invoke_start << "Make<ServerExcept, " << field.exception;
} else if (field.retval) {
server_invoke_start << "Make<ServerRet";
} else {
server_invoke_start << "MakeServerField<" << field.args;
}
server_invoke_start << ", Accessor<" << base_name << "_fields::" << Cap(field_name) << ", "
<< field_flags.str() << ">>(";
server_invoke_end << ")";
}
const std::string static_str{is_construct || is_destroy ? "static " : ""};
const std::string super_str{is_construct || is_destroy ? "Super& super" : ""};
const std::string self_str{is_construct || is_destroy ? "super" : "*this"};
client << " using M" << method_ordinal << " = ProxyClientMethodTraits<" << method_prefix
<< "Params>;\n";
client << " " << static_str << "typename M" << method_ordinal << "::Result " << method_name << "("
<< super_str << client_args.str() << ")";
client << ";\n";
def_client << "ProxyClient<" << message_namespace << "::" << node_name << ">::M" << method_ordinal
<< "::Result ProxyClient<" << message_namespace << "::" << node_name << ">::" << method_name
<< "(" << super_str << client_args.str() << ") {\n";
if (has_result) {
def_client << " typename M" << method_ordinal << "::Result result;\n";
}
def_client << " clientInvoke(" << self_str << ", &" << message_namespace << "::" << node_name
<< "::Client::" << method_name << "Request" << client_invoke.str() << ");\n";
if (has_result) def_client << " return result;\n";
def_client << "}\n";
server << " kj::Promise<void> " << method_name << "(" << Cap(method_name)
<< "Context call_context) override;\n";
def_server << "kj::Promise<void> ProxyServer<" << message_namespace << "::" << node_name
<< ">::" << method_name << "(" << Cap(method_name)
<< "Context call_context) {\n"
" return serverInvoke(*this, call_context, "
<< server_invoke_start.str();
if (is_destroy) {
def_server << "ServerDestroy()";
} else {
def_server << "ServerCall()";
}
def_server << server_invoke_end.str() << ");\n}\n";
++method_ordinal;
});
client << "};\n";
server << "};\n";
dec << "\n" << client.str() << "\n" << server.str() << "\n";
KJ_IF_MAYBE(bracket, proxied_class_type.findFirst('<')) {
// Skip ProxyType definition for complex type expressions which
// could lead to duplicate definitions. They can be defined
// manually if actually needed.
} else {
dec << "template<>\nstruct ProxyType<" << proxied_class_type << ">\n{\n";
dec << " using Type = " << proxied_class_type << ";\n";
dec << " using Message = " << message_namespace << "::" << node_name << ";\n";
dec << " using Client = ProxyClient<Message>;\n";
dec << " using Server = ProxyServer<Message>;\n";
dec << "};\n";
int_client << "ProxyTypeRegister t" << node_nested.getId() << "{TypeList<" << proxied_class_type << ">{}};\n";
}
def_types << "ProxyClient<" << message_namespace << "::" << node_name
<< ">::~ProxyClient() { clientDestroy(*this); " << client_destroy.str() << " }\n";
def_types << "ProxyServer<" << message_namespace << "::" << node_name
<< ">::~ProxyServer() { serverDestroy(*this); }\n";
}
}
h << methods.str() << "namespace " << base_name << "_fields {\n"
<< accessors.str() << "} // namespace " << base_name << "_fields\n"
<< dec.str();
cpp_server << def_server.str();
cpp_server << "} // namespace mp\n";
cpp_client << def_client.str();
cpp_client << "namespace {\n" << int_client.str() << "} // namespace\n";
cpp_client << "} // namespace mp\n";
cpp_types << def_types.str();
cpp_types << "} // namespace mp\n";
inl << "} // namespace mp\n";
inl << "#endif\n";
h << "} // namespace mp\n";
h << "#if defined(__GNUC__)\n";
h << "#pragma GCC diagnostic pop\n";
h << "#endif\n";
h << "#endif\n";
}
int main(int argc, char** argv)
{
if (argc < 3) {
std::cerr << "Usage: " << PROXY_BIN << " SRC_PREFIX INCLUDE_PREFIX SRC_FILE [IMPORT_PATH...]\n";
exit(1);
}
std::vector<kj::StringPtr> import_paths;
std::vector<kj::Own<const kj::ReadableDirectory>> import_dirs;
auto fs = kj::newDiskFilesystem();
auto cwd = fs->getCurrentPath();
kj::Own<const kj::ReadableDirectory> src_dir;
KJ_IF_MAYBE(dir, fs->getRoot().tryOpenSubdir(cwd.evalNative(argv[1]))) {
src_dir = kj::mv(*dir);
} else {
throw std::runtime_error(std::string("Failed to open src_prefix prefix directory: ") + argv[1]);
}
for (int i = 4; i < argc; ++i) {
KJ_IF_MAYBE(dir, fs->getRoot().tryOpenSubdir(cwd.evalNative(argv[i]))) {
import_paths.emplace_back(argv[i]);
import_dirs.emplace_back(kj::mv(*dir));
} else {
throw std::runtime_error(std::string("Failed to open import directory: ") + argv[i]);
}
}
for (const char* path : {CMAKE_INSTALL_PREFIX "/include", capnp_PREFIX "/include"}) {
KJ_IF_MAYBE(dir, fs->getRoot().tryOpenSubdir(cwd.evalNative(path))) {
import_paths.emplace_back(path);
import_dirs.emplace_back(kj::mv(*dir));
}
// No exception thrown if _PREFIX directories do not exist
}
Generate(argv[1], argv[2], argv[3], import_paths, *src_dir, import_dirs);
return 0;
}

399
src/mp/proxy.cpp Normal file
View File

@@ -0,0 +1,399 @@
// Copyright (c) 2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <mp/proxy.h>
#include <mp/proxy-io.h>
#include <mp/proxy-types.h>
#include <mp/proxy.capnp.h>
#include <mp/type-threadmap.h>
#include <mp/util.h>
#include <assert.h>
#include <atomic>
#include <capnp/blob.h>
#include <capnp/capability.h>
#include <condition_variable>
#include <functional>
#include <future>
#include <kj/async-io.h>
#include <kj/async.h>
#include <kj/common.h>
#include <kj/debug.h>
#include <kj/exception.h>
#include <kj/memory.h>
#include <map>
#include <memory>
#include <mutex>
#include <stddef.h>
#include <stdexcept>
#include <string>
#include <sys/socket.h>
#include <thread>
#include <tuple>
#include <unistd.h>
#include <utility>
namespace mp {
template <typename Interface>
struct ProxyServer;
thread_local ThreadContext g_thread_context;
void LoggingErrorHandler::taskFailed(kj::Exception&& exception)
{
KJ_LOG(ERROR, "Uncaught exception in daemonized task.", exception);
m_loop.log() << "Uncaught exception in daemonized task.";
}
Connection::~Connection()
{
// Shut down RPC system first, since this will garbage collect Server
// objects that were not freed before the connection was closed, some of
// which may call addAsyncCleanup and add more cleanup callbacks which can
// run below.
m_rpc_system.reset();
// ProxyClient cleanup handlers are in sync list, and ProxyServer cleanup
// handlers are in the async list.
//
// The ProxyClient cleanup handlers are synchronous because they are fast
// and don't do anything besides release capnp resources and reset state so
// future calls to client methods immediately throw exceptions instead of
// trying to communicating across the socket. The synchronous callbacks set
// ProxyClient capability pointers to null, so new method calls on client
// objects fail without triggering i/o or relying on event loop which may go
// out of scope or trigger obscure capnp i/o errors.
//
// The ProxySever cleanup handlers call user defined destructors on server
// object, which can run arbitrary blocking bitcoin code so they have to run
// asynchronously in a different thread. The asynchronous cleanup functions
// intentionally aren't started until after the synchronous cleanup
// functions run, so client objects are fully disconnected before bitcoin
// code in the destructors are run. This way if the bitcoin code tries to
// make client requests the requests will just fail immediately instead of
// sending i/o or accessing the event loop.
//
// The context where Connection objects are destroyed and this destructor is invoked
// is different depending on whether this is an outgoing connection being used
// to make an Init.makeX call() (e.g. Init.makeNode or Init.makeWalletClient) or an incoming
// connection implementing the Init interface and handling the Init.makeX() calls.
//
// Either way when a connection is closed, capnp behavior is to call all
// ProxyServer object destructors first, and then trigger an onDisconnect
// callback.
//
// On incoming side of the connection, the onDisconnect callback is written
// to delete the Connection object from the m_incoming_connections and call
// this destructor which calls Connection::disconnect.
//
// On the outgoing side, the Connection object is owned by top level client
// object client, which onDisconnect handler doesn't have ready access to,
// so onDisconnect handler just calls Connection::disconnect directly
// instead.
//
// Either way disconnect code runs in the event loop thread and called both
// on clean and unclean shutdowns. In unclean shutdown case when the
// connection is broken, sync and async cleanup lists will filled with
// callbacks. In the clean shutdown case both lists will be empty.
while (!m_sync_cleanup_fns.empty()) {
m_sync_cleanup_fns.front()();
m_sync_cleanup_fns.pop_front();
}
while (!m_async_cleanup_fns.empty()) {
const std::unique_lock<std::mutex> lock(m_loop.m_mutex);
m_loop.m_async_fns.emplace_back(std::move(m_async_cleanup_fns.front()));
m_async_cleanup_fns.pop_front();
}
std::unique_lock<std::mutex> lock(m_loop.m_mutex);
m_loop.startAsyncThread(lock);
m_loop.removeClient(lock);
}
CleanupIt Connection::addSyncCleanup(std::function<void()> fn)
{
const std::unique_lock<std::mutex> lock(m_loop.m_mutex);
// Add cleanup callbacks to the front of list, so sync cleanup functions run
// in LIFO order. This is a good approach because sync cleanup functions are
// added as client objects are created, and it is natural to clean up
// objects in the reverse order they were created. In practice, however,
// order should not be significant because the cleanup callbacks run
// synchronously in a single batch when the connection is broken, and they
// only reset the connection pointers in the client objects without actually
// deleting the client objects.
return m_sync_cleanup_fns.emplace(m_sync_cleanup_fns.begin(), std::move(fn));
}
void Connection::removeSyncCleanup(CleanupIt it)
{
const std::unique_lock<std::mutex> lock(m_loop.m_mutex);
m_sync_cleanup_fns.erase(it);
}
void Connection::addAsyncCleanup(std::function<void()> fn)
{
const std::unique_lock<std::mutex> lock(m_loop.m_mutex);
// Add async cleanup callbacks to the back of the list. Unlike the sync
// cleanup list, this list order is more significant because it determines
// the order server objects are destroyed when there is a sudden disconnect,
// and it is possible objects may need to be destroyed in a certain order.
// This function is called in ProxyServerBase destructors, and since capnp
// destroys ProxyServer objects in LIFO order, we should preserve this
// order, and add cleanup callbacks to the end of the list so they can be
// run starting from the beginning of the list.
//
// In bitcoin core, running these callbacks in the right order is
// particularly important for the wallet process, because it uses blocking
// shared_ptrs and requires Chain::Notification pointers owned by the node
// process to be destroyed before the WalletLoader objects owned by the node
// process, otherwise shared pointer counts of the CWallet objects (which
// inherit from Chain::Notification) will not be 1 when WalletLoader
// destructor runs and it will wait forever for them to be released.
m_async_cleanup_fns.emplace(m_async_cleanup_fns.end(), std::move(fn));
}
EventLoop::EventLoop(const char* exe_name, LogFn log_fn, void* context)
: m_exe_name(exe_name),
m_io_context(kj::setupAsyncIo()),
m_task_set(new kj::TaskSet(m_error_handler)),
m_log_fn(std::move(log_fn)),
m_context(context)
{
int fds[2];
KJ_SYSCALL(socketpair(AF_UNIX, SOCK_STREAM, 0, fds));
m_wait_fd = fds[0];
m_post_fd = fds[1];
}
EventLoop::~EventLoop()
{
if (m_async_thread.joinable()) m_async_thread.join();
const std::lock_guard<std::mutex> lock(m_mutex);
KJ_ASSERT(m_post_fn == nullptr);
KJ_ASSERT(m_async_fns.empty());
KJ_ASSERT(m_wait_fd == -1);
KJ_ASSERT(m_post_fd == -1);
KJ_ASSERT(m_num_clients == 0);
// Spin event loop. wait for any promises triggered by RPC shutdown.
// auto cleanup = kj::evalLater([]{});
// cleanup.wait(m_io_context.waitScope);
}
void EventLoop::loop()
{
assert(!g_thread_context.loop_thread);
g_thread_context.loop_thread = true;
KJ_DEFER(g_thread_context.loop_thread = false);
kj::Own<kj::AsyncIoStream> wait_stream{
m_io_context.lowLevelProvider->wrapSocketFd(m_wait_fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP)};
int post_fd{m_post_fd};
char buffer = 0;
for (;;) {
const size_t read_bytes = wait_stream->read(&buffer, 0, 1).wait(m_io_context.waitScope);
if (read_bytes != 1) throw std::logic_error("EventLoop wait_stream closed unexpectedly");
std::unique_lock<std::mutex> lock(m_mutex);
if (m_post_fn) {
Unlock(lock, *m_post_fn);
m_post_fn = nullptr;
m_cv.notify_all();
} else if (done(lock)) {
// Intentionally do not break if m_post_fn was set, even if done()
// would return true, to ensure that the removeClient write(post_fd)
// call always succeeds and the loop does not exit between the time
// that the done condition is set and the write call is made.
break;
}
}
log() << "EventLoop::loop done, cancelling event listeners.";
m_task_set.reset();
log() << "EventLoop::loop bye.";
wait_stream = nullptr;
KJ_SYSCALL(::close(post_fd));
const std::unique_lock<std::mutex> lock(m_mutex);
m_wait_fd = -1;
m_post_fd = -1;
}
void EventLoop::post(const std::function<void()>& fn)
{
if (std::this_thread::get_id() == m_thread_id) {
fn();
return;
}
std::unique_lock<std::mutex> lock(m_mutex);
addClient(lock);
m_cv.wait(lock, [this] { return m_post_fn == nullptr; });
m_post_fn = &fn;
int post_fd{m_post_fd};
Unlock(lock, [&] {
char buffer = 0;
KJ_SYSCALL(write(post_fd, &buffer, 1));
});
m_cv.wait(lock, [this, &fn] { return m_post_fn != &fn; });
removeClient(lock);
}
void EventLoop::addClient(std::unique_lock<std::mutex>& lock) { m_num_clients += 1; }
bool EventLoop::removeClient(std::unique_lock<std::mutex>& lock)
{
m_num_clients -= 1;
if (done(lock)) {
m_cv.notify_all();
int post_fd{m_post_fd};
lock.unlock();
char buffer = 0;
KJ_SYSCALL(write(post_fd, &buffer, 1)); // NOLINT(bugprone-suspicious-semicolon)
return true;
}
return false;
}
void EventLoop::startAsyncThread(std::unique_lock<std::mutex>& lock)
{
if (m_async_thread.joinable()) {
m_cv.notify_all();
} else if (!m_async_fns.empty()) {
m_async_thread = std::thread([this] {
std::unique_lock<std::mutex> lock(m_mutex);
while (true) {
if (!m_async_fns.empty()) {
addClient(lock);
const std::function<void()> fn = std::move(m_async_fns.front());
m_async_fns.pop_front();
Unlock(lock, fn);
if (removeClient(lock)) break;
continue;
} else if (m_num_clients == 0) {
break;
}
m_cv.wait(lock);
}
});
}
}
bool EventLoop::done(std::unique_lock<std::mutex>& lock)
{
assert(m_num_clients >= 0);
assert(lock.owns_lock());
assert(lock.mutex() == &m_mutex);
return m_num_clients == 0 && m_async_fns.empty();
}
std::tuple<ConnThread, bool> SetThread(ConnThreads& threads, std::mutex& mutex, Connection* connection, const std::function<Thread::Client()>& make_thread)
{
const std::unique_lock<std::mutex> lock(mutex);
auto thread = threads.find(connection);
if (thread != threads.end()) return {thread, false};
thread = threads.emplace(
std::piecewise_construct, std::forward_as_tuple(connection),
std::forward_as_tuple(make_thread(), connection, /* destroy_connection= */ false)).first;
thread->second.setCleanup([&threads, &mutex, thread] {
// Note: it is safe to use the `thread` iterator in this cleanup
// function, because the iterator would only be invalid if the map entry
// was removed, and if the map entry is removed the ProxyClient<Thread>
// destructor unregisters the cleanup.
// Connection is being destroyed before thread client is, so reset
// thread client m_cleanup_it member so thread client destructor does not
// try unregister this callback after connection is destroyed.
thread->second.m_cleanup_it.reset();
// Remove connection pointer about to be destroyed from the map
const std::unique_lock<std::mutex> lock(mutex);
threads.erase(thread);
});
return {thread, true};
}
ProxyClient<Thread>::~ProxyClient()
{
// If thread is being destroyed before connection is destroyed, remove the
// cleanup callback that was registered to handle the connection being
// destroyed before the thread being destroyed.
if (m_cleanup_it) {
m_context.connection->removeSyncCleanup(*m_cleanup_it);
}
}
void ProxyClient<Thread>::setCleanup(const std::function<void()>& fn)
{
assert(fn);
assert(!m_cleanup_it);
m_cleanup_it = m_context.connection->addSyncCleanup(fn);
}
ProxyServer<Thread>::ProxyServer(ThreadContext& thread_context, std::thread&& thread)
: m_thread_context(thread_context), m_thread(std::move(thread))
{
assert(m_thread_context.waiter.get() != nullptr);
}
ProxyServer<Thread>::~ProxyServer()
{
if (!m_thread.joinable()) return;
// Stop async thread and wait for it to exit. Need to wait because the
// m_thread handle needs to outlive the thread to avoid "terminate called
// without an active exception" error. An alternative to waiting would be
// detach the thread, but this would introduce nondeterminism which could
// make code harder to debug or extend.
assert(m_thread_context.waiter.get());
std::unique_ptr<Waiter> waiter;
{
const std::unique_lock<std::mutex> lock(m_thread_context.waiter->m_mutex);
//! Reset thread context waiter pointer, as shutdown signal for done
//! lambda passed as waiter->wait() argument in makeThread code below.
waiter = std::move(m_thread_context.waiter);
//! Assert waiter is idle. This destructor shouldn't be getting called if it is busy.
assert(!waiter->m_fn);
// Clear client maps now to avoid deadlock in m_thread.join() call
// below. The maps contain Thread::Client objects that need to be
// destroyed from the event loop thread (this thread), which can't
// happen if this thread is busy calling join.
m_thread_context.request_threads.clear();
m_thread_context.callback_threads.clear();
//! Ping waiter.
waiter->m_cv.notify_all();
}
m_thread.join();
}
kj::Promise<void> ProxyServer<Thread>::getName(GetNameContext context)
{
context.getResults().setResult(m_thread_context.thread_name);
return kj::READY_NOW;
}
ProxyServer<ThreadMap>::ProxyServer(Connection& connection) : m_connection(connection) {}
kj::Promise<void> ProxyServer<ThreadMap>::makeThread(MakeThreadContext context)
{
const std::string from = context.getParams().getName();
std::promise<ThreadContext*> thread_context;
std::thread thread([&thread_context, from, this]() {
g_thread_context.thread_name = ThreadName(m_connection.m_loop.m_exe_name) + " (from " + from + ")";
g_thread_context.waiter = std::make_unique<Waiter>();
thread_context.set_value(&g_thread_context);
std::unique_lock<std::mutex> lock(g_thread_context.waiter->m_mutex);
// Wait for shutdown signal from ProxyServer<Thread> destructor (signal
// is just waiter getting set to null.)
g_thread_context.waiter->wait(lock, [] { return !g_thread_context.waiter; });
});
auto thread_server = kj::heap<ProxyServer<Thread>>(*thread_context.get_future().get(), std::move(thread));
auto thread_client = m_connection.m_threads.add(kj::mv(thread_server));
context.getResults().setResult(kj::mv(thread_client));
return kj::READY_NOW;
}
std::atomic<int> server_reqs{0};
std::string LongThreadName(const char* exe_name)
{
return g_thread_context.thread_name.empty() ? ThreadName(exe_name) : g_thread_context.thread_name;
}
} // namespace mp

154
src/mp/util.cpp Normal file
View File

@@ -0,0 +1,154 @@
// Copyright (c) 2018-2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <mp/config.h>
#include <mp/util.h>
#include <errno.h>
#include <kj/common.h>
#include <kj/string-tree.h>
#include <pthread.h>
#include <sstream>
#include <stdio.h>
#include <string>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <system_error>
#include <unistd.h>
#include <utility>
#include <vector>
#ifdef __linux__
#include <sys/syscall.h>
#endif
#ifdef HAVE_PTHREAD_GETTHREADID_NP
#include <pthread_np.h>
#endif // HAVE_PTHREAD_GETTHREADID_NP
namespace mp {
namespace {
//! Return highest possible file descriptor.
size_t MaxFd()
{
struct rlimit nofile;
if (getrlimit(RLIMIT_NOFILE, &nofile) == 0) {
return nofile.rlim_cur - 1;
} else {
return 1023;
}
}
} // namespace
std::string ThreadName(const char* exe_name)
{
char thread_name[16] = {0};
#ifdef HAVE_PTHREAD_GETNAME_NP
pthread_getname_np(pthread_self(), thread_name, sizeof(thread_name));
#endif // HAVE_PTHREAD_GETNAME_NP
std::ostringstream buffer;
buffer << (exe_name ? exe_name : "") << "-" << getpid() << "/";
if (thread_name[0] != '\0') {
buffer << thread_name << "-";
}
// Prefer platform specific thread ids over the standard C++11 ones because
// the former are shorter and are the same as what gdb prints "LWP ...".
#ifdef __linux__
buffer << syscall(SYS_gettid);
#elif defined(HAVE_PTHREAD_THREADID_NP)
uint64_t tid = 0;
pthread_threadid_np(NULL, &tid);
buffer << tid;
#elif defined(HAVE_PTHREAD_GETTHREADID_NP)
buffer << pthread_getthreadid_np();
#else
buffer << std::this_thread::get_id();
#endif
return std::move(buffer).str();
}
std::string LogEscape(const kj::StringTree& string)
{
const int MAX_SIZE = 1000;
std::string result;
string.visit([&](const kj::ArrayPtr<const char>& piece) {
if (result.size() > MAX_SIZE) return;
for (const char c : piece) {
if (c == '\\') {
result.append("\\\\");
} else if (c < 0x20 || c > 0x7e) {
char escape[4];
snprintf(escape, 4, "\\%02x", c);
result.append(escape);
} else {
result.push_back(c);
}
if (result.size() > MAX_SIZE) {
result += "...";
break;
}
}
});
return result;
}
int SpawnProcess(int& pid, FdToArgsFn&& fd_to_args)
{
int fds[2];
if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0) {
throw std::system_error(errno, std::system_category(), "socketpair");
}
pid = fork();
if (pid == -1) {
throw std::system_error(errno, std::system_category(), "fork");
}
// Parent process closes the descriptor for socket 0, child closes the descriptor for socket 1.
if (close(fds[pid ? 0 : 1]) != 0) {
throw std::system_error(errno, std::system_category(), "close");
}
if (!pid) {
// Child process must close all potentially open descriptors, except socket 0.
const int maxFd = MaxFd();
for (int fd = 3; fd < maxFd; ++fd) {
if (fd != fds[0]) {
close(fd);
}
}
ExecProcess(fd_to_args(fds[0]));
}
return fds[1];
}
void ExecProcess(const std::vector<std::string>& args)
{
std::vector<char*> argv;
argv.reserve(args.size());
for (const auto& arg : args) {
argv.push_back(const_cast<char*>(arg.c_str()));
}
argv.push_back(nullptr);
if (execvp(argv[0], argv.data()) != 0) {
perror("execlp failed");
_exit(1);
}
}
int WaitProcess(int pid)
{
int status;
if (::waitpid(pid, &status, 0 /* options */) != pid) {
throw std::system_error(errno, std::system_category(), "waitpid");
}
return status;
}
} // namespace mp