Files
bitcoin/src/test/fuzz/rpc.cpp
Andrew Chow 1af72e728d Merge bitcoin/bitcoin#27501: mempool / rpc: add getprioritisedtransactions, delete a mapDeltas entry when delta==0
67b7fecacd [mempool] clear mapDeltas entry if prioritisetransaction sets delta to 0 (glozow)
c1061acb9d [functional test] prioritisation is not removed during replacement and expiry (glozow)
0e5874f0b0 [functional test] getprioritisedtransactions RPC (glozow)
99f8046829 [rpc] add getprioritisedtransactions (glozow)
9e9ca36c80 [mempool] add GetPrioritisedTransactions (glozow)

Pull request description:

  Add an RPC to get prioritised transactions (also tells you whether the tx is in mempool or not), helping users clean up `mapDeltas` manually. When `CTxMemPool::PrioritiseTransaction` sets a delta to 0, remove the entry from `mapDeltas`.

  Motivation / Background
  - `mapDeltas` entries are never removed from mapDeltas except when the tx is mined in a block or conflicted.
  - Mostly it is a feature to allow `prioritisetransaction` for a tx that isn't in the mempool {yet, anymore}. A user can may resbumit a tx and it retains its priority, or mark a tx as "definitely accept" before it is seen.
  - Since #8448, `mapDeltas` is persisted to mempool.dat and loaded on restart. This is also good, otherwise we lose prioritisation on restart.
  - Note the removal due to block/conflict is only done when `removeForBlock` is called, i.e. when the block is received. If you load a mempool.dat containing `mapDeltas` with transactions that were mined already (e.g. the file was saved prior to the last few blocks), you don't delete them.
  - Related: #4818 and #6464.
  - There is no way to query the node for not-in-mempool `mapDeltas`. If you add a priority and forget what the value was, the only way to get that information is to inspect mempool.dat.
  - Calling `prioritisetransaction` with an inverse value does not remove it from `mapDeltas`, it just sets the value to 0. It disappears on a restart (`LoadMempool` checks if delta is 0), but that might not happen for a while.

  Added together, if a user calls `prioritisetransaction` very regularly and not all those transactions get mined/conflicted, `mapDeltas` might keep lots of entries of delta=0 around. A user should clean up the not-in-mempool prioritisations, but that's currently difficult without keeping track of what those txids/amounts are.

ACKs for top commit:
  achow101:
    ACK 67b7fecacd
  theStack:
    Code-review ACK 67b7fecacd
  instagibbs:
    code review ACK 67b7fecacd
  ajtowns:
    ACK 67b7fecacd code review only, some nits

Tree-SHA512: 9df48b622ef27f33db1a2748f682bb3f16abe8172fcb7ac3c1a3e1654121ffb9b31aeaad5570c4162261f7e2ff5b5912ddc61a1b8beac0e9f346a86f5952260a
2023-06-07 03:29:05 -04:00

378 lines
13 KiB
C++

// Copyright (c) 2021-2022 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <base58.h>
#include <core_io.h>
#include <key.h>
#include <key_io.h>
#include <node/context.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <psbt.h>
#include <rpc/blockchain.h>
#include <rpc/client.h>
#include <rpc/request.h>
#include <rpc/server.h>
#include <rpc/util.h>
#include <span.h>
#include <streams.h>
#include <test/fuzz/FuzzedDataProvider.h>
#include <test/fuzz/fuzz.h>
#include <test/fuzz/util.h>
#include <test/util/setup_common.h>
#include <tinyformat.h>
#include <univalue.h>
#include <util/chaintype.h>
#include <util/strencodings.h>
#include <util/string.h>
#include <util/time.h>
#include <cstdint>
#include <iostream>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <vector>
namespace {
struct RPCFuzzTestingSetup : public TestingSetup {
RPCFuzzTestingSetup(const ChainType chain_type, const std::vector<const char*>& extra_args) : TestingSetup{chain_type, extra_args}
{
}
void CallRPC(const std::string& rpc_method, const std::vector<std::string>& arguments)
{
JSONRPCRequest request;
request.context = &m_node;
request.strMethod = rpc_method;
try {
request.params = RPCConvertValues(rpc_method, arguments);
} catch (const std::runtime_error&) {
return;
}
tableRPC.execute(request);
}
std::vector<std::string> GetRPCCommands() const
{
return tableRPC.listCommands();
}
};
RPCFuzzTestingSetup* rpc_testing_setup = nullptr;
std::string g_limit_to_rpc_command;
// RPC commands which are not appropriate for fuzzing: such as RPC commands
// reading or writing to a filename passed as an RPC parameter, RPC commands
// resulting in network activity, etc.
const std::vector<std::string> RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{
"addconnection", // avoid DNS lookups
"addnode", // avoid DNS lookups
"addpeeraddress", // avoid DNS lookups
"dumptxoutset", // avoid writing to disk
"dumpwallet", // avoid writing to disk
"echoipc", // avoid assertion failure (Assertion `"EnsureAnyNodeContext(request.context).init" && check' failed.)
"generatetoaddress", // avoid prohibitively slow execution (when `num_blocks` is large)
"generatetodescriptor", // avoid prohibitively slow execution (when `nblocks` is large)
"gettxoutproof", // avoid prohibitively slow execution
"importwallet", // avoid reading from disk
"loadwallet", // avoid reading from disk
"savemempool", // disabled as a precautionary measure: may take a file path argument in the future
"setban", // avoid DNS lookups
"stop", // avoid shutdown state
};
// RPC commands which are safe for fuzzing.
const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
"analyzepsbt",
"clearbanned",
"combinepsbt",
"combinerawtransaction",
"converttopsbt",
"createmultisig",
"createpsbt",
"createrawtransaction",
"decodepsbt",
"decoderawtransaction",
"decodescript",
"deriveaddresses",
"descriptorprocesspsbt",
"disconnectnode",
"echo",
"echojson",
"estimaterawfee",
"estimatesmartfee",
"finalizepsbt",
"generate",
"generateblock",
"getaddednodeinfo",
"getbestblockhash",
"getblock",
"getblockchaininfo",
"getblockcount",
"getblockfilter",
"getblockfrompeer", // when no peers are connected, no p2p message is sent
"getblockhash",
"getblockheader",
"getblockstats",
"getblocktemplate",
"getchaintips",
"getchaintxstats",
"getconnectioncount",
"getdeploymentinfo",
"getdescriptorinfo",
"getdifficulty",
"getindexinfo",
"getmemoryinfo",
"getmempoolancestors",
"getmempooldescendants",
"getmempoolentry",
"getmempoolinfo",
"getmininginfo",
"getnettotals",
"getnetworkhashps",
"getnetworkinfo",
"getnodeaddresses",
"getpeerinfo",
"getprioritisedtransactions",
"getrawmempool",
"getrawtransaction",
"getrpcinfo",
"gettxout",
"gettxoutsetinfo",
"gettxspendingprevout",
"help",
"invalidateblock",
"joinpsbts",
"listbanned",
"logging",
"mockscheduler",
"ping",
"preciousblock",
"prioritisetransaction",
"pruneblockchain",
"reconsiderblock",
"scanblocks",
"scantxoutset",
"sendrawtransaction",
"setmocktime",
"setnetworkactive",
"signmessagewithprivkey",
"signrawtransactionwithkey",
"submitblock",
"submitheader",
"submitpackage",
"syncwithvalidationinterfacequeue",
"testmempoolaccept",
"uptime",
"utxoupdatepsbt",
"validateaddress",
"verifychain",
"verifymessage",
"verifytxoutproof",
"waitforblock",
"waitforblockheight",
"waitfornewblock",
};
std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
{
const size_t max_string_length = 4096;
const size_t max_base58_bytes_length{64};
std::string r;
CallOneOf(
fuzzed_data_provider,
[&] {
// string argument
r = fuzzed_data_provider.ConsumeRandomLengthString(max_string_length);
},
[&] {
// base64 argument
r = EncodeBase64(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length));
},
[&] {
// hex argument
r = HexStr(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length));
},
[&] {
// bool argument
r = fuzzed_data_provider.ConsumeBool() ? "true" : "false";
},
[&] {
// range argument
r = "[" + ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()) + "," + ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()) + "]";
},
[&] {
// integral argument (int64_t)
r = ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>());
},
[&] {
// integral argument (uint64_t)
r = ToString(fuzzed_data_provider.ConsumeIntegral<uint64_t>());
},
[&] {
// floating point argument
r = strprintf("%f", fuzzed_data_provider.ConsumeFloatingPoint<double>());
},
[&] {
// tx destination argument
r = EncodeDestination(ConsumeTxDestination(fuzzed_data_provider));
},
[&] {
// uint160 argument
r = ConsumeUInt160(fuzzed_data_provider).ToString();
},
[&] {
// uint256 argument
r = ConsumeUInt256(fuzzed_data_provider).ToString();
},
[&] {
// base32 argument
r = EncodeBase32(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length));
},
[&] {
// base58 argument
r = EncodeBase58(MakeUCharSpan(fuzzed_data_provider.ConsumeRandomLengthString(max_base58_bytes_length)));
},
[&] {
// base58 argument with checksum
r = EncodeBase58Check(MakeUCharSpan(fuzzed_data_provider.ConsumeRandomLengthString(max_base58_bytes_length)));
},
[&] {
// hex encoded block
std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider);
if (!opt_block) {
return;
}
CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION};
data_stream << *opt_block;
r = HexStr(data_stream);
},
[&] {
// hex encoded block header
std::optional<CBlockHeader> opt_block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider);
if (!opt_block_header) {
return;
}
DataStream data_stream{};
data_stream << *opt_block_header;
r = HexStr(data_stream);
},
[&] {
// hex encoded tx
std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider);
if (!opt_tx) {
return;
}
CDataStream data_stream{SER_NETWORK, fuzzed_data_provider.ConsumeBool() ? PROTOCOL_VERSION : (PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)};
data_stream << *opt_tx;
r = HexStr(data_stream);
},
[&] {
// base64 encoded psbt
std::optional<PartiallySignedTransaction> opt_psbt = ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider);
if (!opt_psbt) {
return;
}
CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION};
data_stream << *opt_psbt;
r = EncodeBase64(data_stream);
},
[&] {
// base58 encoded key
const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(32);
CKey key;
key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool());
if (!key.IsValid()) {
return;
}
r = EncodeSecret(key);
},
[&] {
// hex encoded pubkey
const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(32);
CKey key;
key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool());
if (!key.IsValid()) {
return;
}
r = HexStr(key.GetPubKey());
});
return r;
}
std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
{
std::vector<std::string> scalar_arguments;
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) {
scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider));
}
return "[\"" + Join(scalar_arguments, "\",\"") + "\"]";
}
std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider)
{
return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider) : ConsumeArrayRPCArgument(fuzzed_data_provider);
}
RPCFuzzTestingSetup* InitializeRPCFuzzTestingSetup()
{
static const auto setup = MakeNoLogFileContext<RPCFuzzTestingSetup>();
SetRPCWarmupFinished();
return setup.get();
}
}; // namespace
void initialize_rpc()
{
rpc_testing_setup = InitializeRPCFuzzTestingSetup();
const std::vector<std::string> supported_rpc_commands = rpc_testing_setup->GetRPCCommands();
for (const std::string& rpc_command : supported_rpc_commands) {
const bool safe_for_fuzzing = std::find(RPC_COMMANDS_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_SAFE_FOR_FUZZING.end();
const bool not_safe_for_fuzzing = std::find(RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.end();
if (!(safe_for_fuzzing || not_safe_for_fuzzing)) {
std::cerr << "Error: RPC command \"" << rpc_command << "\" not found in RPC_COMMANDS_SAFE_FOR_FUZZING or RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n";
std::terminate();
}
if (safe_for_fuzzing && not_safe_for_fuzzing) {
std::cerr << "Error: RPC command \"" << rpc_command << "\" found in *both* RPC_COMMANDS_SAFE_FOR_FUZZING and RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n";
std::terminate();
}
}
const char* limit_to_rpc_command_env = std::getenv("LIMIT_TO_RPC_COMMAND");
if (limit_to_rpc_command_env != nullptr) {
g_limit_to_rpc_command = std::string{limit_to_rpc_command_env};
}
}
FUZZ_TARGET_INIT(rpc, initialize_rpc)
{
FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()};
SetMockTime(ConsumeTime(fuzzed_data_provider));
const std::string rpc_command = fuzzed_data_provider.ConsumeRandomLengthString(64);
if (!g_limit_to_rpc_command.empty() && rpc_command != g_limit_to_rpc_command) {
return;
}
const bool safe_for_fuzzing = std::find(RPC_COMMANDS_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_SAFE_FOR_FUZZING.end();
if (!safe_for_fuzzing) {
return;
}
std::vector<std::string> arguments;
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) {
arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider));
}
try {
rpc_testing_setup->CallRPC(rpc_command, arguments);
} catch (const UniValue& json_rpc_error) {
const std::string error_msg{json_rpc_error.find_value("message").get_str()};
// Once c++20 is allowed, starts_with can be used.
// if (error_msg.starts_with("Internal bug detected")) {
if (0 == error_msg.rfind("Internal bug detected", 0)) {
// Only allow the intentional internal bug
assert(error_msg.find("trigger_internal_bug") != std::string::npos);
}
}
}