Merge bitcoin/bitcoin#34927: test: Check that RPCs do not time out, even under load

fa7bc26d12 test: Check that RPCs do not time out, even under load (MarcoFalke)
fa2bd96cc0 test: Map cli CalledProcessError on server error to JSONRPCException (MarcoFalke)

Pull request description:

  It turns out there is no test currently to check that the RPC server does not time out under load. With "load" I mean a flood of trivial payloads. That is, the only work needed is JSON encoding and decoding of (let's say) a block of data of 2 MB or so. This may take a few milliseconds, but should never take more than a few seconds.

  So add a test for this.

ACKs for top commit:
  enirox001:
    ACK fa7bc26d12
  sedited:
    ACK fa7bc26d12

Tree-SHA512: c60646981b7449c757e9fad499e1cd71030376ffb2ae687c8136c6f70accd0a2d76a4cbfc7dd1bc6626ea3a5310a33616a36cd682cc5c4371ec86aa2c8641aeb

test/functional/interface_rpc.py

@@ -8,10 +8,9 @@ import json
 import os
 from dataclasses import dataclass
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal, assert_greater_than_or_equal
+from test_framework.util import JSONRPCException, assert_equal, assert_greater_than_or_equal
 from threading import Thread
 from typing import Optional
-import subprocess
 
 
 RPC_INVALID_PARAMETER      = -8
@@ -83,8 +82,8 @@ def test_work_queue_getblock(node, got_exceeded_error):
     while not got_exceeded_error:
         try:
             node.cli("waitfornewblock", "500").send_cli()
-        except subprocess.CalledProcessError as e:
-            assert_equal(e.output, 'error: Server response: Work queue depth exceeded\n')
+        except JSONRPCException as e:
+            assert_equal(e.error["message"], "non-JSON HTTP response with '503 Service Unavailable' from server: Work queue depth exceeded")
             got_exceeded_error.append(True)
 
 

test/functional/rpc_echo_payload.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+"""Ensure RPCs with a (possibly large) payload will either be rejected or handled, but will never time out."""
+
+from concurrent.futures import ThreadPoolExecutor
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal, JSONRPCException
+import random
+
+
+class RpcEchoPayloadTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+        self.setup_clean_chain = True
+        self.extra_args = [["-rpcworkqueue=2", "-rpcthreads=2"]]
+        # enough to possibly fill the running threads as well as the queue:
+        self.num_threads = 6
+
+    def run_test(self):
+        node = self.nodes[0]
+        # Use json-serializable, but non-hex data
+        data = "z" + random.randbytes(1_999_000).hex()
+
+        def check_results(rpc):
+            self.log.info("Starting thread ...")
+            for i in range(200):
+                payload = data[: random.randrange(0, len(data))]
+                try:
+                    if random.getrandbits(1):
+                        assert_equal(payload, rpc.echo(payload)[0])
+                    else:
+                        rpc.sendrawtransaction(payload)
+                except JSONRPCException as e:
+                    msg = e.error["message"]
+                    if msg not in [
+                        "TX decode failed. Make sure the tx has at least one input.",
+                        "non-JSON HTTP response with '503 Service Unavailable' from server: Work queue depth exceeded",
+                    ]:
+                        raise AssertionError(f"Unexpected msg: {msg}")
+
+        rpcs = [node.create_new_rpc_connection() for _ in range(self.num_threads)]
+        self.log.info("Starting threadpool ...")
+        with ThreadPoolExecutor(max_workers=len(rpcs)) as threads:
+            list(threads.map(check_results, rpcs))
+
+
+if __name__ == "__main__":
+    RpcEchoPayloadTest(__file__).main()

test/functional/test_framework/test_node.py

@@ -967,6 +967,13 @@ class TestNodeCLI():
             if match:
                 code, message = match.groups()
                 raise JSONRPCException(dict(code=int(code), message=message))
+            match = re.match(r'error: Server response: (.*)\n?$', cli_stderr)
+            if match:
+                message = match.group(1)
+                raise JSONRPCException(dict(
+                    code=-342,
+                    message=f"non-JSON HTTP response with '503 Service Unavailable' from server: {message}",
+                ), http_status=503)
             # Ignore cli_stdout, raise with cli_stderr
             raise subprocess.CalledProcessError(returncode, p_args, output=cli_stderr)
         try:

test/functional/test_runner.py

@@ -153,6 +153,7 @@ BASE_SCRIPTS = [
     # vv Tests less than 30s vv
     'wallet_deprecated_rbf.py',
     'p2p_invalid_messages.py',
+    'rpc_echo_payload.py',
     'rpc_createmultisig.py',
     'p2p_timeouts.py --v1transport',
     'p2p_timeouts.py --v2transport',