diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63d18a93494..2d4985c9974 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -390,9 +390,6 @@ jobs: (Get-Content "test/config.ini") -replace '(?<=^SRCDIR=).*', '${{ github.workspace }}' -replace '(?<=^BUILDDIR=).*', '${{ github.workspace }}' -replace '(?<=^RPCAUTH=).*', '${{ github.workspace }}/share/rpcauth/rpcauth.py' | Set-Content "test/config.ini" Get-Content "test/config.ini" - - name: Run util tests - run: py -3 test/util/test_runner.py - - name: Run rpcauth test run: py -3 test/util/rpcauth-test.py diff --git a/CMakeLists.txt b/CMakeLists.txt index f7241020815..18d23142a10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -594,7 +594,7 @@ if(Python3_EXECUTABLE) set(PYTHON_COMMAND ${Python3_EXECUTABLE}) else() list(APPEND configure_warnings - "Minimum required Python not found. Utils and rpcauth tests are disabled." + "Minimum required Python not found. Rpcauth tests are disabled." ) endif() diff --git a/cmake/tests.cmake b/cmake/tests.cmake index 27913298001..46104593c85 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -2,12 +2,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. -if(TARGET bitcoin-util AND TARGET bitcoin-tx AND PYTHON_COMMAND) - add_test(NAME util_test_runner - COMMAND ${CMAKE_COMMAND} -E env BITCOINUTIL=$ BITCOINTX=$ ${PYTHON_COMMAND} ${PROJECT_BINARY_DIR}/test/util/test_runner.py - ) -endif() - if(PYTHON_COMMAND) add_test(NAME util_rpcauth_test COMMAND ${PYTHON_COMMAND} ${PROJECT_BINARY_DIR}/test/util/rpcauth-test.py diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ab67b1f1d86..b7fde825604 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -37,7 +37,7 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/fuzz) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/util) file(GLOB_RECURSE functional_tests RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} functional/*) -foreach(script ${functional_tests} fuzz/test_runner.py util/rpcauth-test.py util/test_runner.py) +foreach(script ${functional_tests} fuzz/test_runner.py util/rpcauth-test.py) if(CMAKE_HOST_WIN32) set(symlink) else() diff --git a/test/README.md b/test/README.md index 74adc644b83..843777da06e 100644 --- a/test/README.md +++ b/test/README.md @@ -10,10 +10,9 @@ This directory contains the following sets of tests: - [functional](/test/functional) which test the functionality of bitcoind and bitcoin-qt by interacting with them through the RPC and P2P interfaces. -- [util](/test/util) which tests the utilities (bitcoin-util, bitcoin-tx, ...). - [lint](/test/lint/) which perform various static analysis checks. -The util tests are run as part of `ctest` invocation. The fuzz tests, functional +The fuzz tests, functional tests and lint scripts can be run as explained in the sections below. # Running tests locally @@ -321,11 +320,6 @@ perf report -i /path/to/datadir/send-big-msgs.perf.data.xxxx --stdio | c++filt | For ways to generate more granular profiles, see the README in [test/functional](/test/functional). -### Util tests - -Util tests can be run locally by running `build/test/util/test_runner.py`. -Use the `-v` option for verbose output. - ### Lint tests See the README in [test/lint](/test/lint). diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 340a418e104..e7105cb7365 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -170,6 +170,7 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py --mineblock', 'tool_bitcoin_chainstate.py', 'tool_wallet.py', + 'tool_utils.py', 'tool_signet_miner.py', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', diff --git a/test/functional/tool_utils.py b/test/functional/tool_utils.py new file mode 100755 index 00000000000..3e36d05a50b --- /dev/null +++ b/test/functional/tool_utils.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Copyright 2014 BitPay Inc. +# Copyright 2016-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. +"""Exercise the utils via json-defined tests.""" + +from test_framework.test_framework import BitcoinTestFramework + +import difflib +import json +import logging +import os +import subprocess +from pathlib import Path + + +class ToolUtils(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 0 # No node/datadir needed + + def setup_network(self): + pass + + def skip_test_if_missing_module(self): + self.skip_if_no_bitcoin_tx() + self.skip_if_no_bitcoin_util() + + def run_test(self): + self.testcase_dir = Path(self.config["environment"]["SRCDIR"]) / "test" / "util" / "data" + self.bins = self.get_binaries() + with open(self.testcase_dir / "bitcoin-util-test.json", encoding="utf8") as f: + input_data = json.loads(f.read()) + + for i, test_obj in enumerate(input_data): + self.log.debug(f"Running [{i}]: " + test_obj["description"]) + self.test_one(test_obj) + + def test_one(self, testObj): + """Runs a single test, comparing output and RC to expected output and RC. + + Raises an error if input can't be read, executable fails, or output/RC + are not as expected. Error is caught by bctester() and reported. + """ + # Get the exec names and arguments + if testObj["exec"] == "./bitcoin-util": + execrun = self.bins.util_argv() + testObj["args"] + elif testObj["exec"] == "./bitcoin-tx": + execrun = self.bins.tx_argv() + testObj["args"] + + # Read the input data (if there is any) + inputData = None + if "input" in testObj: + with open(self.testcase_dir / testObj["input"], encoding="utf8") as f: + inputData = f.read() + + # Read the expected output data (if there is any) + outputFn = None + outputData = None + outputType = None + if "output_cmp" in testObj: + outputFn = testObj['output_cmp'] + outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) + try: + with open(self.testcase_dir / outputFn, encoding="utf8") as f: + outputData = f.read() + except Exception: + logging.error("Output file " + outputFn + " cannot be opened") + raise + if not outputData: + logging.error("Output data missing for " + outputFn) + raise Exception + if not outputType: + logging.error("Output file %s does not have a file extension" % outputFn) + raise Exception + + # Run the test + try: + res = subprocess.run(execrun, capture_output=True, text=True, input=inputData) + except OSError: + logging.error("OSError, Failed to execute " + str(execrun)) + raise + + if outputData: + data_mismatch, formatting_mismatch = False, False + # Parse command output and expected output + try: + a_parsed = parse_output(res.stdout, outputType) + except Exception as e: + logging.error(f"Error parsing command output as {outputType}: '{str(e)}'; res: {str(res)}") + raise + try: + b_parsed = parse_output(outputData, outputType) + except Exception as e: + logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e)) + raise + # Compare data + if a_parsed != b_parsed: + logging.error(f"Output data mismatch for {outputFn} (format {outputType}); res: {str(res)}") + data_mismatch = True + # Compare formatting + if res.stdout != outputData: + error_message = f"Output formatting mismatch for {outputFn}:\nres: {str(res)}\n" + error_message += "".join(difflib.context_diff(outputData.splitlines(True), + res.stdout.splitlines(True), + fromfile=outputFn, + tofile="returned")) + logging.error(error_message) + formatting_mismatch = True + + assert not data_mismatch and not formatting_mismatch + + # Compare the return code to the expected return code + wantRC = 0 + if "return_code" in testObj: + wantRC = testObj['return_code'] + if res.returncode != wantRC: + logging.error(f"Return code mismatch for {outputFn}; res: {str(res)}") + raise Exception + + if "error_txt" in testObj: + want_error = testObj["error_txt"] + # A partial match instead of an exact match makes writing tests easier + # and should be sufficient. + if want_error not in res.stderr: + logging.error(f"Error mismatch:\nExpected: {want_error}\nReceived: {res.stderr.rstrip()}\nres: {str(res)}") + raise Exception + else: + if res.stderr: + logging.error(f"Unexpected error received: {res.stderr.rstrip()}\nres: {str(res)}") + raise Exception + + +def parse_output(a, fmt): + """Parse the output according to specified format. + + Raise an error if the output can't be parsed.""" + if fmt == 'json': # json: compare parsed data + return json.loads(a) + elif fmt == 'hex': # hex: parse and compare binary data + return bytes.fromhex(a.strip()) + else: + raise NotImplementedError("Don't know how to compare %s" % fmt) + + +if __name__ == "__main__": + ToolUtils(__file__).main() diff --git a/test/util/test_runner.py b/test/util/test_runner.py deleted file mode 100755 index 11400b32ba3..00000000000 --- a/test/util/test_runner.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2014 BitPay Inc. -# Copyright 2016-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test framework for bitcoin utils. - -Runs automatically during `ctest --test-dir build/`. - -Can also be run manually.""" - -import argparse -import configparser -import difflib -import json -import logging -import os -import pprint -import subprocess -import sys - -def main(): - config = configparser.ConfigParser() - config.optionxform = str - with open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8") as f: - config.read_file(f) - env_conf = dict(config.items('environment')) - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('-v', '--verbose', action='store_true') - args = parser.parse_args() - verbose = args.verbose - - if verbose: - level = logging.DEBUG - else: - level = logging.ERROR - formatter = '%(asctime)s - %(levelname)s - %(message)s' - # Add the format/level to the logger - logging.basicConfig(format=formatter, level=level) - - bctester(os.path.join(env_conf["SRCDIR"], "test", "util", "data"), "bitcoin-util-test.json", env_conf) - -def bctester(testDir, input_basename, buildenv): - """ Loads and parses the input file, runs all tests and reports results""" - input_filename = os.path.join(testDir, input_basename) - with open(input_filename, encoding="utf8") as f: - raw_data = f.read() - input_data = json.loads(raw_data) - - failed_testcases = [] - - for testObj in input_data: - try: - bctest(testDir, testObj, buildenv) - logging.info("PASSED: " + testObj["description"]) - except Exception: - logging.info("FAILED: " + testObj["description"]) - failed_testcases.append(testObj["description"]) - - if failed_testcases: - error_message = "FAILED_TESTCASES:\n" - error_message += pprint.pformat(failed_testcases, width=400) - logging.error(error_message) - sys.exit(1) - else: - sys.exit(0) - -def bctest(testDir, testObj, buildenv): - """Runs a single test, comparing output and RC to expected output and RC. - - Raises an error if input can't be read, executable fails, or output/RC - are not as expected. Error is caught by bctester() and reported. - """ - # Get the exec names and arguments - execprog = os.path.join(buildenv["BUILDDIR"], "bin", testObj["exec"] + buildenv["EXEEXT"]) - if testObj["exec"] == "./bitcoin-util": - execprog = os.getenv("BITCOINUTIL", default=execprog) - elif testObj["exec"] == "./bitcoin-tx": - execprog = os.getenv("BITCOINTX", default=execprog) - - execargs = testObj['args'] - execrun = [execprog] + execargs - - # Read the input data (if there is any) - inputData = None - if "input" in testObj: - filename = os.path.join(testDir, testObj["input"]) - with open(filename, encoding="utf8") as f: - inputData = f.read() - - # Read the expected output data (if there is any) - outputFn = None - outputData = None - outputType = None - if "output_cmp" in testObj: - outputFn = testObj['output_cmp'] - outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) - try: - with open(os.path.join(testDir, outputFn), encoding="utf8") as f: - outputData = f.read() - except Exception: - logging.error("Output file " + outputFn + " cannot be opened") - raise - if not outputData: - logging.error("Output data missing for " + outputFn) - raise Exception - if not outputType: - logging.error("Output file %s does not have a file extension" % outputFn) - raise Exception - - # Run the test - try: - res = subprocess.run(execrun, capture_output=True, text=True, input=inputData) - except OSError: - logging.error("OSError, Failed to execute " + execprog) - raise - - if outputData: - data_mismatch, formatting_mismatch = False, False - # Parse command output and expected output - try: - a_parsed = parse_output(res.stdout, outputType) - except Exception as e: - logging.error(f"Error parsing command output as {outputType}: '{str(e)}'; res: {str(res)}") - raise - try: - b_parsed = parse_output(outputData, outputType) - except Exception as e: - logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e)) - raise - # Compare data - if a_parsed != b_parsed: - logging.error(f"Output data mismatch for {outputFn} (format {outputType}); res: {str(res)}") - data_mismatch = True - # Compare formatting - if res.stdout != outputData: - error_message = f"Output formatting mismatch for {outputFn}:\nres: {str(res)}\n" - error_message += "".join(difflib.context_diff(outputData.splitlines(True), - res.stdout.splitlines(True), - fromfile=outputFn, - tofile="returned")) - logging.error(error_message) - formatting_mismatch = True - - assert not data_mismatch and not formatting_mismatch - - # Compare the return code to the expected return code - wantRC = 0 - if "return_code" in testObj: - wantRC = testObj['return_code'] - if res.returncode != wantRC: - logging.error(f"Return code mismatch for {outputFn}; res: {str(res)}") - raise Exception - - if "error_txt" in testObj: - want_error = testObj["error_txt"] - # A partial match instead of an exact match makes writing tests easier - # and should be sufficient. - if want_error not in res.stderr: - logging.error(f"Error mismatch:\nExpected: {want_error}\nReceived: {res.stderr.rstrip()}\nres: {str(res)}") - raise Exception - else: - if res.stderr: - logging.error(f"Unexpected error received: {res.stderr.rstrip()}\nres: {str(res)}") - raise Exception - - -def parse_output(a, fmt): - """Parse the output according to specified format. - - Raise an error if the output can't be parsed.""" - if fmt == 'json': # json: compare parsed data - return json.loads(a) - elif fmt == 'hex': # hex: parse and compare binary data - return bytes.fromhex(a.strip()) - else: - raise NotImplementedError("Don't know how to compare %s" % fmt) - -if __name__ == '__main__': - main()