From fa579d663d716c967ccd45d67b46e779e2fa0b48 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Tue, 18 Feb 2025 21:31:32 +0100 Subject: [PATCH] contrib: Add deterministic-unittest-coverage This replaces the bash script with a tool based on clang/llvm tools. --- contrib/devtools/README.md | 21 +++ .../Cargo.lock | 7 + .../Cargo.toml | 6 + .../src/main.rs | 124 ++++++++++++++ .../devtools/test_deterministic_coverage.sh | 151 ------------------ 5 files changed, 158 insertions(+), 151 deletions(-) create mode 100644 contrib/devtools/deterministic-unittest-coverage/Cargo.lock create mode 100644 contrib/devtools/deterministic-unittest-coverage/Cargo.toml create mode 100644 contrib/devtools/deterministic-unittest-coverage/src/main.rs delete mode 100755 contrib/devtools/test_deterministic_coverage.sh diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 6ef84e7af58..82539567801 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -25,6 +25,27 @@ before running the tool: RUST_BACKTRACE=1 cargo run --manifest-path ./contrib/devtools/deterministic-fuzz-coverage/Cargo.toml -- $PWD/build_dir $PWD/qa-assets/fuzz_corpora fuzz_target_name ``` +deterministic-unittest-coverage +=========================== + +A tool to check for non-determinism in unit-test coverage. To get the help, run: + +``` +RUST_BACKTRACE=1 cargo run --manifest-path ./contrib/devtools/deterministic-unittest-coverage/Cargo.toml -- --help +``` + +To execute the tool, compilation has to be done with the build options: + +``` +-DCMAKE_C_COMPILER='clang' -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_CXX_FLAGS='-fprofile-instr-generate -fcoverage-mapping' +``` + +Both llvm-profdata and llvm-cov must be installed. + +``` +RUST_BACKTRACE=1 cargo run --manifest-path ./contrib/devtools/deterministic-unittest-coverage/Cargo.toml -- $PWD/build_dir +``` + clang-format-diff.py =================== diff --git a/contrib/devtools/deterministic-unittest-coverage/Cargo.lock b/contrib/devtools/deterministic-unittest-coverage/Cargo.lock new file mode 100644 index 00000000000..39013304451 --- /dev/null +++ b/contrib/devtools/deterministic-unittest-coverage/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "deterministic-unittest-coverage" +version = "0.1.0" diff --git a/contrib/devtools/deterministic-unittest-coverage/Cargo.toml b/contrib/devtools/deterministic-unittest-coverage/Cargo.toml new file mode 100644 index 00000000000..d5810213055 --- /dev/null +++ b/contrib/devtools/deterministic-unittest-coverage/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "deterministic-unittest-coverage" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/contrib/devtools/deterministic-unittest-coverage/src/main.rs b/contrib/devtools/deterministic-unittest-coverage/src/main.rs new file mode 100644 index 00000000000..b941f37e22a --- /dev/null +++ b/contrib/devtools/deterministic-unittest-coverage/src/main.rs @@ -0,0 +1,124 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +use std::env; +use std::fs::File; +use std::path::Path; +use std::process::{exit, Command}; +use std::str; + +const LLVM_PROFDATA: &str = "llvm-profdata"; +const LLVM_COV: &str = "llvm-cov"; +const GIT: &str = "git"; + +fn exit_help(err: &str) -> ! { + eprintln!("Error: {}", err); + eprintln!(); + eprintln!("Usage: program ./build_dir boost_unittest_filter"); + eprintln!(); + eprintln!("Refer to the devtools/README.md for more details."); + exit(1) +} + +fn sanity_check(test_exe: &Path) { + for tool in [LLVM_PROFDATA, LLVM_COV, GIT] { + let output = Command::new(tool).arg("--help").output(); + match output { + Ok(output) if output.status.success() => {} + _ => { + exit_help(&format!("The tool {} is not installed", tool)); + } + } + } + if !test_exe.exists() { + exit_help(&format!( + "Test executable ({}) not found", + test_exe.display() + )); + } +} + +fn main() { + // Parse args + let args = env::args().collect::>(); + let build_dir = args + .get(1) + .unwrap_or_else(|| exit_help("Must set build dir")); + if build_dir == "--help" { + exit_help("--help requested") + } + let filter = args + .get(2) + // Require filter for now. In the future it could be optional and the tool could provide a + // default filter. + .unwrap_or_else(|| exit_help("Must set boost test filter")); + if args.get(3).is_some() { + exit_help("Too many args") + } + + let build_dir = Path::new(build_dir); + let test_exe = build_dir.join("src/test/test_bitcoin"); + + sanity_check(&test_exe); + + deterministic_coverage(build_dir, &test_exe, filter); +} + +fn deterministic_coverage(build_dir: &Path, test_exe: &Path, filter: &str) { + let profraw_file = build_dir.join("test_det_cov.profraw"); + let profdata_file = build_dir.join("test_det_cov.profdata"); + let run_single = |run_id: u8| { + let cov_txt_path = build_dir.join(format!("test_det_cov.show.{run_id}.txt")); + assert!(Command::new(test_exe) + .env("LLVM_PROFILE_FILE", &profraw_file) + .env("BOOST_TEST_RUN_FILTERS", filter) + .env("RANDOM_CTX_SEED", "21") + .status() + .expect("test failed") + .success()); + assert!(Command::new(LLVM_PROFDATA) + .arg("merge") + .arg("--sparse") + .arg(&profraw_file) + .arg("-o") + .arg(&profdata_file) + .status() + .expect("merge failed") + .success()); + let cov_file = File::create(&cov_txt_path).expect("Failed to create coverage txt file"); + assert!(Command::new(LLVM_COV) + .args([ + "show", + "--show-line-counts-or-regions", + "--show-branches=count", + "--show-expansions", + &format!("--instr-profile={}", profdata_file.display()), + ]) + .arg(test_exe) + .stdout(cov_file) + .status() + .expect("llvm-cov failed") + .success()); + cov_txt_path + }; + let check_diff = |a: &Path, b: &Path| { + let same = Command::new(GIT) + .args(["--no-pager", "diff", "--no-index"]) + .arg(a) + .arg(b) + .status() + .expect("Failed to execute git command") + .success(); + if !same { + eprintln!(); + eprintln!("The coverage was not deterministic between runs."); + eprintln!("Exiting."); + exit(1); + } + }; + let r0 = run_single(0); + let r1 = run_single(1); + check_diff(&r0, &r1); + println!("The coverage was deterministic across two runs."); +} diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh deleted file mode 100755 index 885396bb259..00000000000 --- a/contrib/devtools/test_deterministic_coverage.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Test for deterministic coverage across unit test runs. - -export LC_ALL=C - -# Use GCOV_EXECUTABLE="gcov" if compiling with gcc. -# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang. -GCOV_EXECUTABLE="gcov" - -# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. -NON_DETERMINISTIC_TESTS=( - "blockfilter_index_tests/blockfilter_index_initial_sync" # src/checkqueue.h: In CCheckQueue::Loop(): while (queue.empty()) { ... } - "coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2)) - "fs_tests/fsbridge_fstream" # deterministic test failure? - "miner_tests/CreateNewBlock_validity" # validation.cpp: if (signals.CallbacksPending() > 10) - "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() - "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() - "txvalidationcache_tests/checkinputs_test" # validation.cpp: if (signals.CallbacksPending() > 10) - "txvalidationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (signals.CallbacksPending() > 10) - "txindex_tests/txindex_initial_sync" # validation.cpp: if (signals.CallbacksPending() > 10) - "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (signals.CallbacksPending() > 10) - "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (signals.CallbacksPending() > 10) - "wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (signals.CallbacksPending() > 10) - "wallet_tests/dummy_input_size_test" # validation.cpp: if (signals.CallbacksPending() > 10) - "wallet_tests/importmulti_rescan" # validation.cpp: if (signals.CallbacksPending() > 10) - "wallet_tests/importwallet_rescan" # validation.cpp: if (signals.CallbacksPending() > 10) - "wallet_tests/ListCoins" # validation.cpp: if (signals.CallbacksPending() > 10) - "wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (signals.CallbacksPending() > 10) - "wallet_tests/wallet_disableprivkeys" # validation.cpp: if (signals.CallbacksPending() > 10) -) - -TEST_BITCOIN_BINARY="src/test/test_bitcoin" - -print_usage() { - echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]" -} - -N_TEST_RUNS=2 -BOOST_TEST_RUN_FILTERS="" -if [[ $# != 0 ]]; then - if [[ $1 == "--help" ]]; then - print_usage - exit - fi - PARSED_ARGUMENTS=0 - if [[ $1 =~ [a-z] ]]; then - BOOST_TEST_RUN_FILTERS=$1 - PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) - shift - fi - if [[ $1 =~ ^[0-9]+$ ]]; then - N_TEST_RUNS=$1 - PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) - shift - fi - if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then - print_usage - exit - fi -fi -if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then - BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')" -else - echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}" - echo -fi - -if ! command -v gcov > /dev/null; then - echo "Error: gcov not installed. Exiting." - exit 1 -fi - -if ! command -v gcovr > /dev/null; then - echo "Error: gcovr not installed. Exiting." - exit 1 -fi - -if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then - echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and compile." - exit 1 -fi - -get_file_suffix_count() { - find src/ -type f -name "*.$1" | wc -l -} - -if [[ $(get_file_suffix_count gcno) == 0 ]]; then - echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and re-compile." - exit 1 -fi - -get_covr_filename() { - echo "gcovr.run-$1.txt" -} - -TEST_RUN_ID=0 -while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do - TEST_RUN_ID=$((TEST_RUN_ID + 1)) - echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}" - find src/ -type f -name "*.gcda" -exec rm {} \; - if [[ $(get_file_suffix_count gcda) != 0 ]]; then - echo "Error: Stale *.gcda files found. Exiting." - exit 1 - fi - TEST_OUTPUT_TEMPFILE=$(mktemp) - if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then - cat "${TEST_OUTPUT_TEMPFILE}" - rm "${TEST_OUTPUT_TEMPFILE}" - exit 1 - fi - rm "${TEST_OUTPUT_TEMPFILE}" - if [[ $(get_file_suffix_count gcda) == 0 ]]; then - echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and re-compile." - exit 1 - fi - GCOVR_TEMPFILE=$(mktemp) - if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then - echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting." - exit 1 - fi - GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID}) - mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}" - if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then - echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)." - exit 1 - fi - if [[ ${TEST_RUN_ID} != 1 ]]; then - COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}") - if [[ ${COVERAGE_DIFF} != "" ]]; then - echo - echo "The line coverage is non-deterministic between runs. Exiting." - echo - echo "The test suite must be deterministic in the sense that the set of lines executed at least" - echo "once must be identical between runs. This is a necessary condition for meaningful" - echo "coverage measuring." - echo - echo "${COVERAGE_DIFF}" - exit 1 - fi - rm "${GCOVR_FILENAME}" - fi -done - -echo -echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs." -exit