mirror of
https://github.com/bitcoin/bitcoin.git
synced 2025-03-17 21:32:00 +01:00
contrib: Add deterministic-unittest-coverage
This replaces the bash script with a tool based on clang/llvm tools.
This commit is contained in:
MarcoFalke
parent
fa3940b1cb
commit
fa579d663d
@ -25,6 +25,27 @@ before running the tool:
|
||||
RUST_BACKTRACE=1 cargo run --manifest-path ./contrib/devtools/deterministic-fuzz-coverage/Cargo.toml -- $PWD/build_dir $PWD/qa-assets/fuzz_corpora fuzz_target_name
|
||||
```
|
||||
|
||||
deterministic-unittest-coverage
|
||||
===========================
|
||||
|
||||
A tool to check for non-determinism in unit-test coverage. To get the help, run:
|
||||
|
||||
```
|
||||
RUST_BACKTRACE=1 cargo run --manifest-path ./contrib/devtools/deterministic-unittest-coverage/Cargo.toml -- --help
|
||||
```
|
||||
|
||||
To execute the tool, compilation has to be done with the build options:
|
||||
|
||||
```
|
||||
-DCMAKE_C_COMPILER='clang' -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_CXX_FLAGS='-fprofile-instr-generate -fcoverage-mapping'
|
||||
```
|
||||
|
||||
Both llvm-profdata and llvm-cov must be installed.
|
||||
|
||||
```
|
||||
RUST_BACKTRACE=1 cargo run --manifest-path ./contrib/devtools/deterministic-unittest-coverage/Cargo.toml -- $PWD/build_dir <boost unittest filter>
|
||||
```
|
||||
|
||||
clang-format-diff.py
|
||||
===================
|
||||
|
||||
|
||||
7
contrib/devtools/deterministic-unittest-coverage/Cargo.lock
generated
Normal file
7
contrib/devtools/deterministic-unittest-coverage/Cargo.lock
generated
Normal file
@ -0,0 +1,7 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "deterministic-unittest-coverage"
|
||||
version = "0.1.0"
|
||||
@ -0,0 +1,6 @@
|
||||
[package]
|
||||
name = "deterministic-unittest-coverage"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
124
contrib/devtools/deterministic-unittest-coverage/src/main.rs
Normal file
124
contrib/devtools/deterministic-unittest-coverage/src/main.rs
Normal file
@ -0,0 +1,124 @@
|
||||
// Copyright (c) The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or https://opensource.org/license/mit/.
|
||||
|
||||
use std::env;
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use std::process::{exit, Command};
|
||||
use std::str;
|
||||
|
||||
const LLVM_PROFDATA: &str = "llvm-profdata";
|
||||
const LLVM_COV: &str = "llvm-cov";
|
||||
const GIT: &str = "git";
|
||||
|
||||
fn exit_help(err: &str) -> ! {
|
||||
eprintln!("Error: {}", err);
|
||||
eprintln!();
|
||||
eprintln!("Usage: program ./build_dir boost_unittest_filter");
|
||||
eprintln!();
|
||||
eprintln!("Refer to the devtools/README.md for more details.");
|
||||
exit(1)
|
||||
}
|
||||
|
||||
fn sanity_check(test_exe: &Path) {
|
||||
for tool in [LLVM_PROFDATA, LLVM_COV, GIT] {
|
||||
let output = Command::new(tool).arg("--help").output();
|
||||
match output {
|
||||
Ok(output) if output.status.success() => {}
|
||||
_ => {
|
||||
exit_help(&format!("The tool {} is not installed", tool));
|
||||
}
|
||||
}
|
||||
}
|
||||
if !test_exe.exists() {
|
||||
exit_help(&format!(
|
||||
"Test executable ({}) not found",
|
||||
test_exe.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
// Parse args
|
||||
let args = env::args().collect::<Vec<_>>();
|
||||
let build_dir = args
|
||||
.get(1)
|
||||
.unwrap_or_else(|| exit_help("Must set build dir"));
|
||||
if build_dir == "--help" {
|
||||
exit_help("--help requested")
|
||||
}
|
||||
let filter = args
|
||||
.get(2)
|
||||
// Require filter for now. In the future it could be optional and the tool could provide a
|
||||
// default filter.
|
||||
.unwrap_or_else(|| exit_help("Must set boost test filter"));
|
||||
if args.get(3).is_some() {
|
||||
exit_help("Too many args")
|
||||
}
|
||||
|
||||
let build_dir = Path::new(build_dir);
|
||||
let test_exe = build_dir.join("src/test/test_bitcoin");
|
||||
|
||||
sanity_check(&test_exe);
|
||||
|
||||
deterministic_coverage(build_dir, &test_exe, filter);
|
||||
}
|
||||
|
||||
fn deterministic_coverage(build_dir: &Path, test_exe: &Path, filter: &str) {
|
||||
let profraw_file = build_dir.join("test_det_cov.profraw");
|
||||
let profdata_file = build_dir.join("test_det_cov.profdata");
|
||||
let run_single = |run_id: u8| {
|
||||
let cov_txt_path = build_dir.join(format!("test_det_cov.show.{run_id}.txt"));
|
||||
assert!(Command::new(test_exe)
|
||||
.env("LLVM_PROFILE_FILE", &profraw_file)
|
||||
.env("BOOST_TEST_RUN_FILTERS", filter)
|
||||
.env("RANDOM_CTX_SEED", "21")
|
||||
.status()
|
||||
.expect("test failed")
|
||||
.success());
|
||||
assert!(Command::new(LLVM_PROFDATA)
|
||||
.arg("merge")
|
||||
.arg("--sparse")
|
||||
.arg(&profraw_file)
|
||||
.arg("-o")
|
||||
.arg(&profdata_file)
|
||||
.status()
|
||||
.expect("merge failed")
|
||||
.success());
|
||||
let cov_file = File::create(&cov_txt_path).expect("Failed to create coverage txt file");
|
||||
assert!(Command::new(LLVM_COV)
|
||||
.args([
|
||||
"show",
|
||||
"--show-line-counts-or-regions",
|
||||
"--show-branches=count",
|
||||
"--show-expansions",
|
||||
&format!("--instr-profile={}", profdata_file.display()),
|
||||
])
|
||||
.arg(test_exe)
|
||||
.stdout(cov_file)
|
||||
.status()
|
||||
.expect("llvm-cov failed")
|
||||
.success());
|
||||
cov_txt_path
|
||||
};
|
||||
let check_diff = |a: &Path, b: &Path| {
|
||||
let same = Command::new(GIT)
|
||||
.args(["--no-pager", "diff", "--no-index"])
|
||||
.arg(a)
|
||||
.arg(b)
|
||||
.status()
|
||||
.expect("Failed to execute git command")
|
||||
.success();
|
||||
if !same {
|
||||
eprintln!();
|
||||
eprintln!("The coverage was not deterministic between runs.");
|
||||
eprintln!("Exiting.");
|
||||
exit(1);
|
||||
}
|
||||
};
|
||||
let r0 = run_single(0);
|
||||
let r1 = run_single(1);
|
||||
check_diff(&r0, &r1);
|
||||
println!("The coverage was deterministic across two runs.");
|
||||
}
|
||||
@ -1,151 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Copyright (c) 2019-2020 The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
#
|
||||
# Test for deterministic coverage across unit test runs.
|
||||
|
||||
export LC_ALL=C
|
||||
|
||||
# Use GCOV_EXECUTABLE="gcov" if compiling with gcc.
|
||||
# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang.
|
||||
GCOV_EXECUTABLE="gcov"
|
||||
|
||||
# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism.
|
||||
NON_DETERMINISTIC_TESTS=(
|
||||
"blockfilter_index_tests/blockfilter_index_initial_sync" # src/checkqueue.h: In CCheckQueue::Loop(): while (queue.empty()) { ... }
|
||||
"coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2))
|
||||
"fs_tests/fsbridge_fstream" # deterministic test failure?
|
||||
"miner_tests/CreateNewBlock_validity" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue()
|
||||
"scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue()
|
||||
"txvalidationcache_tests/checkinputs_test" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"txvalidationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"txindex_tests/txindex_initial_sync" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"wallet_tests/dummy_input_size_test" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"wallet_tests/importmulti_rescan" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"wallet_tests/importwallet_rescan" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"wallet_tests/ListCoins" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
"wallet_tests/wallet_disableprivkeys" # validation.cpp: if (signals.CallbacksPending() > 10)
|
||||
)
|
||||
|
||||
TEST_BITCOIN_BINARY="src/test/test_bitcoin"
|
||||
|
||||
print_usage() {
|
||||
echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]"
|
||||
}
|
||||
|
||||
N_TEST_RUNS=2
|
||||
BOOST_TEST_RUN_FILTERS=""
|
||||
if [[ $# != 0 ]]; then
|
||||
if [[ $1 == "--help" ]]; then
|
||||
print_usage
|
||||
exit
|
||||
fi
|
||||
PARSED_ARGUMENTS=0
|
||||
if [[ $1 =~ [a-z] ]]; then
|
||||
BOOST_TEST_RUN_FILTERS=$1
|
||||
PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1))
|
||||
shift
|
||||
fi
|
||||
if [[ $1 =~ ^[0-9]+$ ]]; then
|
||||
N_TEST_RUNS=$1
|
||||
PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1))
|
||||
shift
|
||||
fi
|
||||
if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then
|
||||
print_usage
|
||||
exit
|
||||
fi
|
||||
fi
|
||||
if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then
|
||||
BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')"
|
||||
else
|
||||
echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}"
|
||||
echo
|
||||
fi
|
||||
|
||||
if ! command -v gcov > /dev/null; then
|
||||
echo "Error: gcov not installed. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v gcovr > /dev/null; then
|
||||
echo "Error: gcovr not installed. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then
|
||||
echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and compile."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
get_file_suffix_count() {
|
||||
find src/ -type f -name "*.$1" | wc -l
|
||||
}
|
||||
|
||||
if [[ $(get_file_suffix_count gcno) == 0 ]]; then
|
||||
echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and re-compile."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
get_covr_filename() {
|
||||
echo "gcovr.run-$1.txt"
|
||||
}
|
||||
|
||||
TEST_RUN_ID=0
|
||||
while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do
|
||||
TEST_RUN_ID=$((TEST_RUN_ID + 1))
|
||||
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}"
|
||||
find src/ -type f -name "*.gcda" -exec rm {} \;
|
||||
if [[ $(get_file_suffix_count gcda) != 0 ]]; then
|
||||
echo "Error: Stale *.gcda files found. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
TEST_OUTPUT_TEMPFILE=$(mktemp)
|
||||
if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then
|
||||
cat "${TEST_OUTPUT_TEMPFILE}"
|
||||
rm "${TEST_OUTPUT_TEMPFILE}"
|
||||
exit 1
|
||||
fi
|
||||
rm "${TEST_OUTPUT_TEMPFILE}"
|
||||
if [[ $(get_file_suffix_count gcda) == 0 ]]; then
|
||||
echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and re-compile."
|
||||
exit 1
|
||||
fi
|
||||
GCOVR_TEMPFILE=$(mktemp)
|
||||
if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then
|
||||
echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID})
|
||||
mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}"
|
||||
if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then
|
||||
echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)."
|
||||
exit 1
|
||||
fi
|
||||
if [[ ${TEST_RUN_ID} != 1 ]]; then
|
||||
COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}")
|
||||
if [[ ${COVERAGE_DIFF} != "" ]]; then
|
||||
echo
|
||||
echo "The line coverage is non-deterministic between runs. Exiting."
|
||||
echo
|
||||
echo "The test suite must be deterministic in the sense that the set of lines executed at least"
|
||||
echo "once must be identical between runs. This is a necessary condition for meaningful"
|
||||
echo "coverage measuring."
|
||||
echo
|
||||
echo "${COVERAGE_DIFF}"
|
||||
exit 1
|
||||
fi
|
||||
rm "${GCOVR_FILENAME}"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs."
|
||||
exit
|
||||
Loading…
x
Reference in New Issue
Block a user