From fa7d75540ed266d76c946d7ab8fdf4aa2d12f40e Mon Sep 17 00:00:00 2001
From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz>
Date: Fri, 5 May 2023 08:45:37 +0200
Subject: [PATCH 1/3] ci: Pass full env to CI pod to avoid missing a var

Instead of enumerating each passed env var, just pass all. This avoids
the risk of missing to enumerate one. Also, it is less code.

The risk could be that an env var causes non-deterministic behavior, but
this can be fixed by explicitly excluding it once the issue is known.

Values with newlines can not be stored in the file and parsed by
docker/podman, so they are excluded.
---
 ci/README.md          | 4 ++--
 ci/test/04_install.sh | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/ci/README.md b/ci/README.md
index de798607df9..d014565f447 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -14,10 +14,10 @@ testing compared to other parts of the codebase. If you want to keep the work tr
 system in a virtual machine with a Linux operating system of your choice.
 
 To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
-requires `docker` to be installed. To install all requirements on Ubuntu, run
+requires `bash`, `docker`, and `python3` to be installed. To install all requirements on Ubuntu, run
 
 ```
-sudo apt install docker.io bash
+sudo apt install bash docker.io python3
 ```
 
 To run the default test stage,
diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
index 1024222e96a..6390667394f 100755
--- a/ci/test/04_install.sh
+++ b/ci/test/04_install.sh
@@ -18,7 +18,6 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=
 export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan"
 export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
-env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env
 if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
   CI_CONTAINER_CAP="--cap-add SYS_PTRACE"
 fi
@@ -27,6 +26,9 @@ export P_CI_DIR="$PWD"
 export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/"
 
 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
+  # Export all env vars to avoid missing some.
+  # Though, exclude those with newlines to avoid parsing problems.
+  python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value]' | tee /tmp/env
   echo "Creating $CI_IMAGE_NAME_TAG container to run in"
   DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \
       --file "${BASE_ROOT_DIR}/ci/test_imagefile" \

From fae8de926abaa9d7e5f0769865ded9491a72f866 Mon Sep 17 00:00:00 2001
From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz>
Date: Fri, 5 May 2023 08:51:41 +0200
Subject: [PATCH 2/3] ci: Move CI container kill out of 06_script_b.sh

This cleans up 06_script_b.sh to only contain code to be executed inside
the CI pod, which avoids confusion and is needed for the next commit.
---
 ci/test/06_script_b.sh | 5 -----
 ci/test_run_all.sh     | 5 +++++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh
index e0fae18b8eb..ea365d8314c 100755
--- a/ci/test/06_script_b.sh
+++ b/ci/test/06_script_b.sh
@@ -99,8 +99,3 @@ fi
 if [ "$RUN_FUZZ_TESTS" = "true" ]; then
   CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}"
 fi
-
-if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
-  echo "Stop and remove CI container by ID"
-  docker container kill "${CI_CONTAINER_ID}"
-fi
diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh
index 93b07aab1ec..f517d1b30e1 100755
--- a/ci/test_run_all.sh
+++ b/ci/test_run_all.sh
@@ -11,3 +11,8 @@ set -o errexit; source ./ci/test/04_install.sh
 set -o errexit; source ./ci/test/05_before_script.sh
 set -o errexit; source ./ci/test/06_script_a.sh
 set -o errexit; source ./ci/test/06_script_b.sh
+
+if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
+  echo "Stop and remove CI container by ID"
+  docker container kill "${CI_CONTAINER_ID}"
+fi

From fa1dbd04cab8039440e721eddabb760a40ba8c61 Mon Sep 17 00:00:00 2001
From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz>
Date: Thu, 4 May 2023 18:40:01 +0200
Subject: [PATCH 3/3] ci: Remove CI_EXEC bloat in test/06_script_b.sh

---
 ci/test/06_script_b.sh | 130 +++++++++++++++++++++--------------------
 ci/test_run_all.sh     |   3 +-
 2 files changed, 68 insertions(+), 65 deletions(-)

diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh
index ea365d8314c..3efbe8afc6c 100755
--- a/ci/test/06_script_b.sh
+++ b/ci/test/06_script_b.sh
@@ -6,96 +6,98 @@
 
 export LC_ALL=C.UTF-8
 
+set -ex
+
 if [[ $HOST = *-mingw32 ]]; then
   # Generate all binaries, so that they can be wrapped
-  CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
-  CI_EXEC make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1
-  CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
+  make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
+  make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1
+  "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
 fi
 
 if [ -n "$QEMU_USER_CMD" ]; then
   # Generate all binaries, so that they can be wrapped
-  CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
-  CI_EXEC make "$MAKEJOBS" -C src minisketch/test VERBOSE=1
-  CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh"
+  make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
+  make "$MAKEJOBS" -C src minisketch/test VERBOSE=1
+  "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh"
 fi
 
 if [ -n "$USE_VALGRIND" ]; then
-  CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
+  "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
 fi
 
 if [ "$RUN_UNIT_TESTS" = "true" ]; then
-  CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" make "$MAKEJOBS" check VERBOSE=1
+  bash -c "${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib make $MAKEJOBS check VERBOSE=1"
 fi
 
 if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
-  CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}/bin/test_bitcoin" --catch_system_errors=no -l test_suite
+  bash -c "${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib ${BASE_OUTDIR}/bin/test_bitcoin --catch_system_errors=no -l test_suite"
 fi
 
 if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
-  CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast
+  bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix ${BASE_SCRATCH_DIR}/test_runner/ --ansi --combinedlogslen=99999999 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast"
 fi
 
 if [ "${RUN_TIDY}" = "true" ]; then
   set -eo pipefail
-  export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/"
-  ( CI_EXEC run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error"
-  export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/"
-  CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\
-          " src/common/args.cpp"\
-          " src/common/config.cpp"\
-          " src/common/init.cpp"\
-          " src/common/url.cpp"\
-          " src/compat"\
-          " src/dbwrapper.cpp"\
-          " src/init"\
-          " src/kernel"\
-          " src/node/chainstate.cpp"\
-          " src/node/chainstatemanager_args.cpp"\
-          " src/node/mempool_args.cpp"\
-          " src/node/minisketchwrapper.cpp"\
-          " src/node/utxo_snapshot.cpp"\
-          " src/node/validation_cache_args.cpp"\
-          " src/policy/feerate.cpp"\
-          " src/policy/packages.cpp"\
-          " src/policy/settings.cpp"\
-          " src/primitives/transaction.cpp"\
-          " src/random.cpp"\
-          " src/rpc/fees.cpp"\
-          " src/rpc/signmessage.cpp"\
-          " src/test/fuzz/string.cpp"\
-          " src/test/fuzz/txorphan.cpp"\
-          " src/test/fuzz/util/"\
-          " src/test/util/coins.cpp"\
-          " src/uint256.cpp"\
-          " src/util/bip32.cpp"\
-          " src/util/bytevectorhash.cpp"\
-          " src/util/check.cpp"\
-          " src/util/error.cpp"\
-          " src/util/exception.cpp"\
-          " src/util/getuniquepath.cpp"\
-          " src/util/hasher.cpp"\
-          " src/util/message.cpp"\
-          " src/util/moneystr.cpp"\
-          " src/util/serfloat.cpp"\
-          " src/util/spanparsing.cpp"\
-          " src/util/strencodings.cpp"\
-          " src/util/string.cpp"\
-          " src/util/syserror.cpp"\
-          " src/util/threadinterrupt.cpp"\
-          " src/zmq"\
-          " -p . ${MAKEJOBS}"\
-          " -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp"\
-          " |& tee /tmp/iwyu_ci.out"
-  export P_CI_DIR="${BASE_ROOT_DIR}/src"
-  CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/fix_includes.py --nosafe_headers < /tmp/iwyu_ci.out"
-  CI_EXEC "git --no-pager diff"
+  cd "${BASE_BUILD_DIR}/bitcoin-$HOST/src/"
+  ( run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error"
+  cd "${BASE_BUILD_DIR}/bitcoin-$HOST/"
+  python3 "${DIR_IWYU}/include-what-you-use/iwyu_tool.py" \
+           src/common/args.cpp \
+           src/common/config.cpp \
+           src/common/init.cpp \
+           src/common/url.cpp \
+           src/compat \
+           src/dbwrapper.cpp \
+           src/init \
+           src/kernel \
+           src/node/chainstate.cpp \
+           src/node/chainstatemanager_args.cpp \
+           src/node/mempool_args.cpp \
+           src/node/minisketchwrapper.cpp \
+           src/node/utxo_snapshot.cpp \
+           src/node/validation_cache_args.cpp \
+           src/policy/feerate.cpp \
+           src/policy/packages.cpp \
+           src/policy/settings.cpp \
+           src/primitives/transaction.cpp \
+           src/random.cpp \
+           src/rpc/fees.cpp \
+           src/rpc/signmessage.cpp \
+           src/test/fuzz/string.cpp \
+           src/test/fuzz/txorphan.cpp \
+           src/test/fuzz/util \
+           src/test/util/coins.cpp \
+           src/uint256.cpp \
+           src/util/bip32.cpp \
+           src/util/bytevectorhash.cpp \
+           src/util/check.cpp \
+           src/util/error.cpp \
+           src/util/exception.cpp \
+           src/util/getuniquepath.cpp \
+           src/util/hasher.cpp \
+           src/util/message.cpp \
+           src/util/moneystr.cpp \
+           src/util/serfloat.cpp \
+           src/util/spanparsing.cpp \
+           src/util/strencodings.cpp \
+           src/util/string.cpp \
+           src/util/syserror.cpp \
+           src/util/threadinterrupt.cpp \
+           src/zmq \
+           -p . "${MAKEJOBS}" \
+           -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" \
+           2>&1 | tee /tmp/iwyu_ci.out
+  cd "${BASE_ROOT_DIR}/src"
+  python3 "${DIR_IWYU}/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out
+  git --no-pager diff
 fi
 
 if [ "$RUN_SECURITY_TESTS" = "true" ]; then
-  CI_EXEC make test-security-check
+  make test-security-check
 fi
 
 if [ "$RUN_FUZZ_TESTS" = "true" ]; then
-  CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}"
+  bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN}"
 fi
diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh
index f517d1b30e1..751a4056df7 100755
--- a/ci/test_run_all.sh
+++ b/ci/test_run_all.sh
@@ -10,7 +10,8 @@ set -o errexit; source ./ci/test/00_setup_env.sh
 set -o errexit; source ./ci/test/04_install.sh
 set -o errexit; source ./ci/test/05_before_script.sh
 set -o errexit; source ./ci/test/06_script_a.sh
-set -o errexit; source ./ci/test/06_script_b.sh
+set -o errexit
+CI_EXEC "${BASE_ROOT_DIR}/ci/test/06_script_b.sh"
 
 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   echo "Stop and remove CI container by ID"