diff --git a/ci/test/04_install.sh b/ci/test/02_run_container.sh
similarity index 92%
rename from ci/test/04_install.sh
rename to ci/test/02_run_container.sh
index b5a84ae08d6..a7c1e22a6db 100755
--- a/ci/test/04_install.sh
+++ b/ci/test/02_run_container.sh
@@ -6,6 +6,8 @@
 
 export LC_ALL=C.UTF-8
 
+set -ex
+
 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   # Export all env vars to avoid missing some.
   # Though, exclude those with newlines to avoid parsing problems.
@@ -27,14 +29,12 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
 
   if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
     echo "Restart docker before run to stop and clear all containers started with --rm"
-    podman container stop --all  # Similar to "systemctl restart docker"
+    podman container rm --force --all  # Similar to "systemctl restart docker"
     echo "Prune all dangling images"
     docker image prune --force
   fi
 
   # shellcheck disable=SC2086
-
-
   CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \
                   --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \
                   --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \
@@ -68,3 +68,10 @@ CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh"
 CI_EXEC git config --global --add safe.directory \"*\"
 
 CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}"
+
+CI_EXEC "${BASE_ROOT_DIR}/ci/test/06_script_b.sh"
+
+if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
+  echo "Stop and remove CI container by ID"
+  docker container kill "${CI_CONTAINER_ID}"
+fi
diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh
index 2284a2903b7..3afc47b23ed 100755
--- a/ci/test_run_all.sh
+++ b/ci/test_run_all.sh
@@ -1,17 +1,11 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
 set -o errexit; source ./ci/test/00_setup_env.sh
-set -o errexit; source ./ci/test/04_install.sh
 set -o errexit
-CI_EXEC "${BASE_ROOT_DIR}/ci/test/06_script_b.sh"
-
-if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
-  echo "Stop and remove CI container by ID"
-  docker container kill "${CI_CONTAINER_ID}"
-fi
+"./ci/test/02_run_container.sh"