diff --git a/ci/test/02_run_container.py b/ci/test/02_run_container.py index c1c6b940d72..4d0bed2ad41 100755 --- a/ci/test/02_run_container.py +++ b/ci/test/02_run_container.py @@ -3,16 +3,19 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. +from pathlib import Path import os import shlex import subprocess import sys +import time def run(cmd, **kwargs): print("+ " + shlex.join(cmd), flush=True) + kwargs.setdefault("check", True) try: - return subprocess.run(cmd, check=True, **kwargs) + return subprocess.run(cmd, **kwargs) except Exception as e: sys.exit(e) @@ -36,8 +39,8 @@ def main(): # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME # to allow support starting multiple runs simultaneously by the same user. env_file = "/tmp/env-{u}-{c}".format( - u=os.getenv("USER"), - c=os.getenv("CONTAINER_NAME"), + u=os.environ["USER"], + c=os.environ["CONTAINER_NAME"], ) with open(env_file, "w", encoding="utf8") as file: for k, v in os.environ.items(): @@ -64,7 +67,92 @@ def main(): cmd_build += [os.environ["BASE_READ_ONLY_DIR"]] print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in") - run(cmd_build) + if run(cmd_build, check=False).returncode != 0: + print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure") + time.sleep(3) + run(cmd_build) + + for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]: + run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False) + + CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}" + CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built" + CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources" + CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}" + CI_BUILD_MOUNT = [] + + if os.getenv("DANGER_CI_ON_HOST_FOLDERS"): + # ensure the directories exist + for create_dir in [ + os.environ["CCACHE_DIR"], + f"{os.environ['DEPENDS_DIR']}/built", + f"{os.environ['DEPENDS_DIR']}/sources", + os.environ["PREVIOUS_RELEASES_DIR"], + os.environ["BASE_BUILD_DIR"], # Unset by default, must be defined externally + ]: + Path(create_dir).mkdir(parents=True, exist_ok=True) + + CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}" + CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built" + CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources" + CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}" + CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"] + + if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"): + if not os.path.isdir(os.environ["CCACHE_DIR"]): + print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.") + sys.exit(1) + CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}" + + run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False) + + if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"): + print("Restart docker before run to stop and clear all containers started with --rm") + run(["podman", "container", "rm", "--force", "--all"]) # Similar to "systemctl restart docker" + + # Still prune everything in case the filtered pruning doesn't work, or if labels were not set + # on a previous run. Belt and suspenders approach, should be fine to remove in the future. + # Prune images used by --external containers (e.g. build containers) when + # using podman. + print("Prune all dangling images") + run(["podman", "image", "prune", "--force", "--external"]) + + print(f"Prune all dangling {CI_IMAGE_LABEL} images") + # When detecting podman-docker, `--external` should be added. + run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"]) + + cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"] + cmd_run += [ + "--cap-add=LINUX_IMMUTABLE", + *shlex.split(os.getenv("CI_CONTAINER_CAP", "")), + f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly", + f"--mount={CI_CCACHE_MOUNT}", + f"--mount={CI_DEPENDS_MOUNT}", + f"--mount={CI_DEPENDS_SOURCES_MOUNT}", + f"--mount={CI_PREVIOUS_RELEASES_MOUNT}", + *CI_BUILD_MOUNT, + f"--env-file={env_file}", + f"--name={os.environ['CONTAINER_NAME']}", + "--network=ci-ip6net", + f"--platform={os.environ['CI_IMAGE_PLATFORM']}", + os.environ["CONTAINER_NAME"], + ] + + container_id = run( + cmd_run, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + os.environ["CI_CONTAINER_ID"] = container_id + + # GNU getopt is required for the CI_RETRY_EXE script + if os.getenv("CI_OS_NAME") == "macos": + prefix = run( + ["brew", "--prefix", "gnu-getopt"], + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + os.environ["IN_GETOPT_BIN"] = f"{prefix}/bin/getopt" run(["./ci/test/02_run_container.sh"]) # run the remainder diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh index 2e976773c29..251b01c0eb9 100755 --- a/ci/test/02_run_container.sh +++ b/ci/test/02_run_container.sh @@ -5,76 +5,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_LABEL="bitcoin-ci-test" set -o errexit -o pipefail -o xtrace if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - docker volume create "${CONTAINER_NAME}_ccache" || true - docker volume create "${CONTAINER_NAME}_depends" || true - docker volume create "${CONTAINER_NAME}_depends_sources" || true - docker volume create "${CONTAINER_NAME}_previous_releases" || true - - CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" - CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built" - CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources" - CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" - CI_BUILD_MOUNT="" - - if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then - # ensure the directories exist - mkdir -p "${CCACHE_DIR}" - mkdir -p "${DEPENDS_DIR}/built" - mkdir -p "${DEPENDS_DIR}/sources" - mkdir -p "${PREVIOUS_RELEASES_DIR}" - mkdir -p "${BASE_BUILD_DIR}" # Unset by default, must be defined externally - - CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR" - CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built" - CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources" - CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR" - CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}" - fi - - if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then - if [ ! -d "${CCACHE_DIR}" ]; then - echo "Error: Directory '${CCACHE_DIR}' must be created in advance." - exit 1 - fi - CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}" - fi - - docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true - - if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then - echo "Restart docker before run to stop and clear all containers started with --rm" - podman container rm --force --all # Similar to "systemctl restart docker" - - # Still prune everything in case the filtered pruning doesn't work, or if labels were not set - # on a previous run. Belt and suspenders approach, should be fine to remove in the future. - # Prune images used by --external containers (e.g. build containers) when - # using podman. - echo "Prune all dangling images" - podman image prune --force --external - fi - echo "Prune all dangling $CI_IMAGE_LABEL images" - # When detecting podman-docker, `--external` should be added. - docker image prune --force --filter "label=$CI_IMAGE_LABEL" - - # shellcheck disable=SC2086 - CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \ - --mount "${CI_CCACHE_MOUNT}" \ - --mount "${CI_DEPENDS_MOUNT}" \ - --mount "${CI_DEPENDS_SOURCES_MOUNT}" \ - --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \ - ${CI_BUILD_MOUNT} \ - --env-file /tmp/env-$USER-$CONTAINER_NAME \ - --name "$CONTAINER_NAME" \ - --network ci-ip6net \ - --platform="${CI_IMAGE_PLATFORM}" \ - "$CONTAINER_NAME") - export CI_CONTAINER_ID export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else echo "Running on host system without docker wrapper" @@ -83,11 +17,6 @@ else mkdir -p "${PREVIOUS_RELEASES_DIR}" fi -if [ "$CI_OS_NAME" == "macos" ]; then - IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" - export IN_GETOPT_BIN -fi - CI_EXEC () { $CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*" }