Merge bitcoin/bitcoin#33677: ci: Retry image building once on failure

5555bce994 ci: Document why IN_GETOPT_BIN env var is needed on macOS (MarcoFalke) fabe516440 ci: Export the container id in python script (MarcoFalke) fa6aa9f42f ci: Retry image building once on failure (MarcoFalke) fa4dbe04d7 ci: Allow overwriting check option in run() helper (MarcoFalke) fa8e4de5c3 ci: Use os.environ[key] access when value must be set (MarcoFalke) Pull request description: This should fix https://github.com/bitcoin/bitcoin/issues/33640. It also contains a few refactor cleanups, which are explained in the corresponding commits. ACKs for top commit: l0rinc: Code review reACK 5555bce994 kevkevinpal: ACK [5555bce](5555bce994) davidgumberg: crACK 5555bce994 Tree-SHA512: f1ea95b0650e57d6a9f97c575a11ee461832c0715c3d1a24dbfe12ccc5366f295639d4c4827f1d01da460ddf00917ecaa627e7dbd12e405770db6c53c3778a9c
2026-01-19 23:03:45 +01:00 · 2025-10-29 10:09:52 +00:00
parent dd82c6c5d0 5555bce994
commit 292ea0eb89
2 changed files with 92 additions and 75 deletions
--- a/ci/test/02_run_container.py
+++ b/ci/test/02_run_container.py
@@ -3,16 +3,19 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or https://opensource.org/license/mit/.

+from pathlib import Path
 import os
 import shlex
 import subprocess
 import sys
+import time


 def run(cmd, **kwargs):
    print("+ " + shlex.join(cmd), flush=True)
+    kwargs.setdefault("check", True)
    try:
-        return subprocess.run(cmd, check=True, **kwargs)
+        return subprocess.run(cmd, **kwargs)
    except Exception as e:
        sys.exit(e)

@@ -36,8 +39,8 @@ def main():
    # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME
    # to allow support starting multiple runs simultaneously by the same user.
    env_file = "/tmp/env-{u}-{c}".format(
-        u=os.getenv("USER"),
-        c=os.getenv("CONTAINER_NAME"),
+        u=os.environ["USER"],
+        c=os.environ["CONTAINER_NAME"],
    )
    with open(env_file, "w", encoding="utf8") as file:
        for k, v in os.environ.items():
@@ -64,7 +67,92 @@ def main():
        cmd_build += [os.environ["BASE_READ_ONLY_DIR"]]

        print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in")
-        run(cmd_build)
+        if run(cmd_build, check=False).returncode != 0:
+            print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure")
+            time.sleep(3)
+            run(cmd_build)
+
+        for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]:
+            run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False)
+
+        CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}"
+        CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built"
+        CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources"
+        CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}"
+        CI_BUILD_MOUNT = []
+
+        if os.getenv("DANGER_CI_ON_HOST_FOLDERS"):
+            # ensure the directories exist
+            for create_dir in [
+                    os.environ["CCACHE_DIR"],
+                    f"{os.environ['DEPENDS_DIR']}/built",
+                    f"{os.environ['DEPENDS_DIR']}/sources",
+                    os.environ["PREVIOUS_RELEASES_DIR"],
+                    os.environ["BASE_BUILD_DIR"],  # Unset by default, must be defined externally
+            ]:
+                Path(create_dir).mkdir(parents=True, exist_ok=True)
+
+            CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
+            CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built"
+            CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources"
+            CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}"
+            CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"]
+
+        if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"):
+            if not os.path.isdir(os.environ["CCACHE_DIR"]):
+                print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.")
+                sys.exit(1)
+            CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
+
+        run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False)
+
+        if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"):
+            print("Restart docker before run to stop and clear all containers started with --rm")
+            run(["podman", "container", "rm", "--force", "--all"])  # Similar to "systemctl restart docker"
+
+            # Still prune everything in case the filtered pruning doesn't work, or if labels were not set
+            # on a previous run. Belt and suspenders approach, should be fine to remove in the future.
+            # Prune images used by --external containers (e.g. build containers) when
+            # using podman.
+            print("Prune all dangling images")
+            run(["podman", "image", "prune", "--force", "--external"])
+
+        print(f"Prune all dangling {CI_IMAGE_LABEL} images")
+        # When detecting podman-docker, `--external` should be added.
+        run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"])
+
+        cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"]
+        cmd_run += [
+            "--cap-add=LINUX_IMMUTABLE",
+            *shlex.split(os.getenv("CI_CONTAINER_CAP", "")),
+            f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly",
+            f"--mount={CI_CCACHE_MOUNT}",
+            f"--mount={CI_DEPENDS_MOUNT}",
+            f"--mount={CI_DEPENDS_SOURCES_MOUNT}",
+            f"--mount={CI_PREVIOUS_RELEASES_MOUNT}",
+            *CI_BUILD_MOUNT,
+            f"--env-file={env_file}",
+            f"--name={os.environ['CONTAINER_NAME']}",
+            "--network=ci-ip6net",
+            f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
+            os.environ["CONTAINER_NAME"],
+        ]
+
+        container_id = run(
+            cmd_run,
+            stdout=subprocess.PIPE,
+            text=True,
+        ).stdout.strip()
+        os.environ["CI_CONTAINER_ID"] = container_id
+
+    # GNU getopt is required for the CI_RETRY_EXE script
+    if os.getenv("CI_OS_NAME") == "macos":
+        prefix = run(
+            ["brew", "--prefix", "gnu-getopt"],
+            stdout=subprocess.PIPE,
+            text=True,
+        ).stdout.strip()
+        os.environ["IN_GETOPT_BIN"] = f"{prefix}/bin/getopt"

    run(["./ci/test/02_run_container.sh"])  # run the remainder

--- a/ci/test/02_run_container.sh
+++ b/ci/test/02_run_container.sh
@@ -5,76 +5,10 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.

 export LC_ALL=C.UTF-8
-export CI_IMAGE_LABEL="bitcoin-ci-test"

 set -o errexit -o pipefail -o xtrace

 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
-  docker volume create "${CONTAINER_NAME}_ccache" || true
-  docker volume create "${CONTAINER_NAME}_depends" || true
-  docker volume create "${CONTAINER_NAME}_depends_sources" || true
-  docker volume create "${CONTAINER_NAME}_previous_releases" || true
-
-  CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR"
-  CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built"
-  CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources"
-  CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR"
-  CI_BUILD_MOUNT=""
-
-  if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then
-    # ensure the directories exist
-    mkdir -p "${CCACHE_DIR}"
-    mkdir -p "${DEPENDS_DIR}/built"
-    mkdir -p "${DEPENDS_DIR}/sources"
-    mkdir -p "${PREVIOUS_RELEASES_DIR}"
-    mkdir -p "${BASE_BUILD_DIR}"  # Unset by default, must be defined externally
-
-    CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR"
-    CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built"
-    CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources"
-    CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR"
-    CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}"
-  fi
-
-  if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then
-    if [ ! -d "${CCACHE_DIR}" ]; then
-      echo "Error: Directory '${CCACHE_DIR}' must be created in advance."
-      exit 1
-    fi
-    CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}"
-  fi
-
-  docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true
-
-  if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
-    echo "Restart docker before run to stop and clear all containers started with --rm"
-    podman container rm --force --all  # Similar to "systemctl restart docker"
-
-    # Still prune everything in case the filtered pruning doesn't work, or if labels were not set
-    # on a previous run. Belt and suspenders approach, should be fine to remove in the future.
-    # Prune images used by --external containers (e.g. build containers) when
-    # using podman.
-    echo "Prune all dangling images"
-    podman image prune --force --external
-  fi
-  echo "Prune all dangling $CI_IMAGE_LABEL images"
-  # When detecting podman-docker, `--external` should be added.
-  docker image prune --force --filter "label=$CI_IMAGE_LABEL"
-
-  # shellcheck disable=SC2086
-  CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \
-                  --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \
-                  --mount "${CI_CCACHE_MOUNT}" \
-                  --mount "${CI_DEPENDS_MOUNT}" \
-                  --mount "${CI_DEPENDS_SOURCES_MOUNT}" \
-                  --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \
-                  ${CI_BUILD_MOUNT} \
-                  --env-file /tmp/env-$USER-$CONTAINER_NAME \
-                  --name "$CONTAINER_NAME" \
-                  --network ci-ip6net \
-                  --platform="${CI_IMAGE_PLATFORM}" \
-                  "$CONTAINER_NAME")
-  export CI_CONTAINER_ID
  export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}"
 else
  echo "Running on host system without docker wrapper"
@@ -83,11 +17,6 @@ else
  mkdir -p "${PREVIOUS_RELEASES_DIR}"
 fi

-if [ "$CI_OS_NAME" == "macos" ]; then
-  IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt"
-  export IN_GETOPT_BIN
-fi
-
 CI_EXEC () {
  $CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*"
 }