Merge bitcoin/bitcoin#33639: ci: Only write docker build images to Cirrus cache

fabe0e07de ci: Only write docker build images to Cirrus cache (MarcoFalke)
fab64a5d6f ci: Move buildx command to python script (MarcoFalke)
fa72a2bd5c ci: Remove unused MAYBE_CPUSET (MarcoFalke)

Pull request description:

  The `DOCKER_BUILD_CACHE_ARG` env var holds the options on how to use cache providers. Storing the image layers is useful for the Cirrus cache provider, because it offers 10GB per runner (https://cirrus-runners.app/setup/#speeding-up-the-cache). The cached image layers can help to avoid issues when the upstream package manager infra (apt native, apt llvm, pip, apk, git clone, ...) has outages or network issues.

  However, on the GitHub Actions cache provider, a *total* cache of 10GB is offered for the whole repo. This cache must be shared with the depends cache, and the ccache, as well as the previous releases cache. So it is already full and trying to put the docker build layers into it will lead to an overflow.

  Fix it by only writing to the docker cache on Cirrus.

  Also, `DOCKER_BUILD_CACHE_ARG` requires a `shellcheck disable=SC2086` on the full build command. Fix that as well by using `shlex.split` from Python on just this variable.

ACKs for top commit:
  m3dwards:
    ACK fabe0e07de
  cedwies:
    reACK fabe0e0
  l0rinc:
    Code review ACK fabe0e07de
  willcl-ark:
    ACK fabe0e07de

Tree-SHA512: 4f471f080007fdd0c3bc97b0cfe0e9c0457e5029a7ccde1d784d30eb4752e5eb309cd4b122b182bce31f1b986c8a9f3e9a49da1768bedbb2b1f64f70183680ba

.github/actions/configure-docker/action.yml

@@ -48,8 +48,8 @@ runs:
         # Always optimistically --cache‑from in case a cache blob exists
         args=(--cache-from "type=gha${url_args:+,${url_args}},scope=${CONTAINER_NAME}")
 
-        # If this is a push to the default branch, also add --cache‑to to save the cache
-        if [[ ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then
+        # Only add --cache-to when using the Cirrus cache provider and pushing to the default branch.
+        if [[ ${{ inputs.cache-provider }} == 'cirrus' && ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then
           args+=(--cache-to "type=gha${url_args:+,${url_args}},mode=max,ignore-error=true,scope=${CONTAINER_NAME}")
         fi
 

ci/test/02_run_container.py

@@ -45,6 +45,27 @@ def main():
                 file.write(f"{k}={v}\n")
     run(["cat", env_file])
 
+    if not os.getenv("DANGER_RUN_CI_ON_HOST"):
+        CI_IMAGE_LABEL = "bitcoin-ci-test"
+
+        # Use buildx unconditionally
+        # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly
+        cmd_build = ["docker", "buildx", "build"]
+        cmd_build += [
+            f"--file={os.environ['BASE_READ_ONLY_DIR']}/ci/test_imagefile",
+            f"--build-arg=CI_IMAGE_NAME_TAG={os.environ['CI_IMAGE_NAME_TAG']}",
+            f"--build-arg=FILE_ENV={os.environ['FILE_ENV']}",
+            f"--build-arg=BASE_ROOT_DIR={os.environ['BASE_ROOT_DIR']}",
+            f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
+            f"--label={CI_IMAGE_LABEL}",
+            f"--tag={os.environ['CONTAINER_NAME']}",
+        ]
+        cmd_build += shlex.split(os.getenv("DOCKER_BUILD_CACHE_ARG", ""))
+        cmd_build += [os.environ["BASE_READ_ONLY_DIR"]]
+
+        print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in")
+        run(cmd_build)
+
     run(["./ci/test/02_run_container.sh"])  # run the remainder
 
 

ci/test/02_run_container.sh

@@ -10,30 +10,6 @@ export CI_IMAGE_LABEL="bitcoin-ci-test"
 set -o errexit -o pipefail -o xtrace
 
 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
-  # Env vars during the build can not be changed. For example, a modified
-  # $MAKEJOBS is ignored in the build process. Use --cpuset-cpus as an
-  # approximation to respect $MAKEJOBS somewhat, if cpuset is available.
-  MAYBE_CPUSET=""
-  if [ "$HAVE_CGROUP_CPUSET" ]; then
-    MAYBE_CPUSET="--cpuset-cpus=$( python3 -c "import random;P=$( nproc );M=min(P,int('$MAKEJOBS'.lstrip('-j')));print(','.join(map(str,sorted(random.sample(range(P),M)))))" )"
-  fi
-  echo "Creating $CI_IMAGE_NAME_TAG container to run in"
-
-  # Use buildx unconditionally
-  # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly
-  # shellcheck disable=SC2086
-  docker buildx build \
-      --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \
-      --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \
-      --build-arg "FILE_ENV=${FILE_ENV}" \
-      --build-arg "BASE_ROOT_DIR=${BASE_ROOT_DIR}" \
-      $MAYBE_CPUSET \
-      --platform="${CI_IMAGE_PLATFORM}" \
-      --label="${CI_IMAGE_LABEL}" \
-      --tag="${CONTAINER_NAME}" \
-      $DOCKER_BUILD_CACHE_ARG \
-      "${BASE_READ_ONLY_DIR}"
-
   docker volume create "${CONTAINER_NAME}_ccache" || true
   docker volume create "${CONTAINER_NAME}_depends" || true
   docker volume create "${CONTAINER_NAME}_depends_sources" || true