bitcoin/ci/test/02_run_container.py

#!/usr/bin/env python3
# Copyright (c) The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or https://opensource.org/license/mit/.

from pathlib import Path
import os
import shlex
import subprocess
import sys
import time


def run(cmd, **kwargs):
    print("+ " + shlex.join(cmd), flush=True)
    kwargs.setdefault("check", True)
    try:
        return subprocess.run(cmd, **kwargs)
    except Exception as e:
        sys.exit(e)


def main():
    print("Export only allowed settings:")
    settings = run(
        ["bash", "-c", "grep export ./ci/test/00_setup_env*.sh"],
        stdout=subprocess.PIPE,
        text=True,
        encoding="utf8",
    ).stdout.splitlines()
    settings = set(l.split("=")[0].split("export ")[1] for l in settings)
    # Add "hidden" settings, which are never exported, manually. Otherwise,
    # they will not be passed on.
    settings.update([
        "BASE_BUILD_DIR",
        "CI_FAILFAST_TEST_LEAVE_DANGLING",
    ])

    # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME
    # to allow support starting multiple runs simultaneously by the same user.
    env_file = "/tmp/env-{u}-{c}".format(
        u=os.environ["USER"],
        c=os.environ["CONTAINER_NAME"],
    )
    with open(env_file, "w", encoding="utf8") as file:
        for k, v in os.environ.items():
            if k in settings:
                file.write(f"{k}={v}\n")
    run(["cat", env_file])

    if os.getenv("DANGER_RUN_CI_ON_HOST"):
        print("Running on host system without docker wrapper")
        print("Create missing folders")
        for create_dir in [
                os.environ["CCACHE_DIR"],
                os.environ["PREVIOUS_RELEASES_DIR"],
        ]:
            Path(create_dir).mkdir(parents=True, exist_ok=True)

        # Modify PATH to prepend the retry script, needed for CI_RETRY_EXE
        os.environ["PATH"] = f"{os.environ['BASE_ROOT_DIR']}/ci/retry:{os.environ['PATH']}"
        # GNU getopt is required for the CI_RETRY_EXE script
        if os.getenv("CI_OS_NAME") == "macos":
            prefix = run(
                ["brew", "--prefix", "gnu-getopt"],
                stdout=subprocess.PIPE,
                text=True,
            ).stdout.strip()
            os.environ["IN_GETOPT_BIN"] = f"{prefix}/bin/getopt"
    else:
        CI_IMAGE_LABEL = "bitcoin-ci-test"

        # Use buildx unconditionally
        # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly
        cmd_build = ["docker", "buildx", "build"]
        cmd_build += [
            f"--file={os.environ['BASE_READ_ONLY_DIR']}/ci/test_imagefile",
            f"--build-arg=CI_IMAGE_NAME_TAG={os.environ['CI_IMAGE_NAME_TAG']}",
            f"--build-arg=FILE_ENV={os.environ['FILE_ENV']}",
            f"--build-arg=BASE_ROOT_DIR={os.environ['BASE_ROOT_DIR']}",
            f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
            f"--label={CI_IMAGE_LABEL}",
            f"--tag={os.environ['CONTAINER_NAME']}",
        ]
        cmd_build += shlex.split(os.getenv("DOCKER_BUILD_CACHE_ARG", ""))
        cmd_build += [os.environ["BASE_READ_ONLY_DIR"]]

        print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in")
        if run(cmd_build, check=False).returncode != 0:
            print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure")
            time.sleep(3)
            run(cmd_build)

        for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]:
            run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False)

        CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}"
        CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built"
        CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources"
        CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}"
        CI_BUILD_MOUNT = []

        if os.getenv("DANGER_CI_ON_HOST_FOLDERS"):
            # ensure the directories exist
            for create_dir in [
                    os.environ["CCACHE_DIR"],
                    f"{os.environ['DEPENDS_DIR']}/built",
                    f"{os.environ['DEPENDS_DIR']}/sources",
                    os.environ["PREVIOUS_RELEASES_DIR"],
                    os.environ["BASE_BUILD_DIR"],  # Unset by default, must be defined externally
            ]:
                Path(create_dir).mkdir(parents=True, exist_ok=True)

            CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
            CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built"
            CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources"
            CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}"
            CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"]

        if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"):
            if not os.path.isdir(os.environ["CCACHE_DIR"]):
                print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.")
                sys.exit(1)
            CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"

        run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False)

        if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"):
            print("Restart docker before run to stop and clear all containers started with --rm")
            run(["podman", "container", "rm", "--force", "--all"])  # Similar to "systemctl restart docker"

            # Still prune everything in case the filtered pruning doesn't work, or if labels were not set
            # on a previous run. Belt and suspenders approach, should be fine to remove in the future.
            # Prune images used by --external containers (e.g. build containers) when
            # using podman.
            print("Prune all dangling images")
            run(["podman", "image", "prune", "--force", "--external"])

        print(f"Prune all dangling {CI_IMAGE_LABEL} images")
        # When detecting podman-docker, `--external` should be added.
        run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"])

        cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"]
        cmd_run += [
            "--cap-add=LINUX_IMMUTABLE",
            *shlex.split(os.getenv("CI_CONTAINER_CAP", "")),
            f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly",
            f"--mount={CI_CCACHE_MOUNT}",
            f"--mount={CI_DEPENDS_MOUNT}",
            f"--mount={CI_DEPENDS_SOURCES_MOUNT}",
            f"--mount={CI_PREVIOUS_RELEASES_MOUNT}",
            *CI_BUILD_MOUNT,
            f"--env-file={env_file}",
            f"--name={os.environ['CONTAINER_NAME']}",
            "--network=ci-ip6net",
            f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
            os.environ["CONTAINER_NAME"],
        ]

        container_id = run(
            cmd_run,
            stdout=subprocess.PIPE,
            text=True,
        ).stdout.strip()
        os.environ["CI_CONTAINER_ID"] = container_id

    run(["./ci/test/02_run_container.sh"])  # run the remainder
    if not os.getenv("DANGER_RUN_CI_ON_HOST"):
        print("Stop and remove CI container by ID")
        run(["docker", "container", "kill", container_id])


if __name__ == "__main__":
    main()