mirror of
https://github.com/bitcoin/bitcoin.git
synced 2026-01-20 07:09:15 +01:00
ci: Export the container id in python script
This refactor does not change behavior, but it has a few benefits: * The shellcheck SC2086 warning is disabled for the whole command, but is only needed for the CI_CONTAINER_CAP env var. So in Python, only pass this one env var to shlex.split() for proper word splitting. * Future logic improvements can be implemented in Python. The comments are moved, which can be checked via the git options: --color-moved=dimmed-zebra --color-moved-ws=ignore-all-space
This commit is contained in:
MarcoFalke
@@ -3,6 +3,7 @@
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or https://opensource.org/license/mit/.
|
||||
|
||||
from pathlib import Path
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
@@ -71,6 +72,79 @@ def main():
|
||||
time.sleep(3)
|
||||
run(cmd_build)
|
||||
|
||||
for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]:
|
||||
run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False)
|
||||
|
||||
CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}"
|
||||
CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built"
|
||||
CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources"
|
||||
CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}"
|
||||
CI_BUILD_MOUNT = []
|
||||
|
||||
if os.getenv("DANGER_CI_ON_HOST_FOLDERS"):
|
||||
# ensure the directories exist
|
||||
for create_dir in [
|
||||
os.environ["CCACHE_DIR"],
|
||||
f"{os.environ['DEPENDS_DIR']}/built",
|
||||
f"{os.environ['DEPENDS_DIR']}/sources",
|
||||
os.environ["PREVIOUS_RELEASES_DIR"],
|
||||
os.environ["BASE_BUILD_DIR"], # Unset by default, must be defined externally
|
||||
]:
|
||||
Path(create_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
|
||||
CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built"
|
||||
CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources"
|
||||
CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}"
|
||||
CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"]
|
||||
|
||||
if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"):
|
||||
if not os.path.isdir(os.environ["CCACHE_DIR"]):
|
||||
print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.")
|
||||
sys.exit(1)
|
||||
CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
|
||||
|
||||
run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False)
|
||||
|
||||
if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"):
|
||||
print("Restart docker before run to stop and clear all containers started with --rm")
|
||||
run(["podman", "container", "rm", "--force", "--all"]) # Similar to "systemctl restart docker"
|
||||
|
||||
# Still prune everything in case the filtered pruning doesn't work, or if labels were not set
|
||||
# on a previous run. Belt and suspenders approach, should be fine to remove in the future.
|
||||
# Prune images used by --external containers (e.g. build containers) when
|
||||
# using podman.
|
||||
print("Prune all dangling images")
|
||||
run(["podman", "image", "prune", "--force", "--external"])
|
||||
|
||||
print(f"Prune all dangling {CI_IMAGE_LABEL} images")
|
||||
# When detecting podman-docker, `--external` should be added.
|
||||
run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"])
|
||||
|
||||
cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"]
|
||||
cmd_run += [
|
||||
"--cap-add=LINUX_IMMUTABLE",
|
||||
*shlex.split(os.getenv("CI_CONTAINER_CAP", "")),
|
||||
f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly",
|
||||
f"--mount={CI_CCACHE_MOUNT}",
|
||||
f"--mount={CI_DEPENDS_MOUNT}",
|
||||
f"--mount={CI_DEPENDS_SOURCES_MOUNT}",
|
||||
f"--mount={CI_PREVIOUS_RELEASES_MOUNT}",
|
||||
*CI_BUILD_MOUNT,
|
||||
f"--env-file={env_file}",
|
||||
f"--name={os.environ['CONTAINER_NAME']}",
|
||||
"--network=ci-ip6net",
|
||||
f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
|
||||
os.environ["CONTAINER_NAME"],
|
||||
]
|
||||
|
||||
container_id = run(
|
||||
cmd_run,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
os.environ["CI_CONTAINER_ID"] = container_id
|
||||
|
||||
run(["./ci/test/02_run_container.sh"]) # run the remainder
|
||||
|
||||
|
||||
|
||||
@@ -5,76 +5,10 @@
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
export LC_ALL=C.UTF-8
|
||||
export CI_IMAGE_LABEL="bitcoin-ci-test"
|
||||
|
||||
set -o errexit -o pipefail -o xtrace
|
||||
|
||||
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
|
||||
docker volume create "${CONTAINER_NAME}_ccache" || true
|
||||
docker volume create "${CONTAINER_NAME}_depends" || true
|
||||
docker volume create "${CONTAINER_NAME}_depends_sources" || true
|
||||
docker volume create "${CONTAINER_NAME}_previous_releases" || true
|
||||
|
||||
CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR"
|
||||
CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built"
|
||||
CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources"
|
||||
CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR"
|
||||
CI_BUILD_MOUNT=""
|
||||
|
||||
if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then
|
||||
# ensure the directories exist
|
||||
mkdir -p "${CCACHE_DIR}"
|
||||
mkdir -p "${DEPENDS_DIR}/built"
|
||||
mkdir -p "${DEPENDS_DIR}/sources"
|
||||
mkdir -p "${PREVIOUS_RELEASES_DIR}"
|
||||
mkdir -p "${BASE_BUILD_DIR}" # Unset by default, must be defined externally
|
||||
|
||||
CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR"
|
||||
CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built"
|
||||
CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources"
|
||||
CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR"
|
||||
CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}"
|
||||
fi
|
||||
|
||||
if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then
|
||||
if [ ! -d "${CCACHE_DIR}" ]; then
|
||||
echo "Error: Directory '${CCACHE_DIR}' must be created in advance."
|
||||
exit 1
|
||||
fi
|
||||
CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}"
|
||||
fi
|
||||
|
||||
docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true
|
||||
|
||||
if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
|
||||
echo "Restart docker before run to stop and clear all containers started with --rm"
|
||||
podman container rm --force --all # Similar to "systemctl restart docker"
|
||||
|
||||
# Still prune everything in case the filtered pruning doesn't work, or if labels were not set
|
||||
# on a previous run. Belt and suspenders approach, should be fine to remove in the future.
|
||||
# Prune images used by --external containers (e.g. build containers) when
|
||||
# using podman.
|
||||
echo "Prune all dangling images"
|
||||
podman image prune --force --external
|
||||
fi
|
||||
echo "Prune all dangling $CI_IMAGE_LABEL images"
|
||||
# When detecting podman-docker, `--external` should be added.
|
||||
docker image prune --force --filter "label=$CI_IMAGE_LABEL"
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \
|
||||
--mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \
|
||||
--mount "${CI_CCACHE_MOUNT}" \
|
||||
--mount "${CI_DEPENDS_MOUNT}" \
|
||||
--mount "${CI_DEPENDS_SOURCES_MOUNT}" \
|
||||
--mount "${CI_PREVIOUS_RELEASES_MOUNT}" \
|
||||
${CI_BUILD_MOUNT} \
|
||||
--env-file /tmp/env-$USER-$CONTAINER_NAME \
|
||||
--name "$CONTAINER_NAME" \
|
||||
--network ci-ip6net \
|
||||
--platform="${CI_IMAGE_PLATFORM}" \
|
||||
"$CONTAINER_NAME")
|
||||
export CI_CONTAINER_ID
|
||||
export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}"
|
||||
else
|
||||
echo "Running on host system without docker wrapper"
|
||||
|
||||
Reference in New Issue
Block a user