From 2c33704491ee03188fc87d03c5b0aa8192c90bbf Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 3 Feb 2025 16:05:49 +0000 Subject: [PATCH 1/2] ci: run in worktrees Run CI in worktrees by mounting the .git root target into the container. --- ci/test/02_run_container.sh | 56 ++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh index 20910e57b72..b6b0d95f5e9 100755 --- a/ci/test/02_run_container.sh +++ b/ci/test/02_run_container.sh @@ -125,21 +125,51 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # When detecting podman-docker, `--external` should be added. docker image prune --force --filter "label=$CI_IMAGE_LABEL" + # Compile the docker run arguments into an array + docker_args=( + --cap-add LINUX_IMMUTABLE + ) + # Filter these optional vars out, which, if empty are passed through the bash + # array incorrectly as empty strings + if [[ -n "$CI_CONTAINER_CAP" ]]; then + read -ra cap_args <<< "$CI_CONTAINER_CAP" + docker_args+=("${cap_args[@]}") + fi + if [[ -n "$CI_BUILD_MOUNT" ]]; then + read -ra build_args <<< "$CI_BUILD_MOUNT" + docker_args+=("${build_args[@]}") + fi + # Main args + docker_args+=( + --rm + --interactive + --detach + --tty + --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" + --mount "${CI_CCACHE_MOUNT}" + --mount "${CI_DEPENDS_MOUNT}" + --mount "${CI_DEPENDS_SOURCES_MOUNT}" + --mount "${CI_PREVIOUS_RELEASES_MOUNT}" + ) + # Add worktree root mount, if needed + GIT_DIR=$(git rev-parse --git-dir) + if [[ "$GIT_DIR" == *".git/worktrees/"* ]]; then + WORKTREE_ROOT=$(echo "$GIT_DIR" | sed 's/\.git\/worktrees\/.*/\.git/') + docker_args+=(--mount "type=bind,src=$WORKTREE_ROOT,dst=$WORKTREE_ROOT,readonly") + fi # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME # to allow support starting multiple runs simultaneously by the same user. - # shellcheck disable=SC2086 - CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \ - --mount "${CI_CCACHE_MOUNT}" \ - --mount "${CI_DEPENDS_MOUNT}" \ - --mount "${CI_DEPENDS_SOURCES_MOUNT}" \ - --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \ - ${CI_BUILD_MOUNT} \ - --env-file /tmp/env-$USER-$CONTAINER_NAME \ - --name "$CONTAINER_NAME" \ - --network ci-ip6net \ - --platform="${CI_IMAGE_PLATFORM}" \ - "$CONTAINER_NAME") + docker_args+=( + --env-file "/tmp/env-$USER-$CONTAINER_NAME" + --name "$CONTAINER_NAME" + --network ci-ip6net + --platform="${CI_IMAGE_PLATFORM}" + ) + + docker_args+=("$CONTAINER_NAME") + # Execute command + CI_CONTAINER_ID=$(docker run "${docker_args[@]}") + export CI_CONTAINER_ID export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else From cc67e4595af0af2f304494e7811e700f25d48a22 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 3 Feb 2025 16:05:53 +0000 Subject: [PATCH 2/2] lint: run in worktrees Run linter in worktrees by mounting the .git root target into the container. Rename `get_git_root` to `get_git_toplevel` to avoid confusion around what this variable represents when modifying the code in the future. --- ci/lint_run_all.sh | 56 ++++++++++++++++++++++++++----- test/lint/README.md | 4 +-- test/lint/test_runner/src/main.rs | 12 +++---- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/ci/lint_run_all.sh b/ci/lint_run_all.sh index c57261d21a6..8c2004d88fe 100755 --- a/ci/lint_run_all.sh +++ b/ci/lint_run_all.sh @@ -6,12 +6,52 @@ export LC_ALL=C.UTF-8 -# Only used in .cirrus.yml. Refer to test/lint/README.md on how to run locally. +set -eo pipefail -cp "./ci/retry/retry" "/ci_retry" -cp "./.python-version" "/.python-version" -mkdir --parents "/test/lint" -cp --recursive "./test/lint/test_runner" "/test/lint/" -set -o errexit; source ./ci/lint/04_install.sh -set -o errexit -./ci/lint/06_script.sh +if [ -n "$CIRRUS_PR" ]; then + # For CI runs + cp "./ci/retry/retry" "/ci_retry" + cp "./.python-version" "/.python-version" + mkdir --parents "/test/lint" + cp --recursive "./test/lint/test_runner" "/test/lint/" + set -o errexit + # shellcheck source=/dev/null + source ./ci/lint/04_install.sh + set -o errexit + ./ci/lint/06_script.sh +else + # For local runs + error() { + echo "ERROR: $1" >&2 + exit 1 + } + + + echo "Building Docker image..." + DOCKER_BUILDKIT=1 docker build \ + -t bitcoin-linter \ + --file "./ci/lint_imagefile" \ + ./ || error "Docker build failed" + + echo "Running linter..." + DOCKER_ARGS=( + --rm + -v "$(pwd):/bitcoin" + -it + ) + + # Check if we are working in a worktree + GIT_DIR=$(git rev-parse --git-dir) + if [[ "$GIT_DIR" == *".git/worktrees/"* ]]; then + # If we are, mount the git toplevel dir to the expected location. + # This is required both so the linter knows which dir to run lint tests from. + # A writeable mount is required so test/lint/commit-script-check.sh can + # write to the index. + echo "Detected git worktree..." + WORKTREE_ROOT=$(echo "$GIT_DIR" | sed 's/\.git\/worktrees\/.*/\.git/') + echo "Worktree root: $WORKTREE_ROOT" + DOCKER_ARGS+=(--mount "type=bind,src=$WORKTREE_ROOT,dst=$WORKTREE_ROOT") + fi + + RUST_BACKTRACE=1 docker run "${DOCKER_ARGS[@]}" bitcoin-linter +fi diff --git a/test/lint/README.md b/test/lint/README.md index 8c1f0fedf07..eae9a69e345 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -4,10 +4,10 @@ Running locally =============== To run linters locally with the same versions as the CI environment, use the included -Dockerfile: +Dockerfile run via helper script to manage worktree workflows: ```sh -DOCKER_BUILDKIT=1 docker build -t bitcoin-linter --file "./ci/lint_imagefile" ./ && docker run --rm -v $(pwd):/bitcoin -it bitcoin-linter +./ci/lint_run_all.sh ``` Building the container can be done every time, because it is fast when the diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index 0d785682bb2..22a2e72c4d3 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -178,8 +178,8 @@ fn check_output(cmd: &mut std::process::Command) -> Result { .to_string()) } -/// Return the git root as utf8, or panic -fn get_git_root() -> PathBuf { +/// Return the git toplevel directory as utf8, or panic +fn get_git_toplevel() -> PathBuf { PathBuf::from(check_output(git().args(["rev-parse", "--show-toplevel"])).unwrap()) } @@ -691,7 +691,7 @@ Markdown link errors found: fn run_all_python_linters() -> LintResult { let mut good = true; - let lint_dir = get_git_root().join("test/lint"); + let lint_dir = get_git_toplevel().join("test/lint"); for entry in fs::read_dir(lint_dir).unwrap() { let entry = entry.unwrap(); let entry_fn = entry.file_name().into_string().unwrap(); @@ -723,7 +723,7 @@ fn main() -> ExitCode { get_linter_list() }; - let git_root = get_git_root(); + let git_toplevel = get_git_toplevel(); let commit_range = commit_range(); let commit_log = check_output(git().args(["log", "--no-merges", "--oneline", &commit_range])) .expect("check_output failed"); @@ -731,8 +731,8 @@ fn main() -> ExitCode { let mut test_failed = false; for linter in linters_to_run { - // chdir to root before each lint test - env::set_current_dir(&git_root).unwrap(); + // chdir to git toplevel before each lint test + env::set_current_dir(&git_toplevel).unwrap(); if let Err(err) = (linter.lint_fn)() { println!( "^^^\n{err}\n^---- ⚠️ Failure generated from lint check '{}' ({})!\n\n",