Merge bitcoin/bitcoin#21652: ci: Switch remaining Linux tasks to self-hosted

fa8e89d5e48c4554eddef611eb002b61f3305272 ci: Remove distro-name from task name (MarcoFalke)
fad006fa0a8de7b527e9f7b50306921df7a2069a ci: Switch remaining tasks to self-hosted (MarcoFalke)

Pull request description:

  Cirrus CI will be capping the free compute soon. For now, switch more tasks to persistent worker, as recommended by Cirrus CI.

  (See slightly related discussion in https://github.com/bitcoin/bitcoin/issues/28098)

ACKs for top commit:
  pinheadmz:
    concept ACK fa8e89d5e48c4554eddef611eb002b61f3305272
  dergoegge:
    ACK fa8e89d5e48c4554eddef611eb002b61f3305272
  hebasto:
    ACK fa8e89d5e48c4554eddef611eb002b61f3305272.

Tree-SHA512: f6b172fee14856021b7a219b2490c8a163ad0137567c34a383080c68f8319b1d846923e508a968f43fb63ed6ce536dcb0611905fa288271f2267764b07bf9ecb

.cirrus.yml

@@ -8,9 +8,6 @@ env:  # Global defaults
   CCACHE_DIR: "/tmp/ccache_dir"
   CCACHE_NOHASHDIR: "1"  # Debug info might contain a stale path if the build dir changes, but this is fine
 
-cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-  DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
-
 # https://cirrus-ci.org/guide/persistent-workers/
 #
 # It is possible to select a specific persistent worker by label. Refer to the
@@ -66,25 +63,11 @@ base_template: &BASE_TEMPLATE
 
 main_template: &MAIN_TEMPLATE
   timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
-  ccache_cache:
-    folder: "/tmp/ccache_dir"
   ci_script:
     - ./ci/test_run_all.sh
 
-container_depends_template: &CONTAINER_DEPENDS_TEMPLATE
-  << : *BASE_TEMPLATE
-  container:
-    # https://cirrus-ci.org/faq/#are-there-any-limits
-    # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
-    cpu: 2
-    memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
-    dockerfile: ci/test_imagefile  # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment
-  base_depends_built_cache:
-    folder: "/ci_container_base/depends/built"
-    fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends)
-
 global_task_template: &GLOBAL_TASK_TEMPLATE
-  << : *CONTAINER_DEPENDS_TEMPLATE
+  << : *BASE_TEMPLATE
   << : *MAIN_TEMPLATE
 
 compute_credits_template: &CREDITS_TEMPLATE
@@ -93,7 +76,7 @@ compute_credits_template: &CREDITS_TEMPLATE
   use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != ""
 
 task:
-  name: 'lint [bookworm]'
+  name: 'lint'
   << : *BASE_TEMPLATE
   container:
     image: debian:bookworm
@@ -108,11 +91,9 @@ task:
     - git fetch --unshallow --no-tags
   lint_script:
     - ./ci/lint_run_all.sh
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: 'tidy [lunar]'
+  name: 'tidy'
   << : *GLOBAL_TASK_TEMPLATE
   persistent_worker:
     labels:
@@ -217,7 +198,7 @@ task:
     - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash
 
 task:
-  name: 'ARM [unit tests, no functional tests] [bullseye]'
+  name: 'ARM, unit tests, no functional tests'
   << : *GLOBAL_TASK_TEMPLATE
   persistent_worker:
     labels:
@@ -226,33 +207,25 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_arm.sh"
 
 task:
-  name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]'
+  name: 'Win64, unit tests, no gui tests, no boost::process, no functional tests'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
+  persistent_worker:
-    docker_arguments:
+    labels:
-      CI_IMAGE_NAME_TAG: ubuntu:jammy
+      type: small
+  env:
     FILE_ENV: "./ci/test/00_setup_env_win64.sh"
-  << : *CREDITS_TEMPLATE
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: '32-bit + dash [gui] [CentOS 9]'
+  name: '32-bit CentOS, dash, gui'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
+  persistent_worker:
-    docker_arguments:
+    labels:
-      CI_IMAGE_NAME_TAG: "quay.io/centos/amd64:stream9"
+      type: small
-      FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
-  # For faster CI feedback, immediately schedule one task that runs all tests
-  << : *CREDITS_TEMPLATE
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
-    PACKAGE_MANAGER_INSTALL: "yum install -y"
 
 task:
-  name: '[previous releases, qt5 dev package and depends packages, DEBUG] [focal]'
+  name: 'previous releases, qt5 dev package and depends packages, DEBUG'
-  previous_releases_cache:
-    folder: "releases"
   << : *GLOBAL_TASK_TEMPLATE
   persistent_worker:
     labels:
@@ -261,30 +234,26 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
 
 task:
-  name: '[TSan, depends, gui] [lunar]'
+  name: 'TSan, depends, gui'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
+  persistent_worker:
-    cpu: 4
+    labels:
-    memory: 16G  # The default memory is too small, so double everything
+      type: medium
-    docker_arguments:
+  env:
-      CI_IMAGE_NAME_TAG: ubuntu:lunar
     FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: '[MSan, depends] [jammy]'
+  name: 'MSan, depends'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
+  persistent_worker:
-    docker_arguments:
+    labels:
-      CI_IMAGE_NAME_TAG: ubuntu:jammy
+      type: small
-      FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
+  timeout_in: 300m  # Use longer timeout for the *rare* case where a full build (llvm + msan + depends + ...) needs to be done.
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
-    MAKEJOBS: "-j4"  # Avoid excessive memory use due to MSan
 
 task:
-  name: '[ASan + LSan + UBSan + integer, no depends, USDT] [lunar]'
+  name: 'ASan + LSan + UBSan + integer, no depends, USDT'
   enable_bpfcc_script:
     # In the image build step, no external environment variables are available,
     # so any settings will need to be written to the settings env file:
@@ -297,46 +266,37 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
 
 task:
-  name: '[fuzzer,address,undefined,integer, no depends] [lunar]'
+  name: 'fuzzer,address,undefined,integer, no depends'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
+  persistent_worker:
-    cpu: 4  # Increase CPU and memory to avoid timeout
+    labels:
-    memory: 16G
+      type: medium
-    docker_arguments:
+  env:
-      CI_IMAGE_NAME_TAG: ubuntu:lunar
     FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: '[multiprocess, i686, DEBUG] [focal]'
+  name: 'multiprocess, i686, DEBUG'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
+  persistent_worker:
-    cpu: 4
+    labels:
-    memory: 16G  # The default memory is too small, so double everything
+      type: medium
-    docker_arguments:
+  env:
-      CI_IMAGE_NAME_TAG: "docker.io/amd64/ubuntu:focal"
     FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: '[no wallet, libbitcoinkernel] [focal]'
+  name: 'no wallet, libbitcoinkernel'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
+  persistent_worker:
-    docker_arguments:
+    labels:
-      CI_IMAGE_NAME_TAG: ubuntu:focal
+      type: small
+  env:
     FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
-  << : *CREDITS_TEMPLATE
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: 'macOS 11.0 [gui, no tests] [jammy]'
+  name: 'macOS-cross 11.0, gui, no tests'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
+  persistent_worker:
-    docker_arguments:
+    labels:
-      CI_IMAGE_NAME_TAG: ubuntu:jammy
+      type: small
-      FILE_ENV: "./ci/test/00_setup_env_mac.sh"
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_mac.sh"

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ env:
 
 jobs:
   macos-native-x86_64:
-    name: macOS 13 native, x86_64 [no depends, sqlite only, gui]
+    name: 'macOS 13 native, x86_64, no depends, sqlite only, gui'
     # Use latest image, but hardcode version to avoid silent upgrades (and breaks).
     # See: https://github.com/actions/runner-images#available-images.
     runs-on: macos-13  # Use M1 once available https://github.com/github/roadmap/issues/528